@fabrika/app 0.9.0-dev.4 → 0.9.0-dev.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +41 -14
  3. package/skill/references/app-setup.md +43 -12
  4. package/skill/references/dev.md +2 -1
  5. package/skill/references/endpoint.md +30 -24
  6. package/skill/references/errors.md +40 -35
  7. package/skill/references/files.md +4 -2
  8. package/skill/references/jobs.md +21 -4
  9. package/skill/references/table-and-migration.md +19 -2
  10. package/skill/references/tests.md +40 -1
  11. package/skill/references/webhooks-links-connectors.md +1 -1
  12. package/src/build-recipe.ts +198 -0
  13. package/src/build.ts +60 -4
  14. package/src/bun.ts +181 -13
  15. package/src/cloudflare/application.ts +822 -0
  16. package/src/cloudflare/bindings.ts +159 -0
  17. package/src/cloudflare/export.ts +641 -0
  18. package/src/cloudflare/import.ts +1021 -0
  19. package/src/cloudflare/maintenance.ts +518 -0
  20. package/src/cloudflare/names.ts +16 -0
  21. package/src/cloudflare/ops.ts +375 -0
  22. package/src/cloudflare/r2-files.ts +172 -0
  23. package/src/cloudflare/restore.ts +167 -0
  24. package/src/cloudflare/tracker.ts +192 -0
  25. package/src/cloudflare.ts +100 -146
  26. package/src/contract/app-id.ts +5 -0
  27. package/src/contract/artifact-schema.ts +9 -1
  28. package/src/contract/build.ts +40 -2
  29. package/src/contract/compile.ts +1 -0
  30. package/src/contract/context.ts +35 -9
  31. package/src/contract/declaration.ts +2 -0
  32. package/src/contract/defaults.ts +22 -2
  33. package/src/contract/define.ts +29 -3
  34. package/src/contract/kit.ts +4 -1
  35. package/src/contract/platform.ts +3 -2
  36. package/src/contract/vocabulary.ts +26 -8
  37. package/src/dev/launcher.ts +50 -4
  38. package/src/disk-assets.ts +8 -3
  39. package/src/files/archive.ts +2 -2
  40. package/src/files/job-names.ts +12 -0
  41. package/src/files/kinds.ts +73 -31
  42. package/src/files/scan.ts +3 -4
  43. package/src/files/streams.ts +3 -1
  44. package/src/implement/anonymous.ts +12 -11
  45. package/src/implement/boot.ts +75 -6
  46. package/src/implement/bounded.ts +21 -2
  47. package/src/implement/capability.ts +15 -3
  48. package/src/implement/dispatch.ts +419 -177
  49. package/src/implement/failures.ts +17 -1
  50. package/src/implement/faults.ts +1 -0
  51. package/src/implement/index.ts +1 -0
  52. package/src/implement/invocation.ts +58 -10
  53. package/src/implement/principal.ts +25 -2
  54. package/src/implement/terminals.ts +93 -0
  55. package/src/implement/workload.ts +42 -16
  56. package/src/jobs/mail.ts +8 -0
  57. package/src/jobs/report.ts +5 -1
  58. package/src/jobs/runner.ts +672 -113
  59. package/src/jobs/schedule.ts +7 -1
  60. package/src/jobs/scrub.ts +14 -1
  61. package/src/jobs/types.ts +6 -0
  62. package/src/spa-fallback.ts +12 -0
  63. package/src/testing/endpoint-access.ts +307 -152
  64. package/src/testing/isolation.ts +8 -1
  65. package/src/testing/kit.ts +80 -2
  66. package/src/testing/proxy.ts +15 -10
  67. package/types/src/build-recipe.d.ts +70 -0
  68. package/types/src/build.d.ts +19 -1
  69. package/types/src/bun.d.ts +34 -1
  70. package/types/src/cloudflare/application.d.ts +73 -0
  71. package/types/src/cloudflare/bindings.d.ts +33 -0
  72. package/types/src/cloudflare/export.d.ts +16 -0
  73. package/types/src/cloudflare/import.d.ts +51 -0
  74. package/types/src/cloudflare/maintenance.d.ts +117 -0
  75. package/types/src/cloudflare/names.d.ts +11 -0
  76. package/types/src/cloudflare/ops.d.ts +81 -0
  77. package/types/src/cloudflare/r2-files.d.ts +9 -0
  78. package/types/src/cloudflare/restore.d.ts +9 -0
  79. package/types/src/cloudflare/tracker.d.ts +29 -0
  80. package/types/src/cloudflare.d.ts +12 -18
  81. package/types/src/contract/app-id.d.ts +2 -0
  82. package/types/src/contract/artifact-schema.d.ts +2 -0
  83. package/types/src/contract/context.d.ts +35 -11
  84. package/types/src/contract/declaration.d.ts +4 -0
  85. package/types/src/contract/defaults.d.ts +19 -1
  86. package/types/src/contract/define.d.ts +20 -3
  87. package/types/src/contract/kit.d.ts +2 -1
  88. package/types/src/contract/platform.d.ts +2 -2
  89. package/types/src/contract/vocabulary.d.ts +9 -7
  90. package/types/src/disk-assets.d.ts +4 -1
  91. package/types/src/files/job-names.d.ts +9 -0
  92. package/types/src/files/kinds.d.ts +21 -14
  93. package/types/src/files/scan.d.ts +1 -3
  94. package/types/src/implement/anonymous.d.ts +2 -4
  95. package/types/src/implement/boot.d.ts +17 -1
  96. package/types/src/implement/bounded.d.ts +11 -1
  97. package/types/src/implement/capability.d.ts +4 -2
  98. package/types/src/implement/dispatch.d.ts +34 -1
  99. package/types/src/implement/failures.d.ts +5 -0
  100. package/types/src/implement/faults.d.ts +1 -0
  101. package/types/src/implement/index.d.ts +1 -0
  102. package/types/src/implement/invocation.d.ts +22 -7
  103. package/types/src/implement/principal.d.ts +6 -1
  104. package/types/src/implement/terminals.d.ts +33 -0
  105. package/types/src/implement/workload.d.ts +5 -0
  106. package/types/src/jobs/runner.d.ts +66 -2
  107. package/types/src/jobs/schedule.d.ts +2 -0
  108. package/types/src/jobs/types.d.ts +6 -0
  109. package/types/src/spa-fallback.d.ts +2 -0
  110. package/types/src/testing/endpoint-access.d.ts +37 -5
  111. package/types/src/testing/isolation.d.ts +1 -1
  112. package/types/src/testing/kit.d.ts +22 -1
  113. package/types/src/testing/proxy.d.ts +4 -4
@@ -3,13 +3,25 @@
3
3
  // that timed out, failed or lost the lease never commits (`@fabrika/db`'s fence). A site's `ctx.kick` sends a run in
4
4
  // the background after its commit, without waiting for any lock; a kick that finds the lease held leaves the shared pending flag, and the run
5
5
  // holding it runs once more after it succeeded. Failures, crashed runs and a job that stopped succeeding are reported
6
- // to Operations, each under a fixed failure code.
6
+ // to Operations, each under a fixed failure code. On Cloudflare the object's alarm drives the jobs through `scheduler`:
7
+ // a run it starts owns the job's due wakes, and when it fails they move to a durable retry instead of an inline one.
7
8
 
8
9
  import type { IamMembership } from '@fabrika/auth'
9
10
  import { type ContractArtifact, JOB_SITE_PREFIX, type JobArtifact, type JobMailer, type PermissionEntry, permits } from '@fabrika/auth-core'
10
11
  import type { CapabilityAuthority } from '@fabrika/db/capability'
11
- import { type DataAccess, type FrameworkTables, type JobLeases, type LeaseFence, LeaseLostError, type Session } from '@fabrika/db/framework'
12
+ import {
13
+ type DataAccess,
14
+ type FinishWakes,
15
+ type FrameworkTables,
16
+ type JobLeases,
17
+ type JobWakes,
18
+ type LeaseFence,
19
+ LeaseLostError,
20
+ type Session,
21
+ type Wake,
22
+ } from '@fabrika/db/framework'
12
23
  import { EmailDeliveryError, type EmailSenderCapabilities, mayRetryDelivery } from '@fabrika/email'
24
+ import { uuidv7 } from '@fabrika/platform'
13
25
  import { NotFoundError } from '../errors'
14
26
  import { FileWork, type StoredFileKinds, storedFilesHandle } from '../files/kinds'
15
27
  import {
@@ -21,6 +33,7 @@ import {
21
33
  runInRequest,
22
34
  type Within,
23
35
  } from '../implement/capability'
36
+ import { admitInvocation, type InvocationAdmission } from '../implement/dispatch'
24
37
  import { runAsSite } from '../implement/egress'
25
38
  import { failureCode, type FailureReason, failureReason } from '../implement/failures'
26
39
  import { ServerFault } from '../implement/faults'
@@ -33,6 +46,7 @@ import {
33
46
  jobHandles,
34
47
  jobSql,
35
48
  jobSqlOutsideTransaction,
49
+ type KickRecorder,
36
50
  kickRecorder,
37
51
  siteData,
38
52
  type SiteDeclaration,
@@ -41,10 +55,10 @@ import {
41
55
  import { jobLinkIssuing, type JobLinkMount } from '../implement/membership'
42
56
  import type { SignedTokens } from '../implement/tokens'
43
57
  import { type AppUrl, appUrl } from '../implement/urls'
44
- import { type WorkloadCredential, workloadSite } from '../implement/workload'
58
+ import { type WorkloadAuthority, type WorkloadCredential, workloadSite } from '../implement/workload'
45
59
  import { jobMail, type MailTransports } from './mail'
46
60
  import type { OperationsReporter } from './report'
47
- import { type CronSchedule, parseCron, periodAt } from './schedule'
61
+ import { type CronSchedule, isDueAt, parseCron, periodAt } from './schedule'
48
62
  import type { AppMail, JobContext, JobPrincipal } from './types'
49
63
 
50
64
  /**
@@ -54,6 +68,11 @@ import type { AppMail, JobContext, JobPrincipal } from './types'
54
68
  export const LEASE_GRACE_MS = 60_000
55
69
  /** How often the serving process checks every job for a crashed run and for a schedule that stopped. */
56
70
  export const JOB_CHECK_INTERVAL_MS = 60_000
71
+ /**
72
+ * A run the Cloudflare alarm started has one segment (`CLOUDFLARE_MAX_JOB_SEGMENT_MS`): its attempt ends this much
73
+ * before the segment, at most a tenth of it, so the finish runs inside the segment too.
74
+ */
75
+ export const SEGMENT_FINISH_RESERVE_MS = 30_000
57
76
  const BACKOFF_BASE_MS = 1_000
58
77
  const BACKOFF_MAX_MS = 5 * 60_000
59
78
 
@@ -75,7 +94,7 @@ export type JobFailureCode =
75
94
  | 'unauthorized'
76
95
  /** IAM could not be asked for the job's token. */
77
96
  | 'iam_unavailable'
78
- /** The process stopped a kicked run while shutting down; the schedule runs the job again. */
97
+ /** The process stopped a kicked run while shutting down, or the object's maintenance stopped a run; the schedule runs it again. */
79
98
  | 'stopped'
80
99
  /** The transport refused a message the handler did not catch; `reason` is the `EmailDeliveryError` code. */
81
100
  | 'mail_failed'
@@ -93,6 +112,8 @@ export interface JobClock {
93
112
  wait(ms: number, signal: AbortSignal): Promise<void>
94
113
  /** In [0, 1): the jitter of a retry's backoff. */
95
114
  random(): number
115
+ /** Epoch milliseconds: what an attempt's deadline is measured on; `Date.now()` when absent. */
116
+ now?(): number
96
117
  }
97
118
 
98
119
  export interface JobRunOptions {
@@ -117,12 +138,67 @@ export interface FrameworkJob {
117
138
  export interface ContractJobs {
118
139
  /** The contract's `c.cron` jobs; `run` also takes the name of a framework job the contract needs. */
119
140
  readonly names: readonly string[]
141
+ /** Every job bound to run, the framework's included, with its schedule: what a minute tick runs when due. */
142
+ readonly scheduled: readonly { readonly name: string; readonly schedule: CronSchedule }[]
120
143
  run(name: string, options?: JobRunOptions): Promise<JobRunResult>
121
144
  /**
122
145
  * Registers every job (its first-seen time), reports a run that crashed and a job that has not succeeded within
123
146
  * twice its period. The serving process calls it every `JOB_CHECK_INTERVAL_MS`; it never throws.
124
147
  */
125
148
  check(options?: Pick<JobRunOptions, 'reporter'>): Promise<void>
149
+ /**
150
+ * The serving process's wake timer on Bun, where crontab runs the schedule: reaps a crashed holder of every job and
151
+ * kicks each contract job with a due wake through the kick path, handing each kick to `track` until it ended. Answers
152
+ * how long until a wake is due next on the database's clock (0 while one still is), `null` when none is pending. The
153
+ * first call deletes the wakes of jobs the contract no longer declares.
154
+ */
155
+ wake(track: (kick: Promise<void>) => void): Promise<number | null>
156
+ /** What the Cloudflare object's alarm drives the jobs through; `null` without `@fabrika/db`. */
157
+ readonly scheduler: JobScheduler | null
158
+ }
159
+
160
+ /** What the scheduler's `take` did for one job in the alarm's transaction. */
161
+ export interface TakenLease {
162
+ /** The lease a run of the job now holds, for `run`; `null` when nothing was due or another run holds it. */
163
+ readonly fence: LeaseFence | null
164
+ /** Whether a crashed holder's lease was freed: `crashed` reports it once the transaction committed. */
165
+ readonly reaped: boolean
166
+ }
167
+
168
+ /**
169
+ * The jobs as the Cloudflare object's alarm drives them (ADR-0065 §5): over the wakes and leases of the object's
170
+ * storage, every time on the database's clock. The object holds the transactions and arms the alarm.
171
+ */
172
+ export interface JobScheduler {
173
+ /** Every job bound to run, the framework's included. */
174
+ readonly jobs: readonly string[]
175
+ /**
176
+ * Startup repair: reaps and reports every crashed run, of a job the contract no longer declares too; registers every
177
+ * job and surveils it (as `check`), deletes the wakes of jobs the contract no longer declares and the cron wakes of a
178
+ * changed expression, and keeps one cron wake per job at its next fire; an overdue one stays, so missed runs coalesce
179
+ * into one. `false` when the object's barrier admits no job.
180
+ */
181
+ repair(options?: Pick<JobRunOptions, 'reporter'>): Promise<boolean>
182
+ /** The jobs the contract no longer declares whose lease a crashed run still holds: `take` reaps them. */
183
+ orphans(): Promise<readonly string[]>
184
+ /** How long until every wake due within `withinMs` of the database's clock is due; 0 when none is early. */
185
+ earlyBy(withinMs: number): Promise<number>
186
+ /**
187
+ * Reaps a crashed holder of `job`'s lease, then, when one of its wakes is due, takes the lease, or sets the pending
188
+ * flag for the run holding it; of a job no longer declared (an orphan), only reaps. Statements only, for the caller's
189
+ * one transaction.
190
+ */
191
+ take(job: string): Promise<TakenLease>
192
+ /** Runs `job` under the lease `take` took. A failed run retries the wakes it owns durably. */
193
+ run(job: string, fence: LeaseFence, options?: JobRunOptions): Promise<JobRunResult>
194
+ /** Reports the crashed run `take` reaped. */
195
+ crashed(job: string, options?: Pick<JobRunOptions, 'reporter'>): Promise<void>
196
+ /** When the scheduler wakes next: `JobWakes.next`, read in `using` when given. */
197
+ next(using?: Session): Promise<Date | null>
198
+ /** How far the next wake is from the database's now, in milliseconds: `JobWakes.untilNext`, read in `using` when given. */
199
+ untilNext(using?: Session): Promise<number | null>
200
+ /** Reports a failure of the scheduler itself: the alarm's or the repair's. Never rejects. */
201
+ failed(step: 'alarm' | 'repair', error: unknown): Promise<void>
126
202
  }
127
203
 
128
204
  /** A run that cannot start: an unknown job. Its message is framework-authored. */
@@ -131,6 +207,7 @@ export class JobError extends Error {
131
207
  }
132
208
 
133
209
  export const systemClock: JobClock = Object.freeze({
210
+ now: () => Date.now(),
134
211
  wait: (ms: number, signal: AbortSignal) =>
135
212
  new Promise<void>((resolve) => {
136
213
  if (signal.aborted) {
@@ -171,6 +248,8 @@ interface RunnerContext {
171
248
  readonly database: ContractDatabase | null
172
249
  readonly ownedDimensions: ReadonlySet<string>
173
250
  readonly leases: JobLeases
251
+ /** The jobs' pending wakes, which a run's finish consumes, retries or schedules. */
252
+ readonly wakes: JobWakes
174
253
  readonly reporter: OperationsReporter
175
254
  readonly mint: CapabilityMint
176
255
  readonly data: DataAccess | null
@@ -307,11 +386,22 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
307
386
  }
308
387
  const names = Object.freeze(entries.map(([name]) => name))
309
388
  const framework = input.framework
310
- const contextFor = (leases: JobLeases, chosen: OperationsReporter | undefined): RunnerContext => ({
389
+ const declaredJobs: readonly (Scheduled & { readonly timeoutMs: number })[] = [
390
+ ...[...bound.values()].map((job) => ({ name: job.name, expression: job.artifact.schedule, schedule: job.schedule, timeoutMs: job.timeoutMs })),
391
+ ...[...frameworkJobs.values()].map((own) => ({
392
+ name: own.name,
393
+ expression: own.job.schedule,
394
+ schedule: own.schedule,
395
+ timeoutMs: own.job.timeoutMs,
396
+ })),
397
+ ]
398
+ let reconciled = false
399
+ const contextFor = (tables: FrameworkTables, chosen: OperationsReporter | undefined): RunnerContext => ({
311
400
  artifact: input.artifact,
312
401
  database: input.database,
313
402
  ownedDimensions: input.ownedDimensions,
314
- leases,
403
+ leases: tables.jobs,
404
+ wakes: tables.wakes,
315
405
  reporter: chosen ?? input.reporter,
316
406
  mint: input.mint,
317
407
  data: input.data,
@@ -328,60 +418,151 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
328
418
  })
329
419
  const jobs: ContractJobs = Object.freeze({
330
420
  names,
421
+ scheduled: Object.freeze([...bound.values(), ...frameworkJobs.values()].map(({ name, schedule }) => ({ name, schedule }))),
331
422
  run(name: string, options: JobRunOptions = {}): Promise<JobRunResult> {
332
423
  const own = frameworkJobs.get(name)
333
424
  if (own !== undefined && framework !== null) {
334
- return runFrameworkJob(own, contextFor(framework.jobs, options.reporter), options.clock ?? systemClock)
425
+ return admitted(
426
+ 'framework-job',
427
+ name,
428
+ (control) => runFrameworkJob(own, contextFor(framework, options.reporter), options.clock ?? systemClock, control, null),
429
+ )
335
430
  }
336
431
  const job = bound.get(name)
337
432
  if (job === undefined || framework === null) {
338
433
  throw new JobError(`no job '${name}' in the contract (jobs: ${names.length === 0 ? 'none' : names.join(', ')})`)
339
434
  }
340
- return runJob(job, contextFor(framework.jobs, options.reporter), options.clock ?? systemClock, 'wait').then((result) => {
341
- if (result.status === 'skipped') {
342
- console.error(`fabrika: a previous run of job '${name}' still holds its lease; this run does not start`)
343
- }
344
- return result
345
- })
435
+ return admitted(
436
+ 'job',
437
+ name,
438
+ (control) => runJob(job, contextFor(framework, options.reporter), options.clock ?? systemClock, 'wait', control, 'inline'),
439
+ )
440
+ .then((result) => {
441
+ if (result.status === 'skipped') {
442
+ console.error(`fabrika: a previous run of job '${name}' still holds its lease; this run does not start`)
443
+ }
444
+ return result
445
+ })
346
446
  },
347
447
  async check(options: Pick<JobRunOptions, 'reporter'> = {}): Promise<void> {
348
- if (framework === null) {
448
+ const admission = framework === null ? null : admitInvocation('framework-job')
449
+ if (framework === null || admission === null) {
349
450
  return
350
451
  }
351
- const context = contextFor(framework.jobs, options.reporter)
352
- for (const job of [...bound.values(), ...frameworkJobs.values()]) {
353
- try {
354
- await context.leases.register(job.name)
355
- await surveil(job, context)
356
- } catch {
357
- // A driver error may quote the database URL.
358
- console.error(`fabrika: the check of job '${job.name}' failed: driver_failed`)
452
+ try {
453
+ const context = contextFor(framework, options.reporter)
454
+ for (const job of [...bound.values(), ...frameworkJobs.values()]) {
455
+ try {
456
+ await context.leases.register(job.name)
457
+ await surveil(job, context)
458
+ } catch {
459
+ // A driver error may quote the database URL.
460
+ console.error(`fabrika: the check of job '${job.name}' failed: driver_failed`)
461
+ }
462
+ }
463
+ } finally {
464
+ admission.release()
465
+ }
466
+ },
467
+ async wake(track: (kick: Promise<void>) => void): Promise<number | null> {
468
+ if (framework === null) {
469
+ return null
470
+ }
471
+ if (!reconciled) {
472
+ await framework.wakes.reconcile(declaredJobs.map((job) => ({ job: job.name, expression: job.expression })))
473
+ reconciled = true
474
+ }
475
+ // A crashed holder's expiry is a wake too, and a kick never takes a lease over.
476
+ for (const job of declaredJobs) {
477
+ if (await framework.jobs.reapCrashed(job.name)) {
478
+ await reportCrashed(job.name, input.reporter)
479
+ }
480
+ }
481
+ const now = (await framework.jobs.now()).getTime()
482
+ for (const job of bound.values()) {
483
+ const due = (await framework.wakes.pending(job.name)).filter((wake) => wake.dueAt.getTime() <= now)
484
+ if (due.length > 0 && !(await ownedByHolder(job.name, now, framework.jobs, framework.wakes))) {
485
+ track(kick(job.name))
359
486
  }
360
487
  }
488
+ const next = await framework.wakes.next()
489
+ return next === null ? null : Math.max(0, next.getTime() - now)
361
490
  },
491
+ scheduler: framework === null ? null : jobScheduler({
492
+ framework,
493
+ declared: declaredJobs,
494
+ contextFor: (reporter) => contextFor(framework, reporter),
495
+ run(name: string, fence: LeaseFence, options: JobRunOptions): Promise<JobRunResult> {
496
+ const clock = options.clock ?? systemClock
497
+ const context = contextFor(framework, options.reporter)
498
+ const own = frameworkJobs.get(name)
499
+ if (own !== undefined) {
500
+ return admitted('framework-job', name, (control) => runFrameworkJob(own, context, clock, control, fence))
501
+ }
502
+ const job = bound.get(name)
503
+ if (job === undefined) {
504
+ throw new JobError(`no job '${name}' in the contract (jobs: ${names.length === 0 ? 'none' : names.join(', ')})`)
505
+ }
506
+ return admitted('job', name, (control) => runJob(job, context, clock, { fence }, control, 'durable'))
507
+ },
508
+ }),
362
509
  })
363
510
  const stopping = new AbortController()
364
511
  const kick = async (name: string): Promise<void> => {
365
512
  const job = bound.get(name)
366
- if (job !== undefined && framework !== null) {
367
- await kickOnce(job, contextFor(framework.jobs, undefined), stopping.signal)
513
+ if (job === undefined || framework === null) {
514
+ return
515
+ }
516
+ const admission = admitInvocation('kick')
517
+ if (admission === null) {
518
+ // A kick hurries a run; the schedule still runs the job once the object opens.
519
+ console.error(`fabrika: job '${name}' was not kicked: the object is in maintenance`)
520
+ return
521
+ }
522
+ try {
523
+ await kickOnce(job, contextFor(framework, undefined), {
524
+ signal: AbortSignal.any([admission.signal, stopping.signal]),
525
+ track: admission.track,
526
+ })
527
+ } finally {
528
+ admission.release()
368
529
  }
369
530
  }
370
531
  return { jobs, declared, kick, stopKicks: () => stopping.abort() }
371
532
  }
372
533
 
534
+ /**
535
+ * What stops one run, and where the handler work that outlives its attempt is counted: the run's admission at the
536
+ * object's barrier, which a drain aborts, and for a kick also the process's stop.
537
+ */
538
+ type RunControl = Pick<InvocationAdmission, 'signal' | 'track'>
539
+
540
+ /** A run admitted at the current event's gate, released once it ended; a refused one is `stopped` and starts nothing. */
541
+ async function admitted(kind: 'job' | 'framework-job', name: string, run: (control: RunControl) => Promise<JobRunResult>): Promise<JobRunResult> {
542
+ const admission = admitInvocation(kind)
543
+ if (admission === null) {
544
+ console.error(`fabrika: job '${name}' did not start: the object is in maintenance; its schedule runs it again`)
545
+ return { status: 'failed', attempts: 0, code: 'stopped' }
546
+ }
547
+ try {
548
+ return await run(admission)
549
+ } finally {
550
+ admission.release()
551
+ }
552
+ }
553
+
373
554
  /**
374
555
  * One kick: a run while the lease is free, else the shared pending flag. Neither waits for a lock, so a kick never
375
556
  * holds a connection requests share while a run writes. After a failed run the flag stays for the schedule.
376
557
  */
377
- async function kickOnce(job: BoundJob, context: RunnerContext, stop: AbortSignal): Promise<void> {
378
- if (stop.aborted) {
558
+ async function kickOnce(job: BoundJob, context: RunnerContext, control: RunControl): Promise<void> {
559
+ if (control.signal.aborted) {
379
560
  // The flag waits for no lock, and another process that is not stopping runs the job.
380
561
  await markPending(job, context)
381
562
  return
382
563
  }
383
- if ((await runJob(job, context, systemClock, 'try', stop)).status === 'skipped') {
384
- await handOver(job, context, systemClock, stop)
564
+ if ((await runJob(job, context, systemClock, 'try', control, 'inline')).status === 'skipped') {
565
+ await handOver(job, context, systemClock, control, 'inline')
385
566
  }
386
567
  }
387
568
 
@@ -389,11 +570,11 @@ async function kickOnce(job: BoundJob, context: RunnerContext, stop: AbortSignal
389
570
  * Leaves the flag to the run holding the lease, which takes it after it succeeded. That run may have finished before
390
571
  * the flag was set, so the lease is tried once more; `null` when another run still holds it.
391
572
  */
392
- async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock, stop?: AbortSignal): Promise<JobRunResult | null> {
573
+ async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock, control: RunControl, retries: Retries): Promise<JobRunResult | null> {
393
574
  if (!(await markPending(job, context))) {
394
575
  return null
395
576
  }
396
- const result = await runJob(job, context, clock, 'try', stop)
577
+ const result = await runJob(job, context, clock, 'try', control, retries)
397
578
  return result.status === 'skipped' ? null : result
398
579
  }
399
580
 
@@ -432,7 +613,7 @@ function jobLinks(where: string, site: string, declaration: SiteDeclaration | nu
432
613
  return { name, prefix: mount.prefix, dimension: mount.dimension, issuedBy: mount.issuedBy }
433
614
  }
434
615
 
435
- const attemptId = (): string => Bun.randomUUIDv7()
616
+ const attemptId = (): string => uuidv7()
436
617
 
437
618
  /** Exponential from one second, capped at five minutes, with half of it jittered. */
438
619
  function backoffMs(attempt: number, random: number): number {
@@ -440,13 +621,123 @@ function backoffMs(attempt: number, random: number): number {
440
621
  return Math.round(ceiling / 2 + (ceiling / 2) * random)
441
622
  }
442
623
 
624
+ const nowOf = (clock: JobClock): number => clock.now?.() ?? Date.now()
625
+
626
+ /** How much of a segment that ends at `deadline` is left, never below zero. */
627
+ const remainingMs = (clock: JobClock, deadline: number): number => Math.max(0, deadline - nowOf(clock))
628
+
629
+ const finishReserveMs = (timeoutMs: number): number => Math.min(SEGMENT_FINISH_RESERVE_MS, Math.floor(timeoutMs / 10))
630
+
631
+ const OUT_OF_SEGMENT: unique symbol = Symbol('fabrika.job.outOfSegment')
632
+
633
+ /**
634
+ * `step` while the segment that ends at `end` lasts (always, without one). Past its end the run stops waiting for it;
635
+ * only a step that is safe to abandon goes here: a read, or the refusal of the attempt's writes, never a finish.
636
+ */
637
+ async function withinSegment<T>(clock: JobClock, end: number | null, step: Promise<T>): Promise<T | typeof OUT_OF_SEGMENT> {
638
+ if (end === null) {
639
+ return await step
640
+ }
641
+ const timer = new AbortController()
642
+ // Abandoned past the segment, its rejection is nobody's: the race below still answers it while it is awaited.
643
+ step.catch(() => undefined)
644
+ try {
645
+ return await Promise.race([step, clock.wait(remainingMs(clock, end), timer.signal).then((): typeof OUT_OF_SEGMENT => OUT_OF_SEGMENT)])
646
+ } finally {
647
+ timer.abort()
648
+ }
649
+ }
650
+
651
+ /** A run whose segment ended before its finish started: its lease expires, the reap reports the crash, and the alarm runs it again. */
652
+ function outOfSegment(job: { readonly name: string }, attempts: number): JobRunResult {
653
+ console.error(`fabrika: job '${job.name}' reached the end of its segment before its finish; its lease expires and the alarm runs it again`)
654
+ return { status: 'failed', attempts, code: 'timeout' }
655
+ }
656
+
657
+ const MINUTE_MS = 60_000
658
+ /** How far ahead a next fire is looked for, as far as `previousDue` looks back. */
659
+ const NEXT_FIRE_DAYS = 366 * 5
660
+
661
+ /** The first due minute of `schedule` after `after`, or `null` when none falls in the window. */
662
+ function nextFire(schedule: CronSchedule, after: Date): Date | null {
663
+ const hours = [...schedule.hours].sort((left, right) => left - right)
664
+ const minutes = [...schedule.minutes].sort((left, right) => left - right)
665
+ const [firstHour, firstMinute] = [hours[0], minutes[0]]
666
+ if (firstHour === undefined || firstMinute === undefined) {
667
+ return null
668
+ }
669
+ const today = Date.UTC(after.getUTCFullYear(), after.getUTCMonth(), after.getUTCDate())
670
+ for (let ahead = 0; ahead <= NEXT_FIRE_DAYS; ahead++) {
671
+ const day = today + ahead * 24 * 60 * MINUTE_MS
672
+ // The day's first scheduled minute is due exactly when the day matches.
673
+ if (!isDueAt(schedule, new Date(day + (firstHour * 60 + firstMinute) * MINUTE_MS))) {
674
+ continue
675
+ }
676
+ for (const hour of hours) {
677
+ for (const minute of minutes) {
678
+ const time = day + (hour * 60 + minute) * MINUTE_MS
679
+ if (time > after.getTime()) {
680
+ return new Date(time)
681
+ }
682
+ }
683
+ }
684
+ }
685
+ return null
686
+ }
687
+
688
+ /** A cron wake's dedupe key: one pending wake per job and expression. */
689
+ const cronKey = (job: string, expression: string): string => `cron:${job}:${expression}`
690
+
691
+ /** A job's schedule as the scheduler and a finish read it. */
692
+ interface Scheduled {
693
+ readonly name: string
694
+ /** The cron expression as declared: its wake's key. */
695
+ readonly expression: string
696
+ readonly schedule: CronSchedule
697
+ }
698
+
699
+ /**
700
+ * Whether a run holds `job`'s lease and owns every wake due by `now`, as it took them with the lease: its finish consumes
701
+ * them, so a kick or a pending run for them would only repeat that run. A post-commit kick's run usually owns its wake.
702
+ */
703
+ async function ownedByHolder(job: string, now: number, leases: JobLeases, wakes: JobWakes): Promise<boolean> {
704
+ const state = await leases.state(job)
705
+ if (state === null || state.holder === null) {
706
+ return false
707
+ }
708
+ return (await wakes.pending(job)).filter((wake) => wake.dueAt.getTime() <= now).every((wake) => wake.owned)
709
+ }
710
+
711
+ /** The wakes the run holding `job`'s lease owns: those it took with the lease. */
712
+ async function ownedWakes(job: string, context: RunnerContext): Promise<readonly Wake[]> {
713
+ return (await context.wakes.pending(job)).filter((wake) => wake.owned)
714
+ }
715
+
716
+ /**
717
+ * What a finish does with the owned wakes when the run is done with them, succeeded or failed past its retries: they
718
+ * are consumed, and when the job keeps a cron wake (always, for a run the alarm started) its next fire is scheduled
719
+ * under its key. On Bun, where crontab runs the schedule, a job keeps none.
720
+ */
721
+ async function completion(job: Scheduled, context: RunnerContext, durable: boolean): Promise<FinishWakes> {
722
+ if (!durable && !(await context.wakes.pending(job.name)).some((wake) => wake.reason === 'cron')) {
723
+ return { kind: 'completed' }
724
+ }
725
+ const dueAt = nextFire(job.schedule, await context.leases.now())
726
+ return dueAt === null ? { kind: 'completed' } : { kind: 'completed', nextCron: { expression: job.expression, dueAt } }
727
+ }
728
+
729
+ /** A crashed run (its lease expired before it finished): one report. */
730
+ async function reportCrashed(job: string, reporter: OperationsReporter): Promise<void> {
731
+ const message = `job '${job}' failed: crashed (its run's lease expired before the run finished)`
732
+ console.error(`fabrika: ${message}`)
733
+ await reporter.report({ fingerprint: ['fabrika-job-failed', job], type: 'JobFailed', message, tags: { job, code: 'crashed' } })
734
+ }
735
+
443
736
  /** A crashed run (its lease expired before it finished) and a job overdue by twice its period: one report each. */
444
737
  async function surveil(job: { readonly name: string; readonly schedule: CronSchedule }, context: RunnerContext): Promise<void> {
445
738
  const tags = { job: job.name }
446
739
  if (await context.leases.reapCrashed(job.name)) {
447
- const message = `job '${job.name}' failed: crashed (its run's lease expired before the run finished)`
448
- console.error(`fabrika: ${message}`)
449
- await context.reporter.report({ fingerprint: ['fabrika-job-failed', job.name], type: 'JobFailed', message, tags: { ...tags, code: 'crashed' } })
740
+ await reportCrashed(job.name, context.reporter)
450
741
  }
451
742
  const period = periodAt(job.schedule, await context.leases.now())
452
743
  const claim = period === null ? null : await context.leases.claimOverdue(job.name, period)
@@ -464,30 +755,156 @@ async function surveil(job: { readonly name: string; readonly schedule: CronSche
464
755
  })
465
756
  }
466
757
 
758
+ interface SchedulerInput {
759
+ readonly framework: FrameworkTables
760
+ /** Every job bound to run, with the attempt time its lease is taken for. */
761
+ readonly declared: readonly (Scheduled & { readonly timeoutMs: number })[]
762
+ readonly contextFor: (reporter: OperationsReporter | undefined) => RunnerContext
763
+ readonly run: (job: string, fence: LeaseFence, options: JobRunOptions) => Promise<JobRunResult>
764
+ }
765
+
766
+ function jobScheduler(input: SchedulerInput): JobScheduler {
767
+ const { framework, declared } = input
768
+ const byName = new Map(declared.map((job) => [job.name, job]))
769
+ return Object.freeze({
770
+ jobs: Object.freeze(declared.map((job) => job.name)),
771
+ async repair(options: Pick<JobRunOptions, 'reporter'> = {}): Promise<boolean> {
772
+ const admission = admitInvocation('framework-job')
773
+ if (admission === null) {
774
+ return false
775
+ }
776
+ try {
777
+ const context = input.contextFor(options.reporter)
778
+ for (const job of await framework.wakes.expiredHolders()) {
779
+ if (await context.leases.reapCrashed(job)) {
780
+ await reportCrashed(job, context.reporter)
781
+ }
782
+ }
783
+ for (const job of declared) {
784
+ await context.leases.register(job.name)
785
+ await surveil(job, context)
786
+ }
787
+ await framework.wakes.reconcile(declared.map((job) => ({ job: job.name, expression: job.expression })))
788
+ const now = await context.leases.now()
789
+ for (const job of declared) {
790
+ const dueAt = nextFire(job.schedule, now)
791
+ if (dueAt !== null) {
792
+ await framework.wakes.schedule({ job: job.name, reason: 'cron', dueAt, dedupe: cronKey(job.name, job.expression) })
793
+ }
794
+ }
795
+ return true
796
+ } finally {
797
+ admission.release()
798
+ }
799
+ },
800
+ async orphans(): Promise<readonly string[]> {
801
+ return (await framework.wakes.expiredHolders()).filter((job) => !byName.has(job))
802
+ },
803
+ async earlyBy(withinMs: number): Promise<number> {
804
+ const now = (await framework.jobs.now()).getTime()
805
+ let early = 0
806
+ for (const job of declared) {
807
+ for (const wake of await framework.wakes.pending(job.name)) {
808
+ const ahead = wake.dueAt.getTime() - now
809
+ if (ahead <= withinMs) {
810
+ early = Math.max(early, ahead)
811
+ }
812
+ }
813
+ }
814
+ return early
815
+ },
816
+ async take(name: string): Promise<TakenLease> {
817
+ const job = byName.get(name)
818
+ const leases = framework.jobs
819
+ if (job === undefined) {
820
+ // A removed job runs no more; only a crashed run's lease is freed, so it stops arming the alarm.
821
+ return { fence: null, reaped: await leases.reapCrashed(name) }
822
+ }
823
+ await leases.register(name)
824
+ const reaped = await leases.reapCrashed(name)
825
+ const now = (await leases.now()).getTime()
826
+ const due = (await framework.wakes.pending(name)).filter((wake) => wake.dueAt.getTime() <= now)
827
+ if (due.length === 0) {
828
+ return { fence: null, reaped }
829
+ }
830
+ const fence = await leases.tryAcquire(name, attemptId(), job.timeoutMs, job.timeoutMs + LEASE_GRACE_MS)
831
+ if (fence === null && !(await ownedByHolder(name, now, leases, framework.wakes))) {
832
+ // The run holding the lease runs once more after it succeeded; a wake it did not own stays pending.
833
+ await leases.markPending(name)
834
+ }
835
+ return { fence, reaped }
836
+ },
837
+ run(job: string, fence: LeaseFence, options: JobRunOptions = {}): Promise<JobRunResult> {
838
+ return input.run(job, fence, options)
839
+ },
840
+ async crashed(job: string, options: Pick<JobRunOptions, 'reporter'> = {}): Promise<void> {
841
+ await reportCrashed(job, input.contextFor(options.reporter).reporter)
842
+ },
843
+ next(using?: Session): Promise<Date | null> {
844
+ return framework.wakes.next(using)
845
+ },
846
+ untilNext(using?: Session): Promise<number | null> {
847
+ return framework.wakes.untilNext(using)
848
+ },
849
+ async failed(step: 'alarm' | 'repair', error: unknown): Promise<void> {
850
+ try {
851
+ await input.contextFor(undefined).reporter.report({
852
+ fingerprint: ['fabrika-alarm-failed', step],
853
+ type: 'AlarmFailed',
854
+ message: `the job scheduler's ${step} failed`,
855
+ tags: { step },
856
+ error,
857
+ })
858
+ } catch {
859
+ console.error(`fabrika: the failure of the job scheduler's ${step} could not be reported`)
860
+ }
861
+ },
862
+ })
863
+ }
864
+
467
865
  interface Progress {
468
866
  attempts: number
469
867
  }
470
868
 
869
+ /** What the attempt that succeeded kicked: `null` until a handler returned. */
870
+ interface Kicked {
871
+ recorder: KickRecorder | null
872
+ }
873
+
471
874
  /**
472
875
  * How a run takes the lease: `wait` is `fabrika job run`'s, in a process of its own; `try` is a kick's and a follow-up
473
- * run's, which never wait for a lock.
876
+ * run's, which never wait for a lock; a fence is one the alarm's scheduler already took.
474
877
  */
475
- type Acquisition = 'wait' | 'try'
878
+ type Acquisition = 'wait' | 'try' | { readonly fence: LeaseFence }
476
879
 
477
880
  /**
478
- * A run of the job, then one more while a kick arrived after the last one started and it succeeded. Once `stop`
479
- * aborted, no follow-up run starts: the pending flag stays for the schedule.
881
+ * How a failed attempt is retried: `inline` in the same run, after a backoff, under a renewed lease; `durable` (a run the
882
+ * Cloudflare alarm started) by moving the wakes it owns to a later due time, so the alarm runs it again. A durable run
883
+ * that owns no wake retries inline: there is nothing to move.
480
884
  */
481
- async function runJob(job: BoundJob, context: RunnerContext, clock: JobClock, acquisition: Acquisition, stop?: AbortSignal): Promise<JobRunResult> {
482
- let result = await runOnce(job, context, clock, acquisition, stop)
483
- while (result.status === 'succeeded' && stop?.aborted !== true && (await pendingTaken(job, context))) {
484
- const next = await runOnce(job, context, clock, 'try', stop)
885
+ type Retries = 'inline' | 'durable'
886
+
887
+ /**
888
+ * A run of the job, then one more while a kick arrived after the last one started and it succeeded. Once the run's
889
+ * control aborted, no follow-up run starts: the pending flag stays for the schedule.
890
+ */
891
+ async function runJob(
892
+ job: BoundJob,
893
+ context: RunnerContext,
894
+ clock: JobClock,
895
+ acquisition: Acquisition,
896
+ control: RunControl,
897
+ retries: Retries,
898
+ ): Promise<JobRunResult> {
899
+ let result = await runOnce(job, context, clock, acquisition, control, retries)
900
+ while (result.status === 'succeeded' && !control.signal.aborted && (await pendingTaken(job, context))) {
901
+ const next = await runOnce(job, context, clock, 'try', control, retries)
485
902
  if (next.status !== 'skipped') {
486
903
  result = next
487
904
  continue
488
905
  }
489
906
  // A newer run holds the lease and may have read before the kick's row: the flag goes back to it.
490
- const handed = await handOver(job, context, clock, stop)
907
+ const handed = await handOver(job, context, clock, control, retries)
491
908
  return handed ?? result
492
909
  }
493
910
  return result
@@ -503,11 +920,23 @@ async function pendingTaken(job: BoundJob, context: RunnerContext): Promise<bool
503
920
  }
504
921
  }
505
922
 
506
- async function runOnce(job: BoundJob, context: RunnerContext, clock: JobClock, acquisition: Acquisition, stop?: AbortSignal): Promise<JobRunResult> {
923
+ async function runOnce(
924
+ job: BoundJob,
925
+ context: RunnerContext,
926
+ clock: JobClock,
927
+ acquisition: Acquisition,
928
+ control: RunControl,
929
+ retries: Retries,
930
+ ): Promise<JobRunResult> {
507
931
  const progress: Progress = { attempts: 0 }
508
932
  try {
509
- return await runLeased(job, context, clock, progress, acquisition, stop)
933
+ return await runLeased(job, context, clock, progress, acquisition, control, retries)
510
934
  } catch {
935
+ if (control.signal.aborted) {
936
+ // A stopped run's lease write is refused by the object's write fence; the lease expires and the schedule runs it.
937
+ console.error(`fabrika: job '${job.name}' stopped before it released its lease; its schedule runs it again`)
938
+ return { status: 'failed', attempts: progress.attempts, code: 'stopped' }
939
+ }
511
940
  // Only the lease store throws here: the handler's errors stay inside its attempt. Its message may quote the URL.
512
941
  return failed(job, progress.attempts, failure('driver_failed'), context.reporter)
513
942
  }
@@ -519,32 +948,47 @@ async function runLeased(
519
948
  clock: JobClock,
520
949
  progress: Progress,
521
950
  acquisition: Acquisition,
522
- stop?: AbortSignal,
951
+ control: RunControl,
952
+ retries: Retries,
523
953
  ): Promise<JobRunResult> {
524
954
  const { leases } = context
525
- await leases.register(job.name)
526
955
  const leaseMs = job.timeoutMs + LEASE_GRACE_MS
527
- if (stop?.aborted === true) {
528
- return { status: 'skipped' }
529
- }
530
956
  let fence: LeaseFence | null
531
- if (acquisition === 'wait') {
532
- await surveil(job, context)
533
- fence = await leases.acquire(job.name, attemptId(), job.timeoutMs, leaseMs)
957
+ if (typeof acquisition === 'object') {
958
+ fence = acquisition.fence
534
959
  } else {
535
- fence = await leases.tryAcquire(job.name, attemptId(), job.timeoutMs, leaseMs)
960
+ await leases.register(job.name)
961
+ if (control.signal.aborted) {
962
+ return { status: 'skipped' }
963
+ }
964
+ if (acquisition === 'wait') {
965
+ await surveil(job, context)
966
+ fence = await leases.acquire(job.name, attemptId(), job.timeoutMs, leaseMs)
967
+ } else {
968
+ fence = await leases.tryAcquire(job.name, attemptId(), job.timeoutMs, leaseMs)
969
+ }
536
970
  }
537
971
  if (fence === null) {
538
972
  return { status: 'skipped' }
539
973
  }
974
+ // The segment is measured from the lease, as the database's attempt deadline is. A run the alarm started has one
975
+ // segment: its attempts end before it by the finish's reserve, and no finish starts past it.
976
+ const segmentEnd = retries === 'durable' ? nowOf(clock) + job.timeoutMs : null
977
+ const attemptEnd = (
978
+ start: number,
979
+ ): number => (segmentEnd === null ? start + job.timeoutMs : Math.min(start + job.timeoutMs, segmentEnd - finishReserveMs(job.timeoutMs)))
980
+ let deadline = attemptEnd(nowOf(clock))
540
981
  // This run covers every kick before it: their rows committed before it reads.
541
982
  await pendingTaken(job, context)
983
+ const owned = retries === 'durable' ? await ownedWakes(job.name, context) : []
984
+ const durable = owned.length > 0
985
+ const scheduled: Scheduled = { name: job.name, expression: job.artifact.schedule, schedule: job.schedule }
542
986
  const attempts = job.artifact.retries + 1
543
- progress.attempts = 1
544
- // The jobs the succeeding attempt kicked; a failed attempt's kicks are dropped with its writes.
545
- const kicked: string[] = []
546
- let outcome = await attemptOnce(job, fence, progress.attempts, clock, context, kicked, stop)
547
- while (outcome !== null && outcome.code !== 'stopped' && !outcome.final && progress.attempts < attempts) {
987
+ progress.attempts = durable ? Math.max(...owned.map((wake) => wake.attempt)) + 1 : 1
988
+ // The kicks of the succeeding attempt; a failed attempt's kicks are dropped with its writes.
989
+ const kicked: Kicked = { recorder: null }
990
+ let outcome = await attemptOnce(job, fence, progress.attempts, deadline, segmentEnd, clock, context, kicked, control)
991
+ while (!durable && outcome !== null && outcome.code !== 'stopped' && !outcome.final && progress.attempts < attempts) {
548
992
  console.error(`fabrika: job '${job.name}' attempt ${progress.attempts} of ${attempts} failed: ${describeFailure(outcome)}`)
549
993
  const delay = backoffMs(progress.attempts, clock.random())
550
994
  // A new generation at once: nothing the failed attempt still has in flight commits from here on.
@@ -553,24 +997,63 @@ async function runLeased(
553
997
  return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
554
998
  }
555
999
  fence = next
556
- await clock.wait(delay, stop ?? new AbortController().signal)
1000
+ deadline = attemptEnd(nowOf(clock) + delay)
1001
+ await clock.wait(delay, control.signal)
557
1002
  progress.attempts++
558
- outcome = await attemptOnce(job, fence, progress.attempts, clock, context, kicked, stop)
1003
+ outcome = await attemptOnce(job, fence, progress.attempts, deadline, segmentEnd, clock, context, kicked, control)
1004
+ }
1005
+ const held = fence
1006
+ // The finish, unless the segment ended first; a finish once started is awaited, as it may be committing.
1007
+ const finish = async (status: 'succeeded' | 'failed', wakes: Promise<FinishWakes | undefined>): Promise<boolean | typeof OUT_OF_SEGMENT> => {
1008
+ const applied = await withinSegment(clock, segmentEnd, wakes)
1009
+ if (applied === OUT_OF_SEGMENT || (segmentEnd !== null && remainingMs(clock, segmentEnd) === 0)) {
1010
+ return OUT_OF_SEGMENT
1011
+ }
1012
+ return await leases.finish(held, status, applied)
559
1013
  }
560
1014
  if (outcome === null) {
561
- if (await leases.finish(fence, 'succeeded')) {
1015
+ // Before the finish, so a crash between it and a kick's start leaves the kick's wake to the scheduler.
1016
+ const kicks = kicked.recorder
1017
+ for (const wake of kicks?.wakes(await leases.now()) ?? []) {
1018
+ await context.wakes.schedule(wake)
1019
+ }
1020
+ const finished = await finish('succeeded', completion(scheduled, context, durable))
1021
+ if (finished === OUT_OF_SEGMENT) {
1022
+ return outOfSegment(job, progress.attempts)
1023
+ }
1024
+ if (finished) {
562
1025
  // After the lease is released, so a job that kicks another never waits on its own lease; a kick never rejects.
563
- await Promise.all(kicked.map((name) => context.kick(name)))
1026
+ await Promise.all((kicks?.kicked() ?? []).map((name) => context.kick(name)))
564
1027
  return { status: 'succeeded', attempts: progress.attempts }
565
1028
  }
566
1029
  return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
567
1030
  }
568
- await leases.finish(fence, 'failed')
569
1031
  if (outcome.code === 'stopped') {
570
- // Not a failure of the job: the schedule runs it again, so Operations hears nothing.
1032
+ // Its wakes stay, so the alarm or the schedule runs it again. Not a failure of the job: Operations hears nothing.
1033
+ await leases.finish(fence, 'failed')
571
1034
  console.error(`fabrika: job '${job.name}' stopped: the process is shutting down; its schedule runs it again`)
572
1035
  return { status: 'failed', attempts: progress.attempts, code: 'stopped' }
573
1036
  }
1037
+ if (durable && !outcome.final && progress.attempts < attempts) {
1038
+ const at = new Date(nowOf(clock) + backoffMs(progress.attempts, clock.random()))
1039
+ console.error(
1040
+ `fabrika: job '${job.name}' attempt ${progress.attempts} of ${attempts} failed: ${describeFailure(outcome)}; it runs again at ${at.toISOString()}`,
1041
+ )
1042
+ const finished = await finish('failed', Promise.resolve({ kind: 'retry', at }))
1043
+ if (finished === OUT_OF_SEGMENT) {
1044
+ return outOfSegment(job, progress.attempts)
1045
+ }
1046
+ if (!finished) {
1047
+ return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
1048
+ }
1049
+ return outcome.why === null
1050
+ ? { status: 'failed', attempts: progress.attempts, code: outcome.code }
1051
+ : { status: 'failed', attempts: progress.attempts, code: outcome.code, reason: outcome.why.reason }
1052
+ }
1053
+ // Past its retries a run is done with its wakes: the next cron fire is scheduled, a kick's wake removed.
1054
+ if ((await finish('failed', completion(scheduled, context, durable))) === OUT_OF_SEGMENT) {
1055
+ return outOfSegment(job, progress.attempts)
1056
+ }
574
1057
  return failed(job, progress.attempts, outcome, context.reporter)
575
1058
  }
576
1059
 
@@ -609,14 +1092,27 @@ async function failed(job: { readonly name: string }, attempts: number, outcome:
609
1092
 
610
1093
  const TIMED_OUT: unique symbol = Symbol('fabrika.job.timedOut')
611
1094
 
612
- /** One run of a framework job: its lease, one attempt in a frame fenced by it, and the outcome reported like a job's. */
613
- async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext, clock: JobClock): Promise<JobRunResult> {
1095
+ /**
1096
+ * One run of a framework job: its lease, one attempt in a frame fenced by it, and the outcome reported like a job's.
1097
+ * A run its control stopped is not a failure: the schedule runs it again. `acquired` is the lease the alarm's scheduler
1098
+ * took: such a run is done with the wakes it owns whether it succeeded or failed, as a framework job has no retries.
1099
+ */
1100
+ async function runFrameworkJob(
1101
+ bound: BoundFrameworkJob,
1102
+ context: RunnerContext,
1103
+ clock: JobClock,
1104
+ control: RunControl,
1105
+ acquired: LeaseFence | null,
1106
+ ): Promise<JobRunResult> {
614
1107
  const { leases } = context
615
- let fence: LeaseFence | null
1108
+ const scheduled: Scheduled = { name: bound.name, expression: bound.job.schedule, schedule: bound.schedule }
1109
+ let fence: LeaseFence | null = acquired
616
1110
  try {
617
- await leases.register(bound.name)
618
- await surveil(bound, context)
619
- fence = await leases.acquire(bound.name, attemptId(), bound.job.timeoutMs, bound.job.timeoutMs + LEASE_GRACE_MS)
1111
+ if (fence === null) {
1112
+ await leases.register(bound.name)
1113
+ await surveil(bound, context)
1114
+ fence = await leases.acquire(bound.name, attemptId(), bound.job.timeoutMs, bound.job.timeoutMs + LEASE_GRACE_MS)
1115
+ }
620
1116
  } catch {
621
1117
  return failed(bound, 0, failure('driver_failed'), context.reporter)
622
1118
  }
@@ -624,14 +1120,18 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
624
1120
  console.error(`fabrika: a previous run of job '${bound.name}' still holds its lease; this run does not start`)
625
1121
  return { status: 'skipped' }
626
1122
  }
1123
+ // A run the alarm took the lease for has one segment, whose end is kept for the finish, as a contract job's.
1124
+ const segmentEnd = acquired === null ? null : nowOf(clock) + bound.job.timeoutMs
627
1125
  const frame = new RequestFrame(fence.holder, { fence })
628
1126
  const cancel = new AbortController()
629
1127
  const timer = new AbortController()
630
1128
  let outcome: JobFailure | null
631
1129
  try {
632
1130
  const ran = runInRequest(frame, () => bound.job.run(context.reporter, cancel.signal)).then(() => null, (error: unknown) => failureOf(error, null))
633
- const timeout = clock.wait(bound.job.timeoutMs, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
634
- const result = await Promise.race([ran, timeout])
1131
+ control.track(ran)
1132
+ const budget = segmentEnd === null ? bound.job.timeoutMs : bound.job.timeoutMs - finishReserveMs(bound.job.timeoutMs)
1133
+ const timeout = clock.wait(budget, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
1134
+ const result = await Promise.race([ran, timeout, stoppedBy(control, timer.signal)])
635
1135
  outcome = result === TIMED_OUT ? failure('timeout') : result
636
1136
  } finally {
637
1137
  frame.end()
@@ -640,52 +1140,103 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
640
1140
  try {
641
1141
  // Writes are refused before the run hears it was stopped: a scanner that answers on abort commits nothing.
642
1142
  try {
643
- await leases.cancelInFlight(fence)
1143
+ await withinSegment(clock, segmentEnd, leases.cancelInFlight(fence))
644
1144
  } finally {
645
1145
  cancel.abort()
646
1146
  }
647
- const finished = await leases.finish(fence, outcome === null ? 'succeeded' : 'failed')
1147
+ // A stopped run keeps its wakes, so the alarm runs it again once the object opens.
1148
+ const done = outcome === null || (acquired !== null && outcome.code !== 'stopped')
1149
+ const wakes = await withinSegment(clock, segmentEnd, done ? completion(scheduled, context, acquired !== null) : Promise.resolve(undefined))
1150
+ if (wakes === OUT_OF_SEGMENT || (segmentEnd !== null && remainingMs(clock, segmentEnd) === 0)) {
1151
+ return outOfSegment(bound, 1)
1152
+ }
1153
+ // Awaited once started: a finish may be committing.
1154
+ const finished = await leases.finish(fence, outcome === null ? 'succeeded' : 'failed', wakes)
648
1155
  if (outcome === null) {
649
1156
  return finished ? { status: 'succeeded', attempts: 1 } : failed(bound, 1, failure('lease_lost'), context.reporter)
650
1157
  }
651
1158
  } catch {
652
- return failed(bound, 1, failure('driver_failed'), context.reporter)
1159
+ // A stopped run's lease write is refused by the object's write fence; the lease expires and the schedule runs it.
1160
+ if (!control.signal.aborted) {
1161
+ return failed(bound, 1, failure('driver_failed'), context.reporter)
1162
+ }
1163
+ }
1164
+ // Here a run without a failure of its own is one whose stop refused its lease write.
1165
+ if (outcome === null || outcome.code === 'stopped' || control.signal.aborted) {
1166
+ console.error(`fabrika: job '${bound.name}' stopped: the object entered maintenance; its schedule runs it again`)
1167
+ return { status: 'failed', attempts: 1, code: 'stopped' }
653
1168
  }
654
1169
  return failed(bound, 1, outcome, context.reporter)
655
1170
  }
656
1171
 
657
1172
  /** Read afresh at each call: the signal may abort while the caller awaits. */
658
- const isStopped = (stop: AbortSignal | undefined): boolean => stop?.aborted === true
1173
+ const isStopped = (control: RunControl): boolean => control.signal.aborted
1174
+
1175
+ /** Resolves `stopped` once the run's control aborts; the listener goes when `done` aborts. */
1176
+ function stoppedBy(control: RunControl, done: AbortSignal): Promise<JobFailure> {
1177
+ return new Promise<JobFailure>((resolve) => {
1178
+ if (control.signal.aborted) {
1179
+ resolve(failure('stopped'))
1180
+ }
1181
+ control.signal.addEventListener('abort', () => resolve(failure('stopped')), { once: true, signal: done })
1182
+ })
1183
+ }
659
1184
 
660
1185
  /**
661
- * `null` when the handler finished, else the attempt's failure code. Either way the attempt's frame ends and the
662
- * statements it still has running are cancelled before the runner touches the lease again.
1186
+ * `null` when the handler finished, else the attempt's failure code. One deadline bounds the attempt: IAM's answer for
1187
+ * the job's key, then the handler with only what is left of it. Either way the attempt's frame ends and the statements
1188
+ * it still has running are cancelled before the runner touches the lease again, waited for only while the segment lasts.
663
1189
  */
664
1190
  async function attemptOnce(
665
1191
  job: BoundJob,
666
1192
  fence: LeaseFence,
667
1193
  attempt: number,
1194
+ deadline: number,
1195
+ segmentEnd: number | null,
668
1196
  clock: JobClock,
669
1197
  context: RunnerContext,
670
- kicked: string[],
671
- stop?: AbortSignal,
1198
+ kicked: Kicked,
1199
+ control: RunControl,
672
1200
  ): Promise<JobFailure | null> {
673
- kicked.length = 0
674
- if (isStopped(stop)) {
1201
+ kicked.recorder = null
1202
+ if (isStopped(control)) {
675
1203
  return failure('stopped')
676
1204
  }
677
1205
  // The job's authority is what IAM signs for its key at the start of the attempt; a revoked key stops the next one.
678
- const authority = await job.credential.authority()
679
- if (isStopped(stop)) {
1206
+ const setup = new AbortController()
1207
+ const exchange = job.credential.authority()
1208
+ // Its settlement, not only the attempt's: a drain waits for an exchange the attempt stopped waiting for.
1209
+ control.track(exchange)
1210
+ let authority: WorkloadAuthority
1211
+ try {
1212
+ const answered = await Promise.race([
1213
+ exchange,
1214
+ clock.wait(remainingMs(clock, deadline), setup.signal).then((): typeof TIMED_OUT => TIMED_OUT),
1215
+ stoppedBy(control, setup.signal),
1216
+ ])
1217
+ if (answered === TIMED_OUT || 'code' in answered) {
1218
+ // The next attempt asks IAM afresh rather than wait on an exchange that outlived this one.
1219
+ job.credential.abandon(exchange)
1220
+ return answered === TIMED_OUT ? failure('timeout') : answered
1221
+ }
1222
+ authority = answered
1223
+ } finally {
1224
+ setup.abort()
1225
+ }
1226
+ if (isStopped(control)) {
680
1227
  return failure('stopped')
681
1228
  }
682
1229
  if (!authority.ok) {
683
1230
  return failure(authority.reason === 'unavailable' ? 'iam_unavailable' : 'unauthorized')
684
1231
  }
1232
+ const budget = remainingMs(clock, deadline)
1233
+ if (budget === 0) {
1234
+ return failure('timeout')
1235
+ }
685
1236
  const frame = new RequestFrame(fence.holder, { fence })
686
1237
  const cancel = new AbortController()
687
1238
  const timer = new AbortController()
688
- const signal = stop === undefined ? cancel.signal : AbortSignal.any([cancel.signal, stop])
1239
+ const signal = AbortSignal.any([cancel.signal, control.signal])
689
1240
  const kicks = kickRecorder(`job '${job.name}'`, job.site?.kicks ?? [])
690
1241
  const ctx = { ...jobContext(job, frame, attempt, signal, context, authority.perms), kick: kicks.kick }
691
1242
  try {
@@ -696,26 +1247,22 @@ async function attemptOnce(
696
1247
  } else {
697
1248
  await runAsSite(`job '${job.name}'`, site.egress, async () => await job.handler({ ctx }))
698
1249
  }
699
- kicked.push(...kicks.kicked())
1250
+ kicked.recorder = kicks
700
1251
  }).then(() => null, (error: unknown) => {
701
1252
  const failed = failureOf(error, job.mailer === null ? null : (context.mail[job.mailer]?.sender.capabilities ?? null))
702
1253
  if (failed.code === 'handler_failed') context.revealHandlerErrors?.(`job '${job.name}' attempt ${attempt}`, error)
703
1254
  return failed
704
1255
  })
705
- const timeout = clock.wait(job.timeoutMs, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
706
- const stopped = new Promise<JobFailure>((resolve) => {
707
- if (stop?.aborted === true) {
708
- resolve(failure('stopped'))
709
- }
710
- stop?.addEventListener('abort', () => resolve(failure('stopped')), { once: true, signal: timer.signal })
711
- })
712
- const result = await Promise.race([ran, timeout, stopped])
1256
+ // Its settlement, not only the end of its frame: a drain waits for a handler that outlived the attempt.
1257
+ control.track(ran)
1258
+ const timeout = clock.wait(budget, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
1259
+ const result = await Promise.race([ran, timeout, stoppedBy(control, timer.signal)])
713
1260
  return result === TIMED_OUT ? failure('timeout') : result
714
1261
  } finally {
715
1262
  frame.end()
716
1263
  cancel.abort()
717
1264
  timer.abort()
718
- await context.leases.cancelInFlight(fence)
1265
+ await withinSegment(clock, segmentEnd, context.leases.cancelInFlight(fence))
719
1266
  }
720
1267
  }
721
1268
 
@@ -898,25 +1445,37 @@ function kitJobContext(
898
1445
  db: siteData(reads, (action) => reads.app(app(action))),
899
1446
  sql: site.sql === null ? undeclaredSql(job, 'job') : jobSqlOutsideTransaction(job),
900
1447
  files: putAndOpen(boundTo(frame, () => false, 'ctx.files')),
901
- transaction: async <R>(work: (tx: object) => Promise<R>): Promise<R> => {
1448
+ transaction: async <R>(body: (tx: object) => Promise<R>): Promise<R> => {
902
1449
  // Outside its attempt there is no lease to fence the transaction with, so it never opens.
903
1450
  boundTo(frame, () => false)()
904
- return data.transaction(async (session) => {
905
- let open = true
906
- const writes = jobHandles(frame, data, site, principal, 'mutation', async () => session, () => !open, undefined, binding)
907
- try {
908
- const sql = site.sql === null ? undeclaredSql(job, 'job', 'tx.sql') : jobSql(frame, data, site.sql, session, () => !open)
909
- // Located in this transaction, so an object it wrote is found before the commit.
910
- const located = (action: string, dimension: string, id: string) => {
911
- boundTo(frame, () => !open, 'tx.ref')()
912
- return ref(action, dimension, id, session)
1451
+ // Its own files: a rollback releases only what this transaction staged, never a file of a concurrent one.
1452
+ const own = new FileWork(frame, frame.id)
1453
+ let rolledBack = false
1454
+ try {
1455
+ return await data.transaction(async (session) => {
1456
+ let open = true
1457
+ const writes = jobHandles(frame, data, site, principal, 'mutation', async () => session, () => !open, undefined, binding)
1458
+ try {
1459
+ const sql = site.sql === null ? undeclaredSql(job, 'job', 'tx.sql') : jobSql(frame, data, site.sql, session, () => !open)
1460
+ // Located in this transaction, so an object it wrote is found before the commit.
1461
+ const located = (action: string, dimension: string, id: string) => {
1462
+ boundTo(frame, () => !open, 'tx.ref')()
1463
+ return ref(action, dimension, id, session)
1464
+ }
1465
+ const files = storedFilesHandle(kinds, own, { where: job, check: boundTo(frame, () => !open, 'tx.files'), writes: true })
1466
+ return await body(Object.freeze({ db: siteData(writes, (action) => writes.app(app(action))), sql, files, ref: located }))
1467
+ } catch (error) {
1468
+ rolledBack = true
1469
+ throw error
1470
+ } finally {
1471
+ open = false
913
1472
  }
914
- const files = putAndOpen(boundTo(frame, () => !open, 'tx.files'))
915
- return await work(Object.freeze({ db: siteData(writes, (action) => writes.app(app(action))), sql, files, ref: located }))
916
- } finally {
917
- open = false
918
- }
919
- })
1473
+ })
1474
+ } catch (error) {
1475
+ // A transaction the work threw out of is a known rollback; a failed commit is not, and the sweep decides.
1476
+ if (rolledBack) await kinds?.release(own)
1477
+ throw error
1478
+ }
920
1479
  },
921
1480
  }
922
1481
  }