@fabrika/app 0.9.0-cloudflare.1 → 0.9.0-cloudflare.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +21 -17
  3. package/skill/references/app-setup.md +42 -11
  4. package/skill/references/endpoint.md +29 -22
  5. package/skill/references/errors.md +32 -31
  6. package/skill/references/jobs.md +22 -3
  7. package/skill/references/tests.md +29 -1
  8. package/src/build.ts +6 -1
  9. package/src/bun.ts +168 -5
  10. package/src/cloudflare/application.ts +546 -76
  11. package/src/cloudflare/bindings.ts +26 -5
  12. package/src/cloudflare/export.ts +621 -0
  13. package/src/cloudflare/import.ts +1097 -0
  14. package/src/cloudflare/maintenance.ts +518 -0
  15. package/src/cloudflare/names.ts +6 -0
  16. package/src/cloudflare/ops.ts +147 -18
  17. package/src/cloudflare/restore.ts +167 -0
  18. package/src/cloudflare/tracker.ts +192 -0
  19. package/src/cloudflare.ts +3 -2
  20. package/src/contract/artifact-schema.ts +13 -2
  21. package/src/contract/build.ts +38 -11
  22. package/src/contract/context.ts +26 -9
  23. package/src/contract/defaults.ts +22 -2
  24. package/src/contract/define.ts +1 -1
  25. package/src/contract/group.ts +10 -1
  26. package/src/contract/kit.ts +2 -1
  27. package/src/disk-assets.ts +8 -3
  28. package/src/files/kinds.ts +54 -24
  29. package/src/implement/anonymous.ts +1 -10
  30. package/src/implement/boot.ts +30 -12
  31. package/src/implement/capability.ts +3 -0
  32. package/src/implement/dispatch.ts +175 -37
  33. package/src/implement/egress.ts +367 -8
  34. package/src/implement/invocation.ts +60 -19
  35. package/src/implement/workload.ts +18 -3
  36. package/src/jobs/runner.ts +646 -98
  37. package/src/spa-fallback.ts +12 -0
  38. package/src/testing/endpoint-access.ts +135 -23
  39. package/src/testing/isolation.ts +8 -1
  40. package/src/testing/kit.ts +60 -0
  41. package/types/src/build.d.ts +1 -1
  42. package/types/src/bun.d.ts +34 -1
  43. package/types/src/cloudflare/application.d.ts +36 -5
  44. package/types/src/cloudflare/bindings.d.ts +6 -0
  45. package/types/src/cloudflare/export.d.ts +16 -0
  46. package/types/src/cloudflare/import.d.ts +58 -0
  47. package/types/src/cloudflare/maintenance.d.ts +117 -0
  48. package/types/src/cloudflare/names.d.ts +5 -0
  49. package/types/src/cloudflare/ops.d.ts +39 -4
  50. package/types/src/cloudflare/restore.d.ts +9 -0
  51. package/types/src/cloudflare/tracker.d.ts +29 -0
  52. package/types/src/cloudflare.d.ts +2 -1
  53. package/types/src/contract/artifact-schema.d.ts +2 -0
  54. package/types/src/contract/context.d.ts +24 -9
  55. package/types/src/contract/defaults.d.ts +19 -1
  56. package/types/src/contract/define.d.ts +1 -1
  57. package/types/src/contract/group.d.ts +9 -1
  58. package/types/src/contract/kit.d.ts +2 -2
  59. package/types/src/disk-assets.d.ts +4 -1
  60. package/types/src/files/kinds.d.ts +17 -6
  61. package/types/src/implement/anonymous.d.ts +0 -4
  62. package/types/src/implement/boot.d.ts +17 -0
  63. package/types/src/implement/capability.d.ts +2 -0
  64. package/types/src/implement/dispatch.d.ts +33 -1
  65. package/types/src/implement/egress.d.ts +36 -1
  66. package/types/src/implement/invocation.d.ts +20 -10
  67. package/types/src/implement/workload.d.ts +5 -0
  68. package/types/src/jobs/runner.d.ts +63 -2
  69. package/types/src/spa-fallback.d.ts +2 -0
  70. package/types/src/testing/endpoint-access.d.ts +25 -1
  71. package/types/src/testing/isolation.d.ts +1 -1
  72. package/types/src/testing/kit.d.ts +16 -0
@@ -3,12 +3,23 @@
3
3
  // that timed out, failed or lost the lease never commits (`@fabrika/db`'s fence). A site's `ctx.kick` sends a run in
4
4
  // the background after its commit, without waiting for any lock; a kick that finds the lease held leaves the shared pending flag, and the run
5
5
  // holding it runs once more after it succeeded. Failures, crashed runs and a job that stopped succeeding are reported
6
- // to Operations, each under a fixed failure code.
6
+ // to Operations, each under a fixed failure code. On Cloudflare the object's alarm drives the jobs through `scheduler`:
7
+ // a run it starts owns the job's due wakes, and when it fails they move to a durable retry instead of an inline one.
7
8
 
8
9
  import type { IamMembership } from '@fabrika/auth'
9
10
  import { type ContractArtifact, JOB_SITE_PREFIX, type JobArtifact, type JobMailer, type PermissionEntry, permits } from '@fabrika/auth-core'
10
11
  import type { CapabilityAuthority } from '@fabrika/db/capability'
11
- import { type DataAccess, type FrameworkTables, type JobLeases, type LeaseFence, LeaseLostError, type Session } from '@fabrika/db/framework'
12
+ import {
13
+ type DataAccess,
14
+ type FinishWakes,
15
+ type FrameworkTables,
16
+ type JobLeases,
17
+ type JobWakes,
18
+ type LeaseFence,
19
+ LeaseLostError,
20
+ type Session,
21
+ type Wake,
22
+ } from '@fabrika/db/framework'
12
23
  import { EmailDeliveryError, type EmailSenderCapabilities, mayRetryDelivery } from '@fabrika/email'
13
24
  import { uuidv7 } from '@fabrika/platform'
14
25
  import { NotFoundError } from '../errors'
@@ -22,7 +33,8 @@ import {
22
33
  runInRequest,
23
34
  type Within,
24
35
  } from '../implement/capability'
25
- import { runAsSite } from '../implement/egress'
36
+ import { admitInvocation, type InvocationAdmission } from '../implement/dispatch'
37
+ import { type PublicEgressRules, runAsSite, siteEgress } from '../implement/egress'
26
38
  import { failureCode, type FailureReason, failureReason } from '../implement/failures'
27
39
  import { ServerFault } from '../implement/faults'
28
40
  import type { ContractDatabase } from '../implement/implement'
@@ -34,6 +46,7 @@ import {
34
46
  jobHandles,
35
47
  jobSql,
36
48
  jobSqlOutsideTransaction,
49
+ type KickRecorder,
37
50
  kickRecorder,
38
51
  siteData,
39
52
  type SiteDeclaration,
@@ -42,10 +55,10 @@ import {
42
55
  import { jobLinkIssuing, type JobLinkMount } from '../implement/membership'
43
56
  import type { SignedTokens } from '../implement/tokens'
44
57
  import { type AppUrl, appUrl } from '../implement/urls'
45
- import { type WorkloadCredential, workloadSite } from '../implement/workload'
58
+ import { type WorkloadAuthority, type WorkloadCredential, workloadSite } from '../implement/workload'
46
59
  import { jobMail, type MailTransports } from './mail'
47
60
  import type { OperationsReporter } from './report'
48
- import { type CronSchedule, parseCron, periodAt } from './schedule'
61
+ import { type CronSchedule, isDueAt, parseCron, periodAt } from './schedule'
49
62
  import type { AppMail, JobContext, JobPrincipal } from './types'
50
63
 
51
64
  /**
@@ -55,6 +68,11 @@ import type { AppMail, JobContext, JobPrincipal } from './types'
55
68
  export const LEASE_GRACE_MS = 60_000
56
69
  /** How often the serving process checks every job for a crashed run and for a schedule that stopped. */
57
70
  export const JOB_CHECK_INTERVAL_MS = 60_000
71
+ /**
72
+ * A run the Cloudflare alarm started has one segment (`CLOUDFLARE_MAX_JOB_SEGMENT_MS`): its attempt ends this much
73
+ * before the segment, at most a tenth of it, so the finish runs inside the segment too.
74
+ */
75
+ export const SEGMENT_FINISH_RESERVE_MS = 30_000
58
76
  const BACKOFF_BASE_MS = 1_000
59
77
  const BACKOFF_MAX_MS = 5 * 60_000
60
78
 
@@ -76,7 +94,7 @@ export type JobFailureCode =
76
94
  | 'unauthorized'
77
95
  /** IAM could not be asked for the job's token. */
78
96
  | 'iam_unavailable'
79
- /** The process stopped a kicked run while shutting down; the schedule runs the job again. */
97
+ /** The process stopped a kicked run while shutting down, or the object's maintenance stopped a run; the schedule runs it again. */
80
98
  | 'stopped'
81
99
  /** The transport refused a message the handler did not catch; `reason` is the `EmailDeliveryError` code. */
82
100
  | 'mail_failed'
@@ -94,6 +112,8 @@ export interface JobClock {
94
112
  wait(ms: number, signal: AbortSignal): Promise<void>
95
113
  /** In [0, 1): the jitter of a retry's backoff. */
96
114
  random(): number
115
+ /** Epoch milliseconds: what an attempt's deadline is measured on; `Date.now()` when absent. */
116
+ now?(): number
97
117
  }
98
118
 
99
119
  export interface JobRunOptions {
@@ -126,6 +146,59 @@ export interface ContractJobs {
126
146
  * twice its period. The serving process calls it every `JOB_CHECK_INTERVAL_MS`; it never throws.
127
147
  */
128
148
  check(options?: Pick<JobRunOptions, 'reporter'>): Promise<void>
149
+ /**
150
+ * The serving process's wake timer on Bun, where crontab runs the schedule: reaps a crashed holder of every job and
151
+ * kicks each contract job with a due wake through the kick path, handing each kick to `track` until it ended. Answers
152
+ * how long until a wake is due next on the database's clock (0 while one still is), `null` when none is pending. The
153
+ * first call deletes the wakes of jobs the contract no longer declares.
154
+ */
155
+ wake(track: (kick: Promise<void>) => void): Promise<number | null>
156
+ /** What the Cloudflare object's alarm drives the jobs through; `null` without `@fabrika/db`. */
157
+ readonly scheduler: JobScheduler | null
158
+ }
159
+
160
+ /** What the scheduler's `take` did for one job in the alarm's transaction. */
161
+ export interface TakenLease {
162
+ /** The lease a run of the job now holds, for `run`; `null` when nothing was due or another run holds it. */
163
+ readonly fence: LeaseFence | null
164
+ /** Whether a crashed holder's lease was freed: `crashed` reports it once the transaction committed. */
165
+ readonly reaped: boolean
166
+ }
167
+
168
+ /**
169
+ * The jobs as the Cloudflare object's alarm drives them (ADR-0065 §5): over the wakes and leases of the object's
170
+ * storage, every time on the database's clock. The object holds the transactions and arms the alarm.
171
+ */
172
+ export interface JobScheduler {
173
+ /** Every job bound to run, the framework's included. */
174
+ readonly jobs: readonly string[]
175
+ /**
176
+ * Startup repair: reaps and reports every crashed run, of a job the contract no longer declares too; registers every
177
+ * job and surveils it (as `check`), deletes the wakes of jobs the contract no longer declares and the cron wakes of a
178
+ * changed expression, and keeps one cron wake per job at its next fire; an overdue one stays, so missed runs coalesce
179
+ * into one. `false` when the object's barrier admits no job.
180
+ */
181
+ repair(options?: Pick<JobRunOptions, 'reporter'>): Promise<boolean>
182
+ /** The jobs the contract no longer declares whose lease a crashed run still holds: `take` reaps them. */
183
+ orphans(): Promise<readonly string[]>
184
+ /** How long until every wake due within `withinMs` of the database's clock is due; 0 when none is early. */
185
+ earlyBy(withinMs: number): Promise<number>
186
+ /**
187
+ * Reaps a crashed holder of `job`'s lease, then, when one of its wakes is due, takes the lease, or sets the pending
188
+ * flag for the run holding it; of a job no longer declared (an orphan), only reaps. Statements only, for the caller's
189
+ * one transaction.
190
+ */
191
+ take(job: string): Promise<TakenLease>
192
+ /** Runs `job` under the lease `take` took. A failed run retries the wakes it owns durably. */
193
+ run(job: string, fence: LeaseFence, options?: JobRunOptions): Promise<JobRunResult>
194
+ /** Reports the crashed run `take` reaped. */
195
+ crashed(job: string, options?: Pick<JobRunOptions, 'reporter'>): Promise<void>
196
+ /** When the scheduler wakes next: `JobWakes.next`, read in `using` when given. */
197
+ next(using?: Session): Promise<Date | null>
198
+ /** How far the next wake is from the database's now, in milliseconds: `JobWakes.untilNext`, read in `using` when given. */
199
+ untilNext(using?: Session): Promise<number | null>
200
+ /** Reports a failure of the scheduler itself: the alarm's or the repair's. Never rejects. */
201
+ failed(step: 'alarm' | 'repair', error: unknown): Promise<void>
129
202
  }
130
203
 
131
204
  /** A run that cannot start: an unknown job. Its message is framework-authored. */
@@ -134,6 +207,7 @@ export class JobError extends Error {
134
207
  }
135
208
 
136
209
  export const systemClock: JobClock = Object.freeze({
210
+ now: () => Date.now(),
137
211
  wait: (ms: number, signal: AbortSignal) =>
138
212
  new Promise<void>((resolve) => {
139
213
  if (signal.aborted) {
@@ -174,6 +248,8 @@ interface RunnerContext {
174
248
  readonly database: ContractDatabase | null
175
249
  readonly ownedDimensions: ReadonlySet<string>
176
250
  readonly leases: JobLeases
251
+ /** The jobs' pending wakes, which a run's finish consumes, retries or schedules. */
252
+ readonly wakes: JobWakes
177
253
  readonly reporter: OperationsReporter
178
254
  readonly mint: CapabilityMint
179
255
  readonly data: DataAccess | null
@@ -188,6 +264,8 @@ interface RunnerContext {
188
264
  /** The environment's signed tokens: `ctx.tokens` of a job that declares `signs`. */
189
265
  readonly tokens: SignedTokens | null
190
266
  readonly url: AppUrl
267
+ /** What a job that declares `egress: 'public'` reaches; such a job reaches nothing without it. */
268
+ readonly publicEgress: PublicEgressRules | null
191
269
  }
192
270
 
193
271
  /** Where `fabrika app dev` shows what a handler threw, on the developer's own machine; nothing else sets one. */
@@ -227,6 +305,8 @@ export interface JobBindingInput {
227
305
  readonly tokens?: SignedTokens
228
306
  /** `ctx.url`: absolute links to the application's own procedures; without it every link is refused. */
229
307
  readonly url?: AppUrl
308
+ /** The rules of public egress, which `boot` reads from the environment; without them a public job reaches nothing. */
309
+ readonly publicEgress?: PublicEgressRules
230
310
  }
231
311
 
232
312
  interface BoundFrameworkJob {
@@ -310,11 +390,22 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
310
390
  }
311
391
  const names = Object.freeze(entries.map(([name]) => name))
312
392
  const framework = input.framework
313
- const contextFor = (leases: JobLeases, chosen: OperationsReporter | undefined): RunnerContext => ({
393
+ const declaredJobs: readonly (Scheduled & { readonly timeoutMs: number })[] = [
394
+ ...[...bound.values()].map((job) => ({ name: job.name, expression: job.artifact.schedule, schedule: job.schedule, timeoutMs: job.timeoutMs })),
395
+ ...[...frameworkJobs.values()].map((own) => ({
396
+ name: own.name,
397
+ expression: own.job.schedule,
398
+ schedule: own.schedule,
399
+ timeoutMs: own.job.timeoutMs,
400
+ })),
401
+ ]
402
+ let reconciled = false
403
+ const contextFor = (tables: FrameworkTables, chosen: OperationsReporter | undefined): RunnerContext => ({
314
404
  artifact: input.artifact,
315
405
  database: input.database,
316
406
  ownedDimensions: input.ownedDimensions,
317
- leases,
407
+ leases: tables.jobs,
408
+ wakes: tables.wakes,
318
409
  reporter: chosen ?? input.reporter,
319
410
  mint: input.mint,
320
411
  data: input.data,
@@ -326,6 +417,7 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
326
417
  mail: input.mail,
327
418
  tokens: input.tokens ?? null,
328
419
  url: input.url ?? appUrl(input.artifact, null),
420
+ publicEgress: input.publicEgress ?? null,
329
421
  // A job's kicks run only after the job is bound, so `kick` below is defined by then.
330
422
  kick: (name: string) => kick(name),
331
423
  })
@@ -335,57 +427,147 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
335
427
  run(name: string, options: JobRunOptions = {}): Promise<JobRunResult> {
336
428
  const own = frameworkJobs.get(name)
337
429
  if (own !== undefined && framework !== null) {
338
- return runFrameworkJob(own, contextFor(framework.jobs, options.reporter), options.clock ?? systemClock)
430
+ return admitted(
431
+ 'framework-job',
432
+ name,
433
+ (control) => runFrameworkJob(own, contextFor(framework, options.reporter), options.clock ?? systemClock, control, null),
434
+ )
339
435
  }
340
436
  const job = bound.get(name)
341
437
  if (job === undefined || framework === null) {
342
438
  throw new JobError(`no job '${name}' in the contract (jobs: ${names.length === 0 ? 'none' : names.join(', ')})`)
343
439
  }
344
- return runJob(job, contextFor(framework.jobs, options.reporter), options.clock ?? systemClock, 'wait').then((result) => {
345
- if (result.status === 'skipped') {
346
- console.error(`fabrika: a previous run of job '${name}' still holds its lease; this run does not start`)
347
- }
348
- return result
349
- })
440
+ return admitted(
441
+ 'job',
442
+ name,
443
+ (control) => runJob(job, contextFor(framework, options.reporter), options.clock ?? systemClock, 'wait', control, 'inline'),
444
+ )
445
+ .then((result) => {
446
+ if (result.status === 'skipped') {
447
+ console.error(`fabrika: a previous run of job '${name}' still holds its lease; this run does not start`)
448
+ }
449
+ return result
450
+ })
350
451
  },
351
452
  async check(options: Pick<JobRunOptions, 'reporter'> = {}): Promise<void> {
352
- if (framework === null) {
453
+ const admission = framework === null ? null : admitInvocation('framework-job')
454
+ if (framework === null || admission === null) {
353
455
  return
354
456
  }
355
- const context = contextFor(framework.jobs, options.reporter)
356
- for (const job of [...bound.values(), ...frameworkJobs.values()]) {
357
- try {
358
- await context.leases.register(job.name)
359
- await surveil(job, context)
360
- } catch {
361
- // A driver error may quote the database URL.
362
- console.error(`fabrika: the check of job '${job.name}' failed: driver_failed`)
457
+ try {
458
+ const context = contextFor(framework, options.reporter)
459
+ for (const job of [...bound.values(), ...frameworkJobs.values()]) {
460
+ try {
461
+ await context.leases.register(job.name)
462
+ await surveil(job, context)
463
+ } catch {
464
+ // A driver error may quote the database URL.
465
+ console.error(`fabrika: the check of job '${job.name}' failed: driver_failed`)
466
+ }
467
+ }
468
+ } finally {
469
+ admission.release()
470
+ }
471
+ },
472
+ async wake(track: (kick: Promise<void>) => void): Promise<number | null> {
473
+ if (framework === null) {
474
+ return null
475
+ }
476
+ if (!reconciled) {
477
+ await framework.wakes.reconcile(declaredJobs.map((job) => ({ job: job.name, expression: job.expression })))
478
+ reconciled = true
479
+ }
480
+ // A crashed holder's expiry is a wake too, and a kick never takes a lease over.
481
+ for (const job of declaredJobs) {
482
+ if (await framework.jobs.reapCrashed(job.name)) {
483
+ await reportCrashed(job.name, input.reporter)
484
+ }
485
+ }
486
+ const now = (await framework.jobs.now()).getTime()
487
+ for (const job of bound.values()) {
488
+ const due = (await framework.wakes.pending(job.name)).filter((wake) => wake.dueAt.getTime() <= now)
489
+ if (due.length > 0 && !(await ownedByHolder(job.name, now, framework.jobs, framework.wakes))) {
490
+ track(kick(job.name))
363
491
  }
364
492
  }
493
+ const next = await framework.wakes.next()
494
+ return next === null ? null : Math.max(0, next.getTime() - now)
365
495
  },
496
+ scheduler: framework === null ? null : jobScheduler({
497
+ framework,
498
+ declared: declaredJobs,
499
+ contextFor: (reporter) => contextFor(framework, reporter),
500
+ run(name: string, fence: LeaseFence, options: JobRunOptions): Promise<JobRunResult> {
501
+ const clock = options.clock ?? systemClock
502
+ const context = contextFor(framework, options.reporter)
503
+ const own = frameworkJobs.get(name)
504
+ if (own !== undefined) {
505
+ return admitted('framework-job', name, (control) => runFrameworkJob(own, context, clock, control, fence))
506
+ }
507
+ const job = bound.get(name)
508
+ if (job === undefined) {
509
+ throw new JobError(`no job '${name}' in the contract (jobs: ${names.length === 0 ? 'none' : names.join(', ')})`)
510
+ }
511
+ return admitted('job', name, (control) => runJob(job, context, clock, { fence }, control, 'durable'))
512
+ },
513
+ }),
366
514
  })
367
515
  const stopping = new AbortController()
368
516
  const kick = async (name: string): Promise<void> => {
369
517
  const job = bound.get(name)
370
- if (job !== undefined && framework !== null) {
371
- await kickOnce(job, contextFor(framework.jobs, undefined), stopping.signal)
518
+ if (job === undefined || framework === null) {
519
+ return
520
+ }
521
+ const admission = admitInvocation('kick')
522
+ if (admission === null) {
523
+ // A kick hurries a run; the schedule still runs the job once the object opens.
524
+ console.error(`fabrika: job '${name}' was not kicked: the object is in maintenance`)
525
+ return
526
+ }
527
+ try {
528
+ await kickOnce(job, contextFor(framework, undefined), {
529
+ signal: AbortSignal.any([admission.signal, stopping.signal]),
530
+ track: admission.track,
531
+ })
532
+ } finally {
533
+ admission.release()
372
534
  }
373
535
  }
374
536
  return { jobs, declared, kick, stopKicks: () => stopping.abort() }
375
537
  }
376
538
 
539
+ /**
540
+ * What stops one run, and where the handler work that outlives its attempt is counted: the run's admission at the
541
+ * object's barrier, which a drain aborts, and for a kick also the process's stop.
542
+ */
543
+ type RunControl = Pick<InvocationAdmission, 'signal' | 'track'>
544
+
545
+ /** A run admitted at the current event's gate, released once it ended; a refused one is `stopped` and starts nothing. */
546
+ async function admitted(kind: 'job' | 'framework-job', name: string, run: (control: RunControl) => Promise<JobRunResult>): Promise<JobRunResult> {
547
+ const admission = admitInvocation(kind)
548
+ if (admission === null) {
549
+ console.error(`fabrika: job '${name}' did not start: the object is in maintenance; its schedule runs it again`)
550
+ return { status: 'failed', attempts: 0, code: 'stopped' }
551
+ }
552
+ try {
553
+ return await run(admission)
554
+ } finally {
555
+ admission.release()
556
+ }
557
+ }
558
+
377
559
  /**
378
560
  * One kick: a run while the lease is free, else the shared pending flag. Neither waits for a lock, so a kick never
379
561
  * holds a connection requests share while a run writes. After a failed run the flag stays for the schedule.
380
562
  */
381
- async function kickOnce(job: BoundJob, context: RunnerContext, stop: AbortSignal): Promise<void> {
382
- if (stop.aborted) {
563
+ async function kickOnce(job: BoundJob, context: RunnerContext, control: RunControl): Promise<void> {
564
+ if (control.signal.aborted) {
383
565
  // The flag waits for no lock, and another process that is not stopping runs the job.
384
566
  await markPending(job, context)
385
567
  return
386
568
  }
387
- if ((await runJob(job, context, systemClock, 'try', stop)).status === 'skipped') {
388
- await handOver(job, context, systemClock, stop)
569
+ if ((await runJob(job, context, systemClock, 'try', control, 'inline')).status === 'skipped') {
570
+ await handOver(job, context, systemClock, control, 'inline')
389
571
  }
390
572
  }
391
573
 
@@ -393,11 +575,11 @@ async function kickOnce(job: BoundJob, context: RunnerContext, stop: AbortSignal
393
575
  * Leaves the flag to the run holding the lease, which takes it after it succeeded. That run may have finished before
394
576
  * the flag was set, so the lease is tried once more; `null` when another run still holds it.
395
577
  */
396
- async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock, stop?: AbortSignal): Promise<JobRunResult | null> {
578
+ async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock, control: RunControl, retries: Retries): Promise<JobRunResult | null> {
397
579
  if (!(await markPending(job, context))) {
398
580
  return null
399
581
  }
400
- const result = await runJob(job, context, clock, 'try', stop)
582
+ const result = await runJob(job, context, clock, 'try', control, retries)
401
583
  return result.status === 'skipped' ? null : result
402
584
  }
403
585
 
@@ -444,13 +626,123 @@ function backoffMs(attempt: number, random: number): number {
444
626
  return Math.round(ceiling / 2 + (ceiling / 2) * random)
445
627
  }
446
628
 
629
+ const nowOf = (clock: JobClock): number => clock.now?.() ?? Date.now()
630
+
631
+ /** How much of a segment that ends at `deadline` is left, never below zero. */
632
+ const remainingMs = (clock: JobClock, deadline: number): number => Math.max(0, deadline - nowOf(clock))
633
+
634
+ const finishReserveMs = (timeoutMs: number): number => Math.min(SEGMENT_FINISH_RESERVE_MS, Math.floor(timeoutMs / 10))
635
+
636
+ const OUT_OF_SEGMENT: unique symbol = Symbol('fabrika.job.outOfSegment')
637
+
638
+ /**
639
+ * `step` while the segment that ends at `end` lasts (always, without one). Past its end the run stops waiting for it;
640
+ * only a step that is safe to abandon goes here: a read, or the refusal of the attempt's writes, never a finish.
641
+ */
642
+ async function withinSegment<T>(clock: JobClock, end: number | null, step: Promise<T>): Promise<T | typeof OUT_OF_SEGMENT> {
643
+ if (end === null) {
644
+ return await step
645
+ }
646
+ const timer = new AbortController()
647
+ // Abandoned past the segment, its rejection is nobody's: the race below still answers it while it is awaited.
648
+ step.catch(() => undefined)
649
+ try {
650
+ return await Promise.race([step, clock.wait(remainingMs(clock, end), timer.signal).then((): typeof OUT_OF_SEGMENT => OUT_OF_SEGMENT)])
651
+ } finally {
652
+ timer.abort()
653
+ }
654
+ }
655
+
656
+ /** A run whose segment ended before its finish started: its lease expires, the reap reports the crash, and the alarm runs it again. */
657
+ function outOfSegment(job: { readonly name: string }, attempts: number): JobRunResult {
658
+ console.error(`fabrika: job '${job.name}' reached the end of its segment before its finish; its lease expires and the alarm runs it again`)
659
+ return { status: 'failed', attempts, code: 'timeout' }
660
+ }
661
+
662
+ const MINUTE_MS = 60_000
663
+ /** How far ahead a next fire is looked for, as far as `previousDue` looks back. */
664
+ const NEXT_FIRE_DAYS = 366 * 5
665
+
666
+ /** The first due minute of `schedule` after `after`, or `null` when none falls in the window. */
667
+ function nextFire(schedule: CronSchedule, after: Date): Date | null {
668
+ const hours = [...schedule.hours].sort((left, right) => left - right)
669
+ const minutes = [...schedule.minutes].sort((left, right) => left - right)
670
+ const [firstHour, firstMinute] = [hours[0], minutes[0]]
671
+ if (firstHour === undefined || firstMinute === undefined) {
672
+ return null
673
+ }
674
+ const today = Date.UTC(after.getUTCFullYear(), after.getUTCMonth(), after.getUTCDate())
675
+ for (let ahead = 0; ahead <= NEXT_FIRE_DAYS; ahead++) {
676
+ const day = today + ahead * 24 * 60 * MINUTE_MS
677
+ // The day's first scheduled minute is due exactly when the day matches.
678
+ if (!isDueAt(schedule, new Date(day + (firstHour * 60 + firstMinute) * MINUTE_MS))) {
679
+ continue
680
+ }
681
+ for (const hour of hours) {
682
+ for (const minute of minutes) {
683
+ const time = day + (hour * 60 + minute) * MINUTE_MS
684
+ if (time > after.getTime()) {
685
+ return new Date(time)
686
+ }
687
+ }
688
+ }
689
+ }
690
+ return null
691
+ }
692
+
693
+ /** A cron wake's dedupe key: one pending wake per job and expression. */
694
+ const cronKey = (job: string, expression: string): string => `cron:${job}:${expression}`
695
+
696
+ /** A job's schedule as the scheduler and a finish read it. */
697
+ interface Scheduled {
698
+ readonly name: string
699
+ /** The cron expression as declared: its wake's key. */
700
+ readonly expression: string
701
+ readonly schedule: CronSchedule
702
+ }
703
+
704
+ /**
705
+ * Whether a run holds `job`'s lease and owns every wake due by `now`, as it took them with the lease: its finish consumes
706
+ * them, so a kick or a pending run for them would only repeat that run. A post-commit kick's run usually owns its wake.
707
+ */
708
+ async function ownedByHolder(job: string, now: number, leases: JobLeases, wakes: JobWakes): Promise<boolean> {
709
+ const state = await leases.state(job)
710
+ if (state === null || state.holder === null) {
711
+ return false
712
+ }
713
+ return (await wakes.pending(job)).filter((wake) => wake.dueAt.getTime() <= now).every((wake) => wake.owned)
714
+ }
715
+
716
+ /** The wakes the run holding `job`'s lease owns: those it took with the lease. */
717
+ async function ownedWakes(job: string, context: RunnerContext): Promise<readonly Wake[]> {
718
+ return (await context.wakes.pending(job)).filter((wake) => wake.owned)
719
+ }
720
+
721
+ /**
722
+ * What a finish does with the owned wakes when the run is done with them, succeeded or failed past its retries: they
723
+ * are consumed, and when the job keeps a cron wake (always, for a run the alarm started) its next fire is scheduled
724
+ * under its key. On Bun, where crontab runs the schedule, a job keeps none.
725
+ */
726
+ async function completion(job: Scheduled, context: RunnerContext, durable: boolean): Promise<FinishWakes> {
727
+ if (!durable && !(await context.wakes.pending(job.name)).some((wake) => wake.reason === 'cron')) {
728
+ return { kind: 'completed' }
729
+ }
730
+ const dueAt = nextFire(job.schedule, await context.leases.now())
731
+ return dueAt === null ? { kind: 'completed' } : { kind: 'completed', nextCron: { expression: job.expression, dueAt } }
732
+ }
733
+
734
+ /** A crashed run (its lease expired before it finished): one report. */
735
+ async function reportCrashed(job: string, reporter: OperationsReporter): Promise<void> {
736
+ const message = `job '${job}' failed: crashed (its run's lease expired before the run finished)`
737
+ console.error(`fabrika: ${message}`)
738
+ await reporter.report({ fingerprint: ['fabrika-job-failed', job], type: 'JobFailed', message, tags: { job, code: 'crashed' } })
739
+ }
740
+
447
741
  /** A crashed run (its lease expired before it finished) and a job overdue by twice its period: one report each. */
448
742
  async function surveil(job: { readonly name: string; readonly schedule: CronSchedule }, context: RunnerContext): Promise<void> {
449
743
  const tags = { job: job.name }
450
744
  if (await context.leases.reapCrashed(job.name)) {
451
- const message = `job '${job.name}' failed: crashed (its run's lease expired before the run finished)`
452
- console.error(`fabrika: ${message}`)
453
- await context.reporter.report({ fingerprint: ['fabrika-job-failed', job.name], type: 'JobFailed', message, tags: { ...tags, code: 'crashed' } })
745
+ await reportCrashed(job.name, context.reporter)
454
746
  }
455
747
  const period = periodAt(job.schedule, await context.leases.now())
456
748
  const claim = period === null ? null : await context.leases.claimOverdue(job.name, period)
@@ -468,30 +760,156 @@ async function surveil(job: { readonly name: string; readonly schedule: CronSche
468
760
  })
469
761
  }
470
762
 
763
+ interface SchedulerInput {
764
+ readonly framework: FrameworkTables
765
+ /** Every job bound to run, with the attempt time its lease is taken for. */
766
+ readonly declared: readonly (Scheduled & { readonly timeoutMs: number })[]
767
+ readonly contextFor: (reporter: OperationsReporter | undefined) => RunnerContext
768
+ readonly run: (job: string, fence: LeaseFence, options: JobRunOptions) => Promise<JobRunResult>
769
+ }
770
+
771
+ function jobScheduler(input: SchedulerInput): JobScheduler {
772
+ const { framework, declared } = input
773
+ const byName = new Map(declared.map((job) => [job.name, job]))
774
+ return Object.freeze({
775
+ jobs: Object.freeze(declared.map((job) => job.name)),
776
+ async repair(options: Pick<JobRunOptions, 'reporter'> = {}): Promise<boolean> {
777
+ const admission = admitInvocation('framework-job')
778
+ if (admission === null) {
779
+ return false
780
+ }
781
+ try {
782
+ const context = input.contextFor(options.reporter)
783
+ for (const job of await framework.wakes.expiredHolders()) {
784
+ if (await context.leases.reapCrashed(job)) {
785
+ await reportCrashed(job, context.reporter)
786
+ }
787
+ }
788
+ for (const job of declared) {
789
+ await context.leases.register(job.name)
790
+ await surveil(job, context)
791
+ }
792
+ await framework.wakes.reconcile(declared.map((job) => ({ job: job.name, expression: job.expression })))
793
+ const now = await context.leases.now()
794
+ for (const job of declared) {
795
+ const dueAt = nextFire(job.schedule, now)
796
+ if (dueAt !== null) {
797
+ await framework.wakes.schedule({ job: job.name, reason: 'cron', dueAt, dedupe: cronKey(job.name, job.expression) })
798
+ }
799
+ }
800
+ return true
801
+ } finally {
802
+ admission.release()
803
+ }
804
+ },
805
+ async orphans(): Promise<readonly string[]> {
806
+ return (await framework.wakes.expiredHolders()).filter((job) => !byName.has(job))
807
+ },
808
+ async earlyBy(withinMs: number): Promise<number> {
809
+ const now = (await framework.jobs.now()).getTime()
810
+ let early = 0
811
+ for (const job of declared) {
812
+ for (const wake of await framework.wakes.pending(job.name)) {
813
+ const ahead = wake.dueAt.getTime() - now
814
+ if (ahead <= withinMs) {
815
+ early = Math.max(early, ahead)
816
+ }
817
+ }
818
+ }
819
+ return early
820
+ },
821
+ async take(name: string): Promise<TakenLease> {
822
+ const job = byName.get(name)
823
+ const leases = framework.jobs
824
+ if (job === undefined) {
825
+ // A removed job runs no more; only a crashed run's lease is freed, so it stops arming the alarm.
826
+ return { fence: null, reaped: await leases.reapCrashed(name) }
827
+ }
828
+ await leases.register(name)
829
+ const reaped = await leases.reapCrashed(name)
830
+ const now = (await leases.now()).getTime()
831
+ const due = (await framework.wakes.pending(name)).filter((wake) => wake.dueAt.getTime() <= now)
832
+ if (due.length === 0) {
833
+ return { fence: null, reaped }
834
+ }
835
+ const fence = await leases.tryAcquire(name, attemptId(), job.timeoutMs, job.timeoutMs + LEASE_GRACE_MS)
836
+ if (fence === null && !(await ownedByHolder(name, now, leases, framework.wakes))) {
837
+ // The run holding the lease runs once more after it succeeded; a wake it did not own stays pending.
838
+ await leases.markPending(name)
839
+ }
840
+ return { fence, reaped }
841
+ },
842
+ run(job: string, fence: LeaseFence, options: JobRunOptions = {}): Promise<JobRunResult> {
843
+ return input.run(job, fence, options)
844
+ },
845
+ async crashed(job: string, options: Pick<JobRunOptions, 'reporter'> = {}): Promise<void> {
846
+ await reportCrashed(job, input.contextFor(options.reporter).reporter)
847
+ },
848
+ next(using?: Session): Promise<Date | null> {
849
+ return framework.wakes.next(using)
850
+ },
851
+ untilNext(using?: Session): Promise<number | null> {
852
+ return framework.wakes.untilNext(using)
853
+ },
854
+ async failed(step: 'alarm' | 'repair', error: unknown): Promise<void> {
855
+ try {
856
+ await input.contextFor(undefined).reporter.report({
857
+ fingerprint: ['fabrika-alarm-failed', step],
858
+ type: 'AlarmFailed',
859
+ message: `the job scheduler's ${step} failed`,
860
+ tags: { step },
861
+ error,
862
+ })
863
+ } catch {
864
+ console.error(`fabrika: the failure of the job scheduler's ${step} could not be reported`)
865
+ }
866
+ },
867
+ })
868
+ }
869
+
471
870
  interface Progress {
472
871
  attempts: number
473
872
  }
474
873
 
874
+ /** What the attempt that succeeded kicked: `null` until a handler returned. */
875
+ interface Kicked {
876
+ recorder: KickRecorder | null
877
+ }
878
+
475
879
  /**
476
880
  * How a run takes the lease: `wait` is `fabrika job run`'s, in a process of its own; `try` is a kick's and a follow-up
477
- * run's, which never wait for a lock.
881
+ * run's, which never wait for a lock; a fence is one the alarm's scheduler already took.
882
+ */
883
+ type Acquisition = 'wait' | 'try' | { readonly fence: LeaseFence }
884
+
885
+ /**
886
+ * How a failed attempt is retried: `inline` in the same run, after a backoff, under a renewed lease; `durable` (a run the
887
+ * Cloudflare alarm started) by moving the wakes it owns to a later due time, so the alarm runs it again. A durable run
888
+ * that owns no wake retries inline: there is nothing to move.
478
889
  */
479
- type Acquisition = 'wait' | 'try'
890
+ type Retries = 'inline' | 'durable'
480
891
 
481
892
  /**
482
- * A run of the job, then one more while a kick arrived after the last one started and it succeeded. Once `stop`
483
- * aborted, no follow-up run starts: the pending flag stays for the schedule.
893
+ * A run of the job, then one more while a kick arrived after the last one started and it succeeded. Once the run's
894
+ * control aborted, no follow-up run starts: the pending flag stays for the schedule.
484
895
  */
485
- async function runJob(job: BoundJob, context: RunnerContext, clock: JobClock, acquisition: Acquisition, stop?: AbortSignal): Promise<JobRunResult> {
486
- let result = await runOnce(job, context, clock, acquisition, stop)
487
- while (result.status === 'succeeded' && stop?.aborted !== true && (await pendingTaken(job, context))) {
488
- const next = await runOnce(job, context, clock, 'try', stop)
896
+ async function runJob(
897
+ job: BoundJob,
898
+ context: RunnerContext,
899
+ clock: JobClock,
900
+ acquisition: Acquisition,
901
+ control: RunControl,
902
+ retries: Retries,
903
+ ): Promise<JobRunResult> {
904
+ let result = await runOnce(job, context, clock, acquisition, control, retries)
905
+ while (result.status === 'succeeded' && !control.signal.aborted && (await pendingTaken(job, context))) {
906
+ const next = await runOnce(job, context, clock, 'try', control, retries)
489
907
  if (next.status !== 'skipped') {
490
908
  result = next
491
909
  continue
492
910
  }
493
911
  // A newer run holds the lease and may have read before the kick's row: the flag goes back to it.
494
- const handed = await handOver(job, context, clock, stop)
912
+ const handed = await handOver(job, context, clock, control, retries)
495
913
  return handed ?? result
496
914
  }
497
915
  return result
@@ -507,11 +925,23 @@ async function pendingTaken(job: BoundJob, context: RunnerContext): Promise<bool
507
925
  }
508
926
  }
509
927
 
510
- async function runOnce(job: BoundJob, context: RunnerContext, clock: JobClock, acquisition: Acquisition, stop?: AbortSignal): Promise<JobRunResult> {
928
+ async function runOnce(
929
+ job: BoundJob,
930
+ context: RunnerContext,
931
+ clock: JobClock,
932
+ acquisition: Acquisition,
933
+ control: RunControl,
934
+ retries: Retries,
935
+ ): Promise<JobRunResult> {
511
936
  const progress: Progress = { attempts: 0 }
512
937
  try {
513
- return await runLeased(job, context, clock, progress, acquisition, stop)
938
+ return await runLeased(job, context, clock, progress, acquisition, control, retries)
514
939
  } catch {
940
+ if (control.signal.aborted) {
941
+ // A stopped run's lease write is refused by the object's write fence; the lease expires and the schedule runs it.
942
+ console.error(`fabrika: job '${job.name}' stopped before it released its lease; its schedule runs it again`)
943
+ return { status: 'failed', attempts: progress.attempts, code: 'stopped' }
944
+ }
515
945
  // Only the lease store throws here: the handler's errors stay inside its attempt. Its message may quote the URL.
516
946
  return failed(job, progress.attempts, failure('driver_failed'), context.reporter)
517
947
  }
@@ -523,32 +953,47 @@ async function runLeased(
523
953
  clock: JobClock,
524
954
  progress: Progress,
525
955
  acquisition: Acquisition,
526
- stop?: AbortSignal,
956
+ control: RunControl,
957
+ retries: Retries,
527
958
  ): Promise<JobRunResult> {
528
959
  const { leases } = context
529
- await leases.register(job.name)
530
960
  const leaseMs = job.timeoutMs + LEASE_GRACE_MS
531
- if (stop?.aborted === true) {
532
- return { status: 'skipped' }
533
- }
534
961
  let fence: LeaseFence | null
535
- if (acquisition === 'wait') {
536
- await surveil(job, context)
537
- fence = await leases.acquire(job.name, attemptId(), job.timeoutMs, leaseMs)
962
+ if (typeof acquisition === 'object') {
963
+ fence = acquisition.fence
538
964
  } else {
539
- fence = await leases.tryAcquire(job.name, attemptId(), job.timeoutMs, leaseMs)
965
+ await leases.register(job.name)
966
+ if (control.signal.aborted) {
967
+ return { status: 'skipped' }
968
+ }
969
+ if (acquisition === 'wait') {
970
+ await surveil(job, context)
971
+ fence = await leases.acquire(job.name, attemptId(), job.timeoutMs, leaseMs)
972
+ } else {
973
+ fence = await leases.tryAcquire(job.name, attemptId(), job.timeoutMs, leaseMs)
974
+ }
540
975
  }
541
976
  if (fence === null) {
542
977
  return { status: 'skipped' }
543
978
  }
979
+ // The segment is measured from the lease, as the database's attempt deadline is. A run the alarm started has one
980
+ // segment: its attempts end before it by the finish's reserve, and no finish starts past it.
981
+ const segmentEnd = retries === 'durable' ? nowOf(clock) + job.timeoutMs : null
982
+ const attemptEnd = (
983
+ start: number,
984
+ ): number => (segmentEnd === null ? start + job.timeoutMs : Math.min(start + job.timeoutMs, segmentEnd - finishReserveMs(job.timeoutMs)))
985
+ let deadline = attemptEnd(nowOf(clock))
544
986
  // This run covers every kick before it: their rows committed before it reads.
545
987
  await pendingTaken(job, context)
988
+ const owned = retries === 'durable' ? await ownedWakes(job.name, context) : []
989
+ const durable = owned.length > 0
990
+ const scheduled: Scheduled = { name: job.name, expression: job.artifact.schedule, schedule: job.schedule }
546
991
  const attempts = job.artifact.retries + 1
547
- progress.attempts = 1
548
- // The jobs the succeeding attempt kicked; a failed attempt's kicks are dropped with its writes.
549
- const kicked: string[] = []
550
- let outcome = await attemptOnce(job, fence, progress.attempts, clock, context, kicked, stop)
551
- while (outcome !== null && outcome.code !== 'stopped' && !outcome.final && progress.attempts < attempts) {
992
+ progress.attempts = durable ? Math.max(...owned.map((wake) => wake.attempt)) + 1 : 1
993
+ // The kicks of the succeeding attempt; a failed attempt's kicks are dropped with its writes.
994
+ const kicked: Kicked = { recorder: null }
995
+ let outcome = await attemptOnce(job, fence, progress.attempts, deadline, segmentEnd, clock, context, kicked, control)
996
+ while (!durable && outcome !== null && outcome.code !== 'stopped' && !outcome.final && progress.attempts < attempts) {
552
997
  console.error(`fabrika: job '${job.name}' attempt ${progress.attempts} of ${attempts} failed: ${describeFailure(outcome)}`)
553
998
  const delay = backoffMs(progress.attempts, clock.random())
554
999
  // A new generation at once: nothing the failed attempt still has in flight commits from here on.
@@ -557,24 +1002,63 @@ async function runLeased(
557
1002
  return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
558
1003
  }
559
1004
  fence = next
560
- await clock.wait(delay, stop ?? new AbortController().signal)
1005
+ deadline = attemptEnd(nowOf(clock) + delay)
1006
+ await clock.wait(delay, control.signal)
561
1007
  progress.attempts++
562
- outcome = await attemptOnce(job, fence, progress.attempts, clock, context, kicked, stop)
1008
+ outcome = await attemptOnce(job, fence, progress.attempts, deadline, segmentEnd, clock, context, kicked, control)
1009
+ }
1010
+ const held = fence
1011
+ // The finish, unless the segment ended first; a finish once started is awaited, as it may be committing.
1012
+ const finish = async (status: 'succeeded' | 'failed', wakes: Promise<FinishWakes | undefined>): Promise<boolean | typeof OUT_OF_SEGMENT> => {
1013
+ const applied = await withinSegment(clock, segmentEnd, wakes)
1014
+ if (applied === OUT_OF_SEGMENT || (segmentEnd !== null && remainingMs(clock, segmentEnd) === 0)) {
1015
+ return OUT_OF_SEGMENT
1016
+ }
1017
+ return await leases.finish(held, status, applied)
563
1018
  }
564
1019
  if (outcome === null) {
565
- if (await leases.finish(fence, 'succeeded')) {
1020
+ // Before the finish, so a crash between it and a kick's start leaves the kick's wake to the scheduler.
1021
+ const kicks = kicked.recorder
1022
+ for (const wake of kicks?.wakes(await leases.now()) ?? []) {
1023
+ await context.wakes.schedule(wake)
1024
+ }
1025
+ const finished = await finish('succeeded', completion(scheduled, context, durable))
1026
+ if (finished === OUT_OF_SEGMENT) {
1027
+ return outOfSegment(job, progress.attempts)
1028
+ }
1029
+ if (finished) {
566
1030
  // After the lease is released, so a job that kicks another never waits on its own lease; a kick never rejects.
567
- await Promise.all(kicked.map((name) => context.kick(name)))
1031
+ await Promise.all((kicks?.kicked() ?? []).map((name) => context.kick(name)))
568
1032
  return { status: 'succeeded', attempts: progress.attempts }
569
1033
  }
570
1034
  return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
571
1035
  }
572
- await leases.finish(fence, 'failed')
573
1036
  if (outcome.code === 'stopped') {
574
- // Not a failure of the job: the schedule runs it again, so Operations hears nothing.
1037
+ // Its wakes stay, so the alarm or the schedule runs it again. Not a failure of the job: Operations hears nothing.
1038
+ await leases.finish(fence, 'failed')
575
1039
  console.error(`fabrika: job '${job.name}' stopped: the process is shutting down; its schedule runs it again`)
576
1040
  return { status: 'failed', attempts: progress.attempts, code: 'stopped' }
577
1041
  }
1042
+ if (durable && !outcome.final && progress.attempts < attempts) {
1043
+ const at = new Date(nowOf(clock) + backoffMs(progress.attempts, clock.random()))
1044
+ console.error(
1045
+ `fabrika: job '${job.name}' attempt ${progress.attempts} of ${attempts} failed: ${describeFailure(outcome)}; it runs again at ${at.toISOString()}`,
1046
+ )
1047
+ const finished = await finish('failed', Promise.resolve({ kind: 'retry', at }))
1048
+ if (finished === OUT_OF_SEGMENT) {
1049
+ return outOfSegment(job, progress.attempts)
1050
+ }
1051
+ if (!finished) {
1052
+ return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
1053
+ }
1054
+ return outcome.why === null
1055
+ ? { status: 'failed', attempts: progress.attempts, code: outcome.code }
1056
+ : { status: 'failed', attempts: progress.attempts, code: outcome.code, reason: outcome.why.reason }
1057
+ }
1058
+ // Past its retries a run is done with its wakes: the next cron fire is scheduled, a kick's wake removed.
1059
+ if ((await finish('failed', completion(scheduled, context, durable))) === OUT_OF_SEGMENT) {
1060
+ return outOfSegment(job, progress.attempts)
1061
+ }
578
1062
  return failed(job, progress.attempts, outcome, context.reporter)
579
1063
  }
580
1064
 
@@ -613,14 +1097,27 @@ async function failed(job: { readonly name: string }, attempts: number, outcome:
613
1097
 
614
1098
  const TIMED_OUT: unique symbol = Symbol('fabrika.job.timedOut')
615
1099
 
616
- /** One run of a framework job: its lease, one attempt in a frame fenced by it, and the outcome reported like a job's. */
617
- async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext, clock: JobClock): Promise<JobRunResult> {
1100
+ /**
1101
+ * One run of a framework job: its lease, one attempt in a frame fenced by it, and the outcome reported like a job's.
1102
+ * A run its control stopped is not a failure: the schedule runs it again. `acquired` is the lease the alarm's scheduler
1103
+ * took: such a run is done with the wakes it owns whether it succeeded or failed, as a framework job has no retries.
1104
+ */
1105
+ async function runFrameworkJob(
1106
+ bound: BoundFrameworkJob,
1107
+ context: RunnerContext,
1108
+ clock: JobClock,
1109
+ control: RunControl,
1110
+ acquired: LeaseFence | null,
1111
+ ): Promise<JobRunResult> {
618
1112
  const { leases } = context
619
- let fence: LeaseFence | null
1113
+ const scheduled: Scheduled = { name: bound.name, expression: bound.job.schedule, schedule: bound.schedule }
1114
+ let fence: LeaseFence | null = acquired
620
1115
  try {
621
- await leases.register(bound.name)
622
- await surveil(bound, context)
623
- fence = await leases.acquire(bound.name, attemptId(), bound.job.timeoutMs, bound.job.timeoutMs + LEASE_GRACE_MS)
1116
+ if (fence === null) {
1117
+ await leases.register(bound.name)
1118
+ await surveil(bound, context)
1119
+ fence = await leases.acquire(bound.name, attemptId(), bound.job.timeoutMs, bound.job.timeoutMs + LEASE_GRACE_MS)
1120
+ }
624
1121
  } catch {
625
1122
  return failed(bound, 0, failure('driver_failed'), context.reporter)
626
1123
  }
@@ -628,14 +1125,18 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
628
1125
  console.error(`fabrika: a previous run of job '${bound.name}' still holds its lease; this run does not start`)
629
1126
  return { status: 'skipped' }
630
1127
  }
1128
+ // A run the alarm took the lease for has one segment, whose end is kept for the finish, as a contract job's.
1129
+ const segmentEnd = acquired === null ? null : nowOf(clock) + bound.job.timeoutMs
631
1130
  const frame = new RequestFrame(fence.holder, { fence })
632
1131
  const cancel = new AbortController()
633
1132
  const timer = new AbortController()
634
1133
  let outcome: JobFailure | null
635
1134
  try {
636
1135
  const ran = runInRequest(frame, () => bound.job.run(context.reporter, cancel.signal)).then(() => null, (error: unknown) => failureOf(error, null))
637
- const timeout = clock.wait(bound.job.timeoutMs, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
638
- const result = await Promise.race([ran, timeout])
1136
+ control.track(ran)
1137
+ const budget = segmentEnd === null ? bound.job.timeoutMs : bound.job.timeoutMs - finishReserveMs(bound.job.timeoutMs)
1138
+ const timeout = clock.wait(budget, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
1139
+ const result = await Promise.race([ran, timeout, stoppedBy(control, timer.signal)])
639
1140
  outcome = result === TIMED_OUT ? failure('timeout') : result
640
1141
  } finally {
641
1142
  frame.end()
@@ -644,52 +1145,103 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
644
1145
  try {
645
1146
  // Writes are refused before the run hears it was stopped: a scanner that answers on abort commits nothing.
646
1147
  try {
647
- await leases.cancelInFlight(fence)
1148
+ await withinSegment(clock, segmentEnd, leases.cancelInFlight(fence))
648
1149
  } finally {
649
1150
  cancel.abort()
650
1151
  }
651
- const finished = await leases.finish(fence, outcome === null ? 'succeeded' : 'failed')
1152
+ // A stopped run keeps its wakes, so the alarm runs it again once the object opens.
1153
+ const done = outcome === null || (acquired !== null && outcome.code !== 'stopped')
1154
+ const wakes = await withinSegment(clock, segmentEnd, done ? completion(scheduled, context, acquired !== null) : Promise.resolve(undefined))
1155
+ if (wakes === OUT_OF_SEGMENT || (segmentEnd !== null && remainingMs(clock, segmentEnd) === 0)) {
1156
+ return outOfSegment(bound, 1)
1157
+ }
1158
+ // Awaited once started: a finish may be committing.
1159
+ const finished = await leases.finish(fence, outcome === null ? 'succeeded' : 'failed', wakes)
652
1160
  if (outcome === null) {
653
1161
  return finished ? { status: 'succeeded', attempts: 1 } : failed(bound, 1, failure('lease_lost'), context.reporter)
654
1162
  }
655
1163
  } catch {
656
- return failed(bound, 1, failure('driver_failed'), context.reporter)
1164
+ // A stopped run's lease write is refused by the object's write fence; the lease expires and the schedule runs it.
1165
+ if (!control.signal.aborted) {
1166
+ return failed(bound, 1, failure('driver_failed'), context.reporter)
1167
+ }
1168
+ }
1169
+ // Here a run without a failure of its own is one whose stop refused its lease write.
1170
+ if (outcome === null || outcome.code === 'stopped' || control.signal.aborted) {
1171
+ console.error(`fabrika: job '${bound.name}' stopped: the object entered maintenance; its schedule runs it again`)
1172
+ return { status: 'failed', attempts: 1, code: 'stopped' }
657
1173
  }
658
1174
  return failed(bound, 1, outcome, context.reporter)
659
1175
  }
660
1176
 
661
1177
  /** Read afresh at each call: the signal may abort while the caller awaits. */
662
- const isStopped = (stop: AbortSignal | undefined): boolean => stop?.aborted === true
1178
+ const isStopped = (control: RunControl): boolean => control.signal.aborted
1179
+
1180
+ /** Resolves `stopped` once the run's control aborts; the listener goes when `done` aborts. */
1181
+ function stoppedBy(control: RunControl, done: AbortSignal): Promise<JobFailure> {
1182
+ return new Promise<JobFailure>((resolve) => {
1183
+ if (control.signal.aborted) {
1184
+ resolve(failure('stopped'))
1185
+ }
1186
+ control.signal.addEventListener('abort', () => resolve(failure('stopped')), { once: true, signal: done })
1187
+ })
1188
+ }
663
1189
 
664
1190
  /**
665
- * `null` when the handler finished, else the attempt's failure code. Either way the attempt's frame ends and the
666
- * statements it still has running are cancelled before the runner touches the lease again.
1191
+ * `null` when the handler finished, else the attempt's failure code. One deadline bounds the attempt: IAM's answer for
1192
+ * the job's key, then the handler with only what is left of it. Either way the attempt's frame ends and the statements
1193
+ * it still has running are cancelled before the runner touches the lease again, waited for only while the segment lasts.
667
1194
  */
668
1195
  async function attemptOnce(
669
1196
  job: BoundJob,
670
1197
  fence: LeaseFence,
671
1198
  attempt: number,
1199
+ deadline: number,
1200
+ segmentEnd: number | null,
672
1201
  clock: JobClock,
673
1202
  context: RunnerContext,
674
- kicked: string[],
675
- stop?: AbortSignal,
1203
+ kicked: Kicked,
1204
+ control: RunControl,
676
1205
  ): Promise<JobFailure | null> {
677
- kicked.length = 0
678
- if (isStopped(stop)) {
1206
+ kicked.recorder = null
1207
+ if (isStopped(control)) {
679
1208
  return failure('stopped')
680
1209
  }
681
1210
  // The job's authority is what IAM signs for its key at the start of the attempt; a revoked key stops the next one.
682
- const authority = await job.credential.authority()
683
- if (isStopped(stop)) {
1211
+ const setup = new AbortController()
1212
+ const exchange = job.credential.authority()
1213
+ // Its settlement, not only the attempt's: a drain waits for an exchange the attempt stopped waiting for.
1214
+ control.track(exchange)
1215
+ let authority: WorkloadAuthority
1216
+ try {
1217
+ const answered = await Promise.race([
1218
+ exchange,
1219
+ clock.wait(remainingMs(clock, deadline), setup.signal).then((): typeof TIMED_OUT => TIMED_OUT),
1220
+ stoppedBy(control, setup.signal),
1221
+ ])
1222
+ if (answered === TIMED_OUT || 'code' in answered) {
1223
+ // The next attempt asks IAM afresh rather than wait on an exchange that outlived this one.
1224
+ job.credential.abandon(exchange)
1225
+ return answered === TIMED_OUT ? failure('timeout') : answered
1226
+ }
1227
+ authority = answered
1228
+ } finally {
1229
+ setup.abort()
1230
+ }
1231
+ if (isStopped(control)) {
684
1232
  return failure('stopped')
685
1233
  }
686
1234
  if (!authority.ok) {
687
1235
  return failure(authority.reason === 'unavailable' ? 'iam_unavailable' : 'unauthorized')
688
1236
  }
1237
+ const budget = remainingMs(clock, deadline)
1238
+ if (budget === 0) {
1239
+ return failure('timeout')
1240
+ }
689
1241
  const frame = new RequestFrame(fence.holder, { fence })
690
1242
  const cancel = new AbortController()
691
1243
  const timer = new AbortController()
692
- const signal = stop === undefined ? cancel.signal : AbortSignal.any([cancel.signal, stop])
1244
+ const signal = AbortSignal.any([cancel.signal, control.signal])
693
1245
  const kicks = kickRecorder(`job '${job.name}'`, job.site?.kicks ?? [])
694
1246
  const ctx = { ...jobContext(job, frame, attempt, signal, context, authority.perms), kick: kicks.kick }
695
1247
  try {
@@ -698,28 +1250,24 @@ async function attemptOnce(
698
1250
  if (site === null) {
699
1251
  await job.handler({ ctx })
700
1252
  } else {
701
- await runAsSite(`job '${job.name}'`, site.egress, async () => await job.handler({ ctx }))
1253
+ await runAsSite(`job '${job.name}'`, siteEgress(site.egress, context.publicEgress), async () => await job.handler({ ctx }))
702
1254
  }
703
- kicked.push(...kicks.kicked())
1255
+ kicked.recorder = kicks
704
1256
  }).then(() => null, (error: unknown) => {
705
1257
  const failed = failureOf(error, job.mailer === null ? null : (context.mail[job.mailer]?.sender.capabilities ?? null))
706
1258
  if (failed.code === 'handler_failed') context.revealHandlerErrors?.(`job '${job.name}' attempt ${attempt}`, error)
707
1259
  return failed
708
1260
  })
709
- const timeout = clock.wait(job.timeoutMs, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
710
- const stopped = new Promise<JobFailure>((resolve) => {
711
- if (stop?.aborted === true) {
712
- resolve(failure('stopped'))
713
- }
714
- stop?.addEventListener('abort', () => resolve(failure('stopped')), { once: true, signal: timer.signal })
715
- })
716
- const result = await Promise.race([ran, timeout, stopped])
1261
+ // Its settlement, not only the end of its frame: a drain waits for a handler that outlived the attempt.
1262
+ control.track(ran)
1263
+ const timeout = clock.wait(budget, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
1264
+ const result = await Promise.race([ran, timeout, stoppedBy(control, timer.signal)])
717
1265
  return result === TIMED_OUT ? failure('timeout') : result
718
1266
  } finally {
719
1267
  frame.end()
720
1268
  cancel.abort()
721
1269
  timer.abort()
722
- await context.leases.cancelInFlight(fence)
1270
+ await withinSegment(clock, segmentEnd, context.leases.cancelInFlight(fence))
723
1271
  }
724
1272
  }
725
1273