@fabrika/app 0.9.0-cloudflare.2 → 0.9.0-cloudflare.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +20 -18
  3. package/skill/references/app-setup.md +29 -16
  4. package/skill/references/endpoint.md +24 -22
  5. package/skill/references/errors.md +3 -2
  6. package/skill/references/jobs.md +13 -3
  7. package/skill/references/tests.md +7 -0
  8. package/src/bun.ts +159 -6
  9. package/src/cloudflare/application.ts +543 -73
  10. package/src/cloudflare/export.ts +621 -0
  11. package/src/cloudflare/import.ts +1097 -0
  12. package/src/cloudflare/maintenance.ts +518 -0
  13. package/src/cloudflare/ops.ts +147 -18
  14. package/src/cloudflare/restore.ts +167 -0
  15. package/src/cloudflare/tracker.ts +192 -0
  16. package/src/cloudflare.ts +3 -2
  17. package/src/contract/artifact-schema.ts +7 -1
  18. package/src/contract/build.ts +6 -2
  19. package/src/contract/context.ts +26 -9
  20. package/src/contract/defaults.ts +22 -2
  21. package/src/contract/define.ts +1 -1
  22. package/src/contract/group.ts +1 -1
  23. package/src/files/kinds.ts +54 -24
  24. package/src/implement/anonymous.ts +1 -10
  25. package/src/implement/boot.ts +21 -9
  26. package/src/implement/dispatch.ts +171 -33
  27. package/src/implement/egress.ts +34 -8
  28. package/src/implement/invocation.ts +50 -10
  29. package/src/implement/workload.ts +18 -3
  30. package/src/jobs/runner.ts +639 -96
  31. package/src/testing/kit.ts +60 -0
  32. package/types/src/bun.d.ts +34 -1
  33. package/types/src/cloudflare/application.d.ts +36 -5
  34. package/types/src/cloudflare/export.d.ts +16 -0
  35. package/types/src/cloudflare/import.d.ts +58 -0
  36. package/types/src/cloudflare/maintenance.d.ts +117 -0
  37. package/types/src/cloudflare/ops.d.ts +39 -4
  38. package/types/src/cloudflare/restore.d.ts +9 -0
  39. package/types/src/cloudflare/tracker.d.ts +29 -0
  40. package/types/src/cloudflare.d.ts +2 -1
  41. package/types/src/contract/artifact-schema.d.ts +2 -0
  42. package/types/src/contract/context.d.ts +24 -9
  43. package/types/src/contract/defaults.d.ts +19 -1
  44. package/types/src/contract/define.d.ts +1 -1
  45. package/types/src/contract/group.d.ts +1 -1
  46. package/types/src/files/kinds.d.ts +17 -6
  47. package/types/src/implement/anonymous.d.ts +0 -4
  48. package/types/src/implement/boot.d.ts +11 -0
  49. package/types/src/implement/dispatch.d.ts +31 -0
  50. package/types/src/implement/invocation.d.ts +17 -7
  51. package/types/src/implement/workload.d.ts +5 -0
  52. package/types/src/jobs/runner.d.ts +60 -2
  53. package/types/src/testing/kit.d.ts +16 -0
@@ -1,27 +1,41 @@
1
- // The Durable Object an application environment runs in (ADR-0065): it migrates its SQLite, boots the application and
2
- // serves every request and minute tick its Worker forwards. A plain class, so Bun and Lopata load it without
3
- // `cloudflare:workers`.
1
+ // The Durable Object an application environment runs in (ADR-0065): it migrates its SQLite, boots the application,
2
+ // serves every request and repair tick its Worker forwards, and runs the jobs from its alarm, which it arms from their
3
+ // durable wakes in the transaction of every change. A plain class, so Bun and Lopata load it without `cloudflare:workers`.
4
4
 
5
- import { FABRIKA_PROXY_SECRET, OPS_PREFIX, type OpsBootFailureContent, PROXY_SECRET_HEADER } from '@fabrika/auth-core'
6
- import { isScopedDatabase } from '@fabrika/db/framework'
5
+ import {
6
+ FABRIKA_APPLICATION_OBJECT,
7
+ FABRIKA_APPLICATION_OBJECTS,
8
+ FABRIKA_CONTRACT_DIGEST,
9
+ FABRIKA_PROXY_SECRET,
10
+ OPS_PREFIX,
11
+ type OpsBootFailureContent,
12
+ type OpsMode,
13
+ parseApplicationObjects,
14
+ PROXY_SECRET_HEADER,
15
+ } from '@fabrika/auth-core'
16
+ import { isScopedDatabase, type LeaseFence, type Session } from '@fabrika/db/framework'
7
17
  import { SQLITE_FRAMEWORK_MIGRATIONS } from '@fabrika/db/migrations-sqlite'
8
- import { openSqlite, type SqliteStorage } from '@fabrika/db/sqlite'
9
- import { migrateSqlite } from '@fabrika/db/sqlite-migrate'
18
+ import { openSqlite, type SqliteCursor, type SqliteStorage } from '@fabrika/db/sqlite'
19
+ import { migrateSqlite, type SqliteMigrationBundle, type SqliteMigrationTarget } from '@fabrika/db/sqlite-migrate'
10
20
  import { AsyncLocalStorage } from 'node:async_hooks'
11
21
  import { timingSafeEqual } from 'node:crypto'
12
22
  import type { RequestExecutionContext } from '../fetch.js'
23
+ import { FILE_TOMBSTONE_RETENTION_MS } from '../files/kinds.js'
13
24
  import { boot, type ContractApp } from '../implement/boot.js'
14
25
  import { type CrashEvents, environmentCrashReporter, reportSurvivedCrashes } from '../implement/crashes.js'
15
- import { proxySecretDigest } from '../implement/dispatch.js'
26
+ import { MaintenanceFenceError, maintenanceRefusal, proxySecretDigest, runGated } from '../implement/dispatch.js'
16
27
  import { type CapturedEnvironment, captureEnvironment } from '../implement/environment.js'
17
28
  import type { ContractImplementation } from '../implement/implement.js'
18
29
  import type { CrashReporter } from '../jobs/report.js'
19
- import { isDueAt } from '../jobs/schedule.js'
30
+ import { type JobClock, type JobScheduler, systemClock } from '../jobs/runner.js'
20
31
  import { environmentScrubber, type Scrubber } from '../jobs/scrub.js'
21
32
  import { bindingAssets, bindingKeySource, bindingMembership, bindingTerminals, bindingWorkloadTokens, connectBindings } from './bindings.js'
33
+ import { importInProgress } from './import.js'
34
+ import { type BarrierEvent, MaintenanceBarrier, r2MaintenanceStore } from './maintenance.js'
22
35
  import { CLOUDFLARE_ASSETS_BINDING, CLOUDFLARE_FILES_BINDING } from './names.js'
23
- import { type OpsAccess, opsAccess, type OpsStorage, serveOps } from './ops.js'
36
+ import { type OpsAccess, opsAccess, type OpsRecovery, type OpsStorage, serveOps } from './ops.js'
24
37
  import { r2FileObjectStore } from './r2-files.js'
38
+ import { trackPending } from './tracker.js'
25
39
 
26
40
  /** The application module, imported only after the environment was captured: the generated entrypoint's `import()`. */
27
41
  export type ImplementationLoader = () => Promise<{ readonly default: ContractImplementation }>
@@ -39,22 +53,74 @@ export interface ContractWorkerOptions {
39
53
  readonly rejections?: CrashEvents
40
54
  /** What posts a report to Operations' ingest; `fetch` by default. */
41
55
  readonly operationsFetch?: (request: Request) => Promise<Response>
56
+ /** The jobs' and the alarm's waits and an attempt's deadline; the system clock by default. */
57
+ readonly clock?: JobClock
42
58
  }
43
59
 
44
60
  /**
45
- * Marks the Worker's minute tick, with its `scheduledTime`. The Worker's `fetch` removes it from every request it
61
+ * Marks the Worker's repair tick, with its `scheduledTime`. The Worker's `fetch` removes it from every request it
46
62
  * forwards, so only its own `scheduled` handler reaches the tick.
47
63
  */
48
64
  export const TICK_HEADER = 'X-Fabrika-Tick'
49
65
 
50
- /** What the object needs of its `ctx.storage`; a Durable Object's `DurableObjectStorage` fits it. */
51
- export type ApplicationStorage = SqliteStorage & OpsStorage
66
+ /** A wake runs at most this early: `Date.now()` is pinned between I/O, so an alarm may start a moment before it (I6). */
67
+ export const ALARM_SLACK_MS = 50
68
+ /** After an internal error the alarm is armed again this far ahead; the repair tick covers a re-arm that failed. */
69
+ export const ALARM_RETRY_MS = 30_000
70
+ /**
71
+ * The object's KV key that holds the name of the object it is, recorded at its first verified event: an alarm names no
72
+ * object, and the one ordinary traffic reaches may since have changed. Outside the SQLite, so no dump carries it.
73
+ */
74
+ export const OBJECT_NAME_KEY = 'fabrika:object'
75
+ /** A repair tick's time: the Worker's `scheduledTime`, digits only. */
76
+ const TICK_TIME = /^\d{1,16}$/
77
+
78
+ /** The object's alarm. On a SQLite-backed object `setAlarm` and `deleteAlarm` commit with the open transaction. */
79
+ export interface AlarmStorage {
80
+ setAlarm(scheduledTime: number): Promise<void>
81
+ deleteAlarm(): Promise<void>
82
+ }
83
+
84
+ /**
85
+ * What the object needs of its `ctx.storage`; a Durable Object's `DurableObjectStorage` fits it. A test double without
86
+ * an alarm arms none.
87
+ */
88
+ export type ApplicationStorage = SqliteStorage & OpsStorage & Partial<AlarmStorage>
52
89
 
53
90
  /** What the object needs of its `DurableObjectState`. */
54
91
  export interface ApplicationState {
55
- readonly storage: ApplicationStorage
92
+ readonly storage: ApplicationStorage & Partial<Omit<OpsRecovery, 'abort'>>
56
93
  blockConcurrencyWhile<T>(callback: () => Promise<T>): Promise<T>
57
94
  waitUntil(promise: Promise<unknown>): void
95
+ /** workerd resets the object; the test double has none. */
96
+ abort?(reason?: string): void
97
+ }
98
+
99
+ /** `state` with every `waitUntil` promise recorded by the isolate's pending-work tracker (the ops operation `timers`). */
100
+ export function trackedState(state: ApplicationState): ApplicationState {
101
+ const { abort } = state
102
+ return {
103
+ storage: state.storage,
104
+ blockConcurrencyWhile: (callback) => state.blockConcurrencyWhile(callback),
105
+ waitUntil: (promise) => state.waitUntil(trackPending(promise)),
106
+ ...(abort === undefined ? {} : { abort: (reason?: string) => abort.call(state, reason) }),
107
+ }
108
+ }
109
+
110
+ /** The object's recovery when its storage and state have every member PITR needs (ADR-0070), else `null`. */
111
+ function objectRecovery(state: ApplicationState): OpsRecovery | null {
112
+ const { storage } = state
113
+ const { getCurrentBookmark, getBookmarkForTime, onNextSessionRestoreBookmark } = storage
114
+ const { abort } = state
115
+ if (getCurrentBookmark === undefined || getBookmarkForTime === undefined || onNextSessionRestoreBookmark === undefined || abort === undefined) {
116
+ return null
117
+ }
118
+ return {
119
+ getCurrentBookmark: () => getCurrentBookmark.call(storage),
120
+ getBookmarkForTime: (time) => getBookmarkForTime.call(storage, time),
121
+ onNextSessionRestoreBookmark: (bookmark) => onNextSessionRestoreBookmark.call(storage, bookmark),
122
+ abort: (reason) => abort.call(state, reason),
123
+ }
58
124
  }
59
125
 
60
126
  /** The framework's bindings: IAM's application entrypoint and the Workers Assets of the SPA. No handler sees them. */
@@ -81,15 +147,120 @@ interface ApplicationSetup {
81
147
 
82
148
  let registered: ApplicationSetup | null = null
83
149
 
84
- /** The storage of the object whose event runs; the work an event hands to `waitUntil` keeps it. */
85
- const objectStorage = new AsyncLocalStorage<ApplicationStorage>()
150
+ /** The storage of the object whose event runs, and the barrier's view of that event; work it hands to `waitUntil` keeps both. */
151
+ interface ObjectScope {
152
+ readonly storage: ApplicationStorage
153
+ readonly event: BarrierEvent
154
+ }
155
+
156
+ const objectScope = new AsyncLocalStorage<ObjectScope>()
157
+
158
+ /** The cursors of the fenced transaction that runs: its commit checks whether any of them wrote. */
159
+ const openTransaction = new AsyncLocalStorage<SqliteCursor[]>()
86
160
 
87
- function eventStorage(): ApplicationStorage {
88
- const storage = objectStorage.getStore()
89
- if (storage === undefined) {
161
+ function eventScope(): ObjectScope {
162
+ const scope = objectScope.getStore()
163
+ if (scope === undefined) {
90
164
  throw new Error('a statement ran outside any event of an application object: the framework runs every event in its object scope')
91
165
  }
92
- return storage
166
+ return scope
167
+ }
168
+
169
+ /** The write fence (ADR-0069 §3): a write of an invocation admitted before a restrictive transition rolls back. */
170
+ function fence(event: BarrierEvent, cursors: readonly SqliteCursor[]): void {
171
+ if (cursors.some((cursor) => cursor.rowsWritten > 0) && !event.writable()) {
172
+ throw new MaintenanceFenceError()
173
+ }
174
+ }
175
+
176
+ /**
177
+ * The storage every framework and application write on SQLite reaches, bound to the object whose event runs. The
178
+ * fence runs inside each transaction before it commits; a statement outside one runs in a transaction of its own.
179
+ */
180
+ function fencedStorage(): SqliteStorage {
181
+ return {
182
+ sql: {
183
+ exec(query, ...bindings) {
184
+ const { storage, event } = eventScope()
185
+ const cursors = openTransaction.getStore()
186
+ if (cursors !== undefined) {
187
+ const cursor = storage.sql.exec(query, ...bindings)
188
+ cursors.push(cursor)
189
+ return cursor
190
+ }
191
+ return storage.transactionSync(() => {
192
+ const cursor = storage.sql.exec(query, ...bindings)
193
+ const rows = cursor.toArray()
194
+ fence(event, [cursor])
195
+ return { toArray: () => rows, rowsWritten: cursor.rowsWritten }
196
+ })
197
+ },
198
+ },
199
+ transaction(closure) {
200
+ const { storage, event } = eventScope()
201
+ // A nested transaction shares its parent's cursors, so the outer commit checks every write inside it.
202
+ const cursors = openTransaction.getStore() ?? []
203
+ return storage.transaction(() =>
204
+ openTransaction.run(cursors, async () => {
205
+ const result = await closure()
206
+ fence(event, cursors)
207
+ return result
208
+ })
209
+ )
210
+ },
211
+ transactionSync(closure) {
212
+ const { storage, event } = eventScope()
213
+ const cursors = openTransaction.getStore() ?? []
214
+ return storage.transactionSync(() =>
215
+ openTransaction.run(cursors, () => {
216
+ const result = closure()
217
+ fence(event, cursors)
218
+ return result
219
+ })
220
+ )
221
+ },
222
+ }
223
+ }
224
+
225
+ /** Whether migrations may be applied: in `open` and the migration phase; a restricted cold start only looks. */
226
+ type MigrationPolicy = 'apply' | 'refuse'
227
+
228
+ /** A restricted cold start found a migration to apply: the application does not boot until the operation's migrate. */
229
+ class MigrationsPending extends Error {
230
+ override readonly name = 'MigrationsPending'
231
+ }
232
+
233
+ /** The object holds an import marker (ADR-0070 §6): it neither boots nor migrates until the import completes or is discarded. */
234
+ class ImportInProgress extends Error {
235
+ override readonly name = 'ImportInProgress'
236
+ }
237
+
238
+ /** Whether the object holds an import marker; a storage without the synchronous KV (the bun:sqlite double) holds none. */
239
+ function importing(storage: ApplicationStorage): boolean {
240
+ const { kv } = storage
241
+ return kv !== undefined && importInProgress({ kv })
242
+ }
243
+
244
+ function isMigrationsPending(error: unknown): boolean {
245
+ return error instanceof MigrationsPending || (error instanceof Error && error.cause instanceof MigrationsPending)
246
+ }
247
+
248
+ /**
249
+ * The migration runner's target that applies nothing: it answers the runner's reads, and the first write it would make
250
+ * (the ledger's DDL, a migration, a digest) throws `MigrationsPending` instead.
251
+ */
252
+ function refusingTarget(storage: ApplicationStorage): SqliteMigrationTarget {
253
+ return {
254
+ exec(query, ...bindings) {
255
+ if (!/^\s*select\b/i.test(query)) {
256
+ throw new MigrationsPending('a migration is pending')
257
+ }
258
+ return storage.sql.exec(query, ...bindings)
259
+ },
260
+ transactionSync() {
261
+ throw new MigrationsPending('a migration is pending')
262
+ },
263
+ }
93
264
  }
94
265
 
95
266
  /** What the object classes of this isolate boot: the generated entrypoint's `createContractWorker` registers it. */
@@ -108,12 +279,23 @@ function globalRejectionEvents(): CrashEvents {
108
279
  }
109
280
  }
110
281
 
111
- /** The framework's tables, then the application's, each migration once: a booted storage applies nothing. */
112
- async function migrate(storage: ApplicationStorage, migrations: readonly ApplicationMigration[]): Promise<void> {
113
- await migrateSqlite({ exec: (query, ...bindings) => storage.sql.exec(query, ...bindings), transactionSync: (fn) => storage.transactionSync(fn) }, [
282
+ /**
283
+ * The framework's tables, then the application's, each migration once: a booted storage applies nothing. Under
284
+ * `refuse` a pending migration throws `MigrationsPending` before anything is written.
285
+ */
286
+ async function migrate(storage: ApplicationStorage, migrations: readonly ApplicationMigration[], policy: MigrationPolicy): Promise<void> {
287
+ const target: SqliteMigrationTarget = policy === 'apply'
288
+ ? { exec: (query, ...bindings) => storage.sql.exec(query, ...bindings), transactionSync: (fn) => storage.transactionSync(fn) }
289
+ : refusingTarget(storage)
290
+ await migrateSqlite(target, migrationBundles(migrations))
291
+ }
292
+
293
+ /** The bundles the object applies, in order: the framework's tables, then the application's. */
294
+ function migrationBundles(migrations: readonly ApplicationMigration[]): readonly SqliteMigrationBundle[] {
295
+ return [
114
296
  { bundle: 'fabrika', migrations: SQLITE_FRAMEWORK_MIGRATIONS },
115
297
  { bundle: 'app', migrations },
116
- ])
298
+ ]
117
299
  }
118
300
 
119
301
  async function bootApplication(
@@ -122,19 +304,14 @@ async function bootApplication(
122
304
  env: Readonly<Record<string, unknown>>,
123
305
  environment: CapturedEnvironment,
124
306
  crashes: CrashReporter | null,
307
+ policy: MigrationPolicy,
125
308
  ): Promise<Booted> {
126
309
  const { default: implementation } = await setup.load()
127
310
  const migrates = isScopedDatabase(implementation.database)
128
311
  if (migrates) {
129
- await migrate(state.storage, setup.options.migrations ?? [])
312
+ await migrate(state.storage, setup.options.migrations ?? [], policy)
130
313
  }
131
- const database = migrates
132
- ? openSqlite({
133
- sql: { exec: (query, ...bindings) => eventStorage().sql.exec(query, ...bindings) },
134
- transaction: (closure) => eventStorage().transaction(closure),
135
- transactionSync: (closure) => eventStorage().transactionSync(closure),
136
- })
137
- : undefined
314
+ const database = migrates ? openSqlite(fencedStorage()) : undefined
138
315
  const app = boot(implementation, {
139
316
  environment,
140
317
  iam: bindingKeySource(env['IAM']),
@@ -145,10 +322,13 @@ async function bootApplication(
145
322
  connectBindings: connectBindings(env),
146
323
  database,
147
324
  files: r2FileObjectStore(env[CLOUDFLARE_FILES_BINDING]),
325
+ // A removed file's object outlives its row for the window, so a side restore finds its bytes (ADR-0070).
326
+ fileRetentionMs: FILE_TOMBSTONE_RETENTION_MS,
148
327
  // The Worker forwards only with the proxy secret, which the object checked; `boot` checks it again.
149
328
  requireProxySecret: true,
150
329
  // The proxy Worker sets `X-Fabrika-Client-Ip` from `CF-Connecting-IP`, the visitor Cloudflare saw.
151
330
  clientAddress: 'vouched',
331
+ projectWakes: (transaction) => projectWakes(setup, transaction),
152
332
  ...(setup.options.operationsFetch === undefined ? {} : { operationsFetch: setup.options.operationsFetch }),
153
333
  })
154
334
  const booted: Booted = { app, migrates }
@@ -167,18 +347,26 @@ async function attach(
167
347
  setup: ApplicationSetup | null,
168
348
  state: ApplicationState,
169
349
  env: Readonly<Record<string, unknown>>,
350
+ policy: MigrationPolicy,
170
351
  failed: (failure: OpsBootFailureContent) => void,
171
352
  ): Promise<Booted> {
172
353
  if (setup === null) {
173
354
  throw new Error('no contract application is registered: the generated entrypoint calls createContractWorker before an object starts')
174
355
  }
356
+ // Before anything loads or migrates, and no boot failure: a half-loaded import never boots.
357
+ if (importing(state.storage)) {
358
+ throw new ImportInProgress('import_in_progress')
359
+ }
175
360
  const environment = captureEnvironment(Object.fromEntries(Object.entries(env).filter(([name]) => !FRAMEWORK_BINDINGS.has(name))))
176
361
  const crashes = environmentCrashReporter(environment, setup.options.operationsFetch)
177
362
  try {
178
- const booted = await attachOnce(setup, state, env, environment, crashes)
363
+ const booted = await attachOnce(setup, state, env, environment, crashes, policy)
179
364
  setup.failureReported = false
180
365
  return booted
181
366
  } catch (error) {
367
+ if (isMigrationsPending(error)) {
368
+ throw error
369
+ }
182
370
  failed(bootFailure(error, environmentScrubber(environment)))
183
371
  if (crashes !== null && !setup.failureReported) {
184
372
  setup.failureReported = true
@@ -194,9 +382,10 @@ async function attachOnce(
194
382
  env: Readonly<Record<string, unknown>>,
195
383
  environment: CapturedEnvironment,
196
384
  crashes: CrashReporter | null,
385
+ policy: MigrationPolicy,
197
386
  ): Promise<Booted> {
198
387
  if (setup.booted === null) {
199
- const booting = bootApplication(setup, state, env, environment, crashes)
388
+ const booting = bootApplication(setup, state, env, environment, crashes, policy)
200
389
  setup.booted = booting
201
390
  // A failed boot is not kept: the next object instance boots again and fails with the same problems.
202
391
  booting.catch(() => {
@@ -206,7 +395,7 @@ async function attachOnce(
206
395
  }
207
396
  const booted = await setup.booted
208
397
  if (booted.migrates) {
209
- await migrate(state.storage, setup.options.migrations ?? [])
398
+ await migrate(state.storage, setup.options.migrations ?? [], policy)
210
399
  }
211
400
  return booted
212
401
  }
@@ -221,18 +410,44 @@ function bootFailure(error: unknown, scrubber: Scrubber): OpsBootFailureContent
221
410
  return { at, message, stack: null }
222
411
  }
223
412
 
224
- type Outcome = { readonly booted: Booted } | { readonly failure: unknown }
413
+ /**
414
+ * `pending`: a restricted cold start found a migration to apply, or the object holds an import marker, so the
415
+ * application did not boot; that is no failure.
416
+ */
417
+ type Outcome = { readonly booted: Booted } | { readonly failure: unknown } | { readonly pending: true }
225
418
 
226
419
  /** The wait after an instance's first failed boot before an event boots again; it doubles with each failure, to the cap. */
227
420
  const BOOT_RETRY_MS = 1_000
228
421
  const BOOT_RETRY_CAP_MS = 60_000
229
422
 
230
- /** A job run handed to `waitUntil`; the runner logs and reports its own failures. */
231
- async function runQuietly(app: ContractApp, name: string): Promise<void> {
423
+ /** The alarm armed at `next`, or deleted when nothing is pending; inside a transaction it commits with it. */
424
+ async function armAlarm(storage: ApplicationStorage, next: Date | null): Promise<void> {
425
+ if (next === null) {
426
+ await storage.deleteAlarm?.()
427
+ } else {
428
+ await storage.setAlarm?.(next.getTime())
429
+ }
430
+ }
431
+
432
+ /**
433
+ * The runtime's projection of the jobs' wakes and leases (ADR-0065 §5, I1): the store runs it in the transaction of
434
+ * every change, and it arms the alarm of the object whose event runs from `next`. A throw rolls the change back.
435
+ */
436
+ async function projectWakes(setup: ApplicationSetup, transaction: Session): Promise<void> {
437
+ // A wake or lease changes only once the application booted.
438
+ const booted = setup.booted === null ? null : await setup.booted
439
+ const scheduler = booted?.app.jobs.scheduler ?? null
440
+ if (scheduler !== null) {
441
+ await armAlarm(eventScope().storage, await scheduler.next(transaction))
442
+ }
443
+ }
444
+
445
+ /** A run the alarm took the lease for, handed to `waitUntil`; the runner logs and reports its own failures. */
446
+ async function runScheduled(scheduler: JobScheduler, job: string, fence: LeaseFence, clock: JobClock): Promise<void> {
232
447
  try {
233
- await app.jobs.run(name)
448
+ await scheduler.run(job, fence, { clock })
234
449
  } catch {
235
- console.error(`fabrika: job '${name}' did not start: driver_failed`)
450
+ console.error(`fabrika: job '${job}' did not start: driver_failed`)
236
451
  }
237
452
  }
238
453
 
@@ -245,21 +460,41 @@ export class FabrikaApplication {
245
460
  readonly #secret: Uint8Array | null
246
461
  /** What the platform wrote for Control's ops channel; `null` refuses every ops request. */
247
462
  readonly #ops: OpsAccess | null
248
- /** The instance's last failed boot, kept in memory for the ops view. */
463
+ /** The object ordinary traffic reaches, so the one this instance is when such traffic arrives; `null` when malformed. */
464
+ readonly #object: string | null
465
+ /** The object this instance is, once a verified event named it or its storage recorded it (`OBJECT_NAME_KEY`). */
466
+ #name: string | null = null
467
+ /** The maintenance barrier, read from R2 once per instance; a failed read is not kept, so the next request reads again. */
468
+ #barrier: Promise<MaintenanceBarrier> | null = null
469
+ #barrierObject: string | null = null
470
+ /** The instance's failed boot, kept in memory for the ops view until a boot succeeds. */
249
471
  #bootFailure: OpsBootFailureContent | null = null
250
472
  /** The instance's boot, `null` until the first event that needs the application: an ops request never starts it. */
251
473
  #outcome: Promise<Outcome> | null = null
252
474
  #failedBoots = 0
253
475
  /** When an event may boot again after a failed boot; until then every request answers the failure. */
254
476
  #retryAt = 0
477
+ /** Whether this instance ran the scheduler's startup repair, which its first open event does. */
478
+ #repaired = false
479
+ readonly #clock: JobClock
255
480
 
256
481
  constructor(state: ApplicationState, env: Readonly<Record<string, unknown>>) {
257
- this.#state = state
482
+ this.#state = trackedState(state)
258
483
  this.#env = env
259
484
  this.#setup = registered
485
+ this.#clock = registered?.options.clock ?? systemClock
260
486
  const secret = env[FABRIKA_PROXY_SECRET]
261
487
  this.#secret = typeof secret === 'string' && secret !== '' ? proxySecretDigest(secret) : null
262
488
  this.#ops = opsAccess(env)
489
+ const object = env[FABRIKA_APPLICATION_OBJECT]
490
+ const objects = env[FABRIKA_APPLICATION_OBJECTS]
491
+ this.#object = parseApplicationObjects(typeof object === 'string' ? object : undefined, typeof objects === 'string' ? objects : undefined)
492
+ ?.object ?? null
493
+ }
494
+
495
+ #contractDigest(): string | null {
496
+ const digest = this.#env[FABRIKA_CONTRACT_DIGEST]
497
+ return typeof digest === 'string' ? digest : null
263
498
  }
264
499
 
265
500
  async fetch(request: Request): Promise<Response> {
@@ -268,28 +503,134 @@ export class FabrikaApplication {
268
503
  return new Response(null, { status: 404 })
269
504
  }
270
505
  if (new URL(request.url).pathname.startsWith(OPS_PREFIX)) {
506
+ // The mode before the request: a transition that opens the object arms its alarm, which repairs.
507
+ let before: OpsMode | null = null
271
508
  // Served from storage alone: it never starts a boot or a retry, and waits for one under way only once authenticated.
272
- return await serveOps(request, this.#ops, {
509
+ const response = await serveOps(request, this.#ops, {
273
510
  storage: this.#state.storage,
511
+ bucket: this.#env[CLOUDFLARE_FILES_BINDING],
512
+ recovery: objectRecovery(this.#state),
513
+ contractDigest: this.#contractDigest(),
514
+ migrations: migrationBundles(this.#setup?.options.migrations ?? []),
274
515
  bootFailure: async () => {
275
516
  await this.#outcome
276
517
  return this.#bootFailure
277
518
  },
519
+ maintenance: async (object) => {
520
+ const barrier = await this.#maintenance(object)
521
+ before ??= barrier.mode
522
+ return barrier
523
+ },
524
+ migrate: () => this.#migrated(),
278
525
  })
526
+ if (before !== null && before !== 'open') {
527
+ await this.#reopened()
528
+ }
529
+ return response
530
+ }
531
+ let barrier: MaintenanceBarrier
532
+ try {
533
+ barrier = await this.#maintenance(this.#object)
534
+ } catch {
535
+ // Fails closed: a state R2 could not give is never read as `open` (ADR-0069 §3, cold starts).
536
+ return maintenanceRefusal()
537
+ }
538
+ if (importing(this.#state.storage)) {
539
+ // ADR-0070 §6: whatever the mode, a half-loaded import serves nothing; the tick repairs nothing.
540
+ return request.headers.get(TICK_HEADER) === null ? maintenanceRefusal() : new Response(null, { status: 204 })
541
+ }
542
+ if (barrier.mode !== 'open' && request.headers.get(TICK_HEADER) !== null) {
543
+ // The tick repairs nothing and writes nothing until the object opens.
544
+ return new Response(null, { status: 204 })
545
+ }
546
+ if (barrier.mode === 'closed' || barrier.mode === 'inspect') {
547
+ return maintenanceRefusal()
548
+ }
549
+ const outcome = await this.#current(barrier.mode === 'open' ? 'apply' : 'refuse')
550
+ if ('pending' in outcome) {
551
+ return maintenanceRefusal()
279
552
  }
280
- const outcome = await this.#current()
281
553
  if ('failure' in outcome) {
282
554
  throw outcome.failure
283
555
  }
284
- return await objectStorage.run(this.#state.storage, () => this.#serve(outcome.booted, request))
556
+ const event = barrier.event()
557
+ return await objectScope.run({ storage: this.#state.storage, event }, () => runGated(event, () => this.#serve(outcome.booted, request)))
558
+ }
559
+
560
+ /** The name of the object this instance is: the one its storage recorded, or `null` before a verified event named it. */
561
+ #recorded(): string | null {
562
+ if (this.#name === null) {
563
+ const stored = this.#state.storage.kv?.get(OBJECT_NAME_KEY)
564
+ if (typeof stored === 'string' && stored !== '') {
565
+ this.#name = stored
566
+ }
567
+ }
568
+ return this.#name
569
+ }
570
+
571
+ /**
572
+ * Whether this instance is the object a verified event names; the first such event records the name, which never
573
+ * changes after. A storage without KV (the bun:sqlite double) keeps it in memory only.
574
+ */
575
+ #named(object: string): boolean {
576
+ const recorded = this.#recorded()
577
+ if (recorded !== null) {
578
+ return recorded === object
579
+ }
580
+ this.#state.storage.kv?.put(OBJECT_NAME_KEY, object)
581
+ this.#name = object
582
+ return true
583
+ }
584
+
585
+ /** The barrier of the object this instance is; every name it is asked under must be the same one. */
586
+ #maintenance(object: string | null): Promise<MaintenanceBarrier> {
587
+ if (object === null) {
588
+ return Promise.reject(new Error(`${FABRIKA_APPLICATION_OBJECT} and ${FABRIKA_APPLICATION_OBJECTS} name no valid set of objects`))
589
+ }
590
+ if (!this.#named(object)) {
591
+ return Promise.reject(new Error('this object is another object than the one the event names'))
592
+ }
593
+ if (this.#barrier !== null && this.#barrierObject !== object) {
594
+ return Promise.reject(new Error('one object instance was asked under two object names'))
595
+ }
596
+ if (this.#barrier === null) {
597
+ const reading = (async () => await MaintenanceBarrier.load(r2MaintenanceStore(this.#env[CLOUDFLARE_FILES_BINDING], object)))()
598
+ this.#barrier = reading
599
+ this.#barrierObject = object
600
+ reading.catch(() => {
601
+ if (this.#barrier === reading) {
602
+ this.#barrier = null
603
+ this.#barrierObject = null
604
+ }
605
+ })
606
+ }
607
+ return this.#barrier
608
+ }
609
+
610
+ /** The migration phase's boot: applies the pending migrations whatever the mode; `false` when the boot failed. */
611
+ async #migrated(): Promise<boolean> {
612
+ const current = this.#outcome === null ? null : await this.#outcome
613
+ if (current !== null && 'booted' in current) {
614
+ return true
615
+ }
616
+ const outcome = this.#boot('apply')
617
+ this.#outcome = outcome
618
+ return 'booted' in this.#settled(outcome, await outcome)
285
619
  }
286
620
 
287
621
  /** Settles either way: a callback that rejects would reset the object, and every request answers the failure instead. */
288
- #boot(): Promise<Outcome> {
622
+ #boot(policy: MigrationPolicy): Promise<Outcome> {
289
623
  return this.#state.blockConcurrencyWhile(() =>
290
- attach(this.#setup, this.#state, this.#env, (failure) => {
624
+ attach(this.#setup, this.#state, this.#env, policy, (failure) => {
291
625
  this.#bootFailure = failure
292
- }).then((booted): Outcome => ({ booted }), (failure: unknown): Outcome => {
626
+ }).then((booted): Outcome => {
627
+ // The view reports the current boot: a failure an earlier attempt left is no longer the object's state.
628
+ this.#bootFailure = null
629
+ return { booted }
630
+ }, (failure: unknown): Outcome => {
631
+ if (isMigrationsPending(failure) || failure instanceof ImportInProgress) {
632
+ return { pending: true }
633
+ }
293
634
  this.#failedBoots++
294
635
  this.#retryAt = Date.now() + Math.min(BOOT_RETRY_MS * 2 ** (this.#failedBoots - 1), BOOT_RETRY_CAP_MS)
295
636
  return { failure }
@@ -299,54 +640,183 @@ export class FabrikaApplication {
299
640
 
300
641
  /**
301
642
  * The instance's boot, started by the first event that needs it and booted again once a failed boot's backoff has
302
- * passed; concurrent events share one boot and one retry.
643
+ * passed; concurrent events share one boot and one retry. A boot that found a migration pending is not kept, so the
644
+ * next event looks again.
303
645
  */
304
- async #current(): Promise<Outcome> {
646
+ async #current(policy: MigrationPolicy): Promise<Outcome> {
305
647
  if (this.#outcome === null) {
306
- this.#outcome = this.#boot()
648
+ this.#outcome = this.#boot(policy)
307
649
  }
308
650
  const pending = this.#outcome
309
651
  const outcome = await pending
310
- if (!('failure' in outcome) || Date.now() < this.#retryAt) {
311
- return outcome
652
+ if ('failure' in outcome && Date.now() >= this.#retryAt) {
653
+ if (this.#outcome === pending) {
654
+ this.#outcome = this.#boot(policy)
655
+ }
656
+ return this.#settled(this.#outcome, await this.#outcome)
312
657
  }
313
- if (this.#outcome === pending) {
314
- this.#outcome = this.#boot()
658
+ return this.#settled(pending, outcome)
659
+ }
660
+
661
+ /** Forgets a boot that found a migration pending, so the next event boots again. */
662
+ #settled(boot: Promise<Outcome>, outcome: Outcome): Outcome {
663
+ if ('pending' in outcome && this.#outcome === boot) {
664
+ this.#outcome = null
315
665
  }
316
- return await this.#outcome
666
+ return outcome
317
667
  }
318
668
 
319
669
  #serve(booted: Booted, request: Request): Promise<Response> {
670
+ const scheduler = booted.app.jobs.scheduler
320
671
  const tick = request.headers.get(TICK_HEADER)
321
672
  if (tick !== null) {
322
- return Promise.resolve(this.#tick(booted, tick))
673
+ return TICK_TIME.test(tick) && Number.isSafeInteger(Number(tick)) ? this.#tick(scheduler) : Promise.resolve(new Response(null, { status: 400 }))
674
+ }
675
+ if (scheduler !== null && !this.#repaired) {
676
+ this.#state.waitUntil(this.#repairOnce(scheduler))
323
677
  }
324
678
  const exec: RequestExecutionContext = { waitUntil: (promise) => this.#state.waitUntil(promise) }
325
679
  return booted.app.fetch(request, exec)
326
680
  }
327
681
 
328
- /** The fixed-size digests are compared in constant time, so neither the length nor a prefix of the secret leaks. */
329
- #fromWorker(request: Request): boolean {
330
- const presented = request.headers.get(PROXY_SECRET_HEADER) ?? ''
331
- return this.#secret !== null && presented !== '' && timingSafeEqual(proxySecretDigest(presented), this.#secret)
682
+ /**
683
+ * workerd calls it when the alarm is due (ADR-0065 §5). Only the object ordinary traffic reaches runs jobs, and only
684
+ * in `open`; otherwise it runs nothing and arms nothing: the maintenance operation owns the object, reopening arms the
685
+ * alarm again, and the repair tick arms the serving object's. It never throws, because after six failures the platform
686
+ * stops retrying: an internal error in `open` arms it again in `ALARM_RETRY_MS` and is reported.
687
+ */
688
+ async alarm(): Promise<void> {
689
+ let scheduler: JobScheduler | null = null
690
+ let barrier: MaintenanceBarrier | null = null
691
+ try {
692
+ const object = this.#recorded()
693
+ if (object === null || object !== this.#object) {
694
+ // No verified event named this object yet, or it is not the one ordinary traffic reaches (a rehearsal, a
695
+ // former serving object after a deploy moved the traffic): it runs no job.
696
+ return
697
+ }
698
+ if (importing(this.#state.storage)) {
699
+ // A half-loaded import runs no job and arms nothing; the repair tick arms the alarm once the object boots.
700
+ return
701
+ }
702
+ barrier = await this.#maintenance(object)
703
+ if (barrier.mode !== 'open') {
704
+ return
705
+ }
706
+ const outcome = await this.#current('apply')
707
+ if (!('booted' in outcome)) {
708
+ throw new Error('the application did not boot')
709
+ }
710
+ const booted = outcome.booted.app.jobs.scheduler
711
+ scheduler = booted
712
+ if (booted === null) {
713
+ return
714
+ }
715
+ const event = barrier.event()
716
+ await objectScope.run({ storage: this.#state.storage, event }, () => runGated(event, () => this.#wake(booted, event)))
717
+ } catch (error) {
718
+ if (error instanceof MaintenanceFenceError || barrier === null || barrier.mode !== 'open') {
719
+ // A transition restricted the object meanwhile, or none was ever read: no failure, and nothing is armed outside open.
720
+ return
721
+ }
722
+ console.error('fabrika: the alarm failed: alarm_failed')
723
+ try {
724
+ await this.#state.storage.setAlarm?.(Date.now() + ALARM_RETRY_MS)
725
+ } catch {
726
+ console.error('fabrika: the alarm could not be armed again; the repair tick arms it')
727
+ }
728
+ if (scheduler !== null) {
729
+ this.#state.waitUntil(scheduler.failed('alarm', error))
730
+ }
731
+ }
332
732
  }
333
733
 
334
- /** Runs every job due in the tick's UTC minute and the jobs' check, each past the answer. */
335
- #tick(booted: Booted, scheduledTime: string): Response {
336
- const time = Number(scheduledTime)
337
- if (!Number.isSafeInteger(time)) {
338
- return new Response(null, { status: 400 })
734
+ /**
735
+ * One alarm: per job, in one transaction, a crashed holder reaped, the lease taken for a due wake (or the pending flag
736
+ * set for the run holding it) and the alarm armed from the projection. The runs go to `waitUntil`: one alarm runs at a
737
+ * time, and awaiting a long job would hold back every other wake. No wake is consumed here, only by its run's finish.
738
+ */
739
+ async #wake(scheduler: JobScheduler, event: BarrierEvent): Promise<void> {
740
+ await this.#repairOnce(scheduler)
741
+ const early = await scheduler.earlyBy(ALARM_SLACK_MS)
742
+ if (early > 0) {
743
+ await this.#clock.wait(early, new AbortController().signal)
339
744
  }
340
- const at = new Date(time)
341
- const jobs = booted.app.jobs.scheduled
342
- for (const job of jobs) {
343
- if (isDueAt(job.schedule, at)) {
344
- this.#state.waitUntil(runQuietly(booted.app, job.name))
745
+ const storage = fencedStorage()
746
+ // A job whose lease is held past its expiry but which the contract no longer declares is reaped too.
747
+ for (const job of [...scheduler.jobs, ...(await scheduler.orphans())]) {
748
+ if (!event.writable()) {
749
+ // A transition restricted the object since the alarm was admitted; reopening arms it again.
750
+ return
751
+ }
752
+ const taken = await storage.transaction(async () => {
753
+ const taken = await scheduler.take(job)
754
+ await armAlarm(eventScope().storage, await scheduler.next())
755
+ return taken
756
+ })
757
+ if (taken.reaped) {
758
+ this.#state.waitUntil(scheduler.crashed(job))
759
+ }
760
+ if (taken.fence !== null) {
761
+ this.#state.waitUntil(runScheduled(scheduler, job, taken.fence, this.#clock))
345
762
  }
346
763
  }
347
- if (jobs.length > 0) {
348
- this.#state.waitUntil(booted.app.jobs.check())
764
+ }
765
+
766
+ /** The startup repair once per instance, at its first open event; a failure is reported and the next event repairs. */
767
+ async #repairOnce(scheduler: JobScheduler): Promise<void> {
768
+ if (this.#repaired) {
769
+ return
770
+ }
771
+ try {
772
+ this.#repaired = await scheduler.repair()
773
+ } catch (error) {
774
+ this.#repairFailed(scheduler, error)
775
+ }
776
+ }
777
+
778
+ /** A repair the write fence refused is no failure: a transition closed the object, and reopening repairs. */
779
+ #repairFailed(scheduler: JobScheduler, error: unknown): void {
780
+ if (error instanceof MaintenanceFenceError) {
781
+ return
782
+ }
783
+ console.error("fabrika: the job scheduler's repair failed: repair_failed")
784
+ this.#state.waitUntil(scheduler.failed('repair', error))
785
+ }
786
+
787
+ /** The repair tick: the startup repair again, which also arms the first alarm of an environment nobody has called. */
788
+ async #tick(scheduler: JobScheduler | null): Promise<Response> {
789
+ if (scheduler === null) {
790
+ return new Response(null, { status: 204 })
791
+ }
792
+ try {
793
+ this.#repaired = (await scheduler.repair()) || this.#repaired
794
+ } catch (error) {
795
+ this.#repairFailed(scheduler, error)
796
+ return new Response(null, { status: error instanceof MaintenanceFenceError ? 503 : 500 })
349
797
  }
350
798
  return new Response(null, { status: 204 })
351
799
  }
800
+
801
+ /** After a transition that opened this object, its alarm fires at once and repairs: an alarm outside `open` armed nothing. */
802
+ async #reopened(): Promise<void> {
803
+ if (this.#barrier === null || this.#barrierObject !== this.#object) {
804
+ return
805
+ }
806
+ try {
807
+ if ((await this.#barrier).mode !== 'open') {
808
+ return
809
+ }
810
+ this.#repaired = false
811
+ await this.#state.storage.setAlarm?.(Date.now())
812
+ } catch {
813
+ console.error('fabrika: the alarm could not be armed after the object opened; the repair tick arms it')
814
+ }
815
+ }
816
+
817
+ /** The fixed-size digests are compared in constant time, so neither the length nor a prefix of the secret leaks. */
818
+ #fromWorker(request: Request): boolean {
819
+ const presented = request.headers.get(PROXY_SECRET_HEADER) ?? ''
820
+ return this.#secret !== null && presented !== '' && timingSafeEqual(proxySecretDigest(presented), this.#secret)
821
+ }
352
822
  }