@fabrika/app 0.9.0-dev.4 → 0.9.0-dev.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +41 -14
  3. package/skill/references/app-setup.md +43 -12
  4. package/skill/references/dev.md +2 -1
  5. package/skill/references/endpoint.md +30 -24
  6. package/skill/references/errors.md +40 -35
  7. package/skill/references/files.md +4 -2
  8. package/skill/references/jobs.md +21 -4
  9. package/skill/references/table-and-migration.md +19 -2
  10. package/skill/references/tests.md +40 -1
  11. package/skill/references/webhooks-links-connectors.md +1 -1
  12. package/src/build-recipe.ts +198 -0
  13. package/src/build.ts +60 -4
  14. package/src/bun.ts +181 -13
  15. package/src/cloudflare/application.ts +822 -0
  16. package/src/cloudflare/bindings.ts +159 -0
  17. package/src/cloudflare/export.ts +621 -0
  18. package/src/cloudflare/import.ts +1013 -0
  19. package/src/cloudflare/maintenance.ts +518 -0
  20. package/src/cloudflare/names.ts +16 -0
  21. package/src/cloudflare/ops.ts +375 -0
  22. package/src/cloudflare/r2-files.ts +172 -0
  23. package/src/cloudflare/restore.ts +167 -0
  24. package/src/cloudflare/tracker.ts +192 -0
  25. package/src/cloudflare.ts +100 -146
  26. package/src/contract/app-id.ts +5 -0
  27. package/src/contract/artifact-schema.ts +9 -1
  28. package/src/contract/build.ts +40 -2
  29. package/src/contract/compile.ts +1 -0
  30. package/src/contract/context.ts +35 -9
  31. package/src/contract/declaration.ts +2 -0
  32. package/src/contract/defaults.ts +22 -2
  33. package/src/contract/define.ts +29 -3
  34. package/src/contract/kit.ts +4 -1
  35. package/src/contract/platform.ts +3 -2
  36. package/src/contract/vocabulary.ts +26 -8
  37. package/src/dev/launcher.ts +50 -4
  38. package/src/disk-assets.ts +8 -3
  39. package/src/files/archive.ts +2 -2
  40. package/src/files/job-names.ts +12 -0
  41. package/src/files/kinds.ts +73 -31
  42. package/src/files/scan.ts +3 -4
  43. package/src/files/streams.ts +3 -1
  44. package/src/implement/anonymous.ts +12 -11
  45. package/src/implement/boot.ts +75 -6
  46. package/src/implement/bounded.ts +21 -2
  47. package/src/implement/capability.ts +15 -3
  48. package/src/implement/dispatch.ts +419 -177
  49. package/src/implement/failures.ts +17 -1
  50. package/src/implement/faults.ts +1 -0
  51. package/src/implement/index.ts +1 -0
  52. package/src/implement/invocation.ts +58 -10
  53. package/src/implement/principal.ts +25 -2
  54. package/src/implement/terminals.ts +93 -0
  55. package/src/implement/workload.ts +42 -16
  56. package/src/jobs/mail.ts +8 -0
  57. package/src/jobs/report.ts +5 -1
  58. package/src/jobs/runner.ts +672 -113
  59. package/src/jobs/schedule.ts +7 -1
  60. package/src/jobs/scrub.ts +14 -1
  61. package/src/jobs/types.ts +6 -0
  62. package/src/spa-fallback.ts +12 -0
  63. package/src/testing/endpoint-access.ts +307 -152
  64. package/src/testing/isolation.ts +8 -1
  65. package/src/testing/kit.ts +80 -2
  66. package/src/testing/proxy.ts +15 -10
  67. package/types/src/build-recipe.d.ts +70 -0
  68. package/types/src/build.d.ts +19 -1
  69. package/types/src/bun.d.ts +34 -1
  70. package/types/src/cloudflare/application.d.ts +73 -0
  71. package/types/src/cloudflare/bindings.d.ts +33 -0
  72. package/types/src/cloudflare/export.d.ts +16 -0
  73. package/types/src/cloudflare/import.d.ts +51 -0
  74. package/types/src/cloudflare/maintenance.d.ts +117 -0
  75. package/types/src/cloudflare/names.d.ts +11 -0
  76. package/types/src/cloudflare/ops.d.ts +81 -0
  77. package/types/src/cloudflare/r2-files.d.ts +9 -0
  78. package/types/src/cloudflare/restore.d.ts +9 -0
  79. package/types/src/cloudflare/tracker.d.ts +29 -0
  80. package/types/src/cloudflare.d.ts +12 -18
  81. package/types/src/contract/app-id.d.ts +2 -0
  82. package/types/src/contract/artifact-schema.d.ts +2 -0
  83. package/types/src/contract/context.d.ts +35 -11
  84. package/types/src/contract/declaration.d.ts +4 -0
  85. package/types/src/contract/defaults.d.ts +19 -1
  86. package/types/src/contract/define.d.ts +20 -3
  87. package/types/src/contract/kit.d.ts +2 -1
  88. package/types/src/contract/platform.d.ts +2 -2
  89. package/types/src/contract/vocabulary.d.ts +9 -7
  90. package/types/src/disk-assets.d.ts +4 -1
  91. package/types/src/files/job-names.d.ts +9 -0
  92. package/types/src/files/kinds.d.ts +21 -14
  93. package/types/src/files/scan.d.ts +1 -3
  94. package/types/src/implement/anonymous.d.ts +2 -4
  95. package/types/src/implement/boot.d.ts +17 -1
  96. package/types/src/implement/bounded.d.ts +11 -1
  97. package/types/src/implement/capability.d.ts +4 -2
  98. package/types/src/implement/dispatch.d.ts +34 -1
  99. package/types/src/implement/failures.d.ts +5 -0
  100. package/types/src/implement/faults.d.ts +1 -0
  101. package/types/src/implement/index.d.ts +1 -0
  102. package/types/src/implement/invocation.d.ts +22 -7
  103. package/types/src/implement/principal.d.ts +6 -1
  104. package/types/src/implement/terminals.d.ts +33 -0
  105. package/types/src/implement/workload.d.ts +5 -0
  106. package/types/src/jobs/runner.d.ts +66 -2
  107. package/types/src/jobs/schedule.d.ts +2 -0
  108. package/types/src/jobs/types.d.ts +6 -0
  109. package/types/src/spa-fallback.d.ts +2 -0
  110. package/types/src/testing/endpoint-access.d.ts +37 -5
  111. package/types/src/testing/isolation.d.ts +1 -1
  112. package/types/src/testing/kit.d.ts +22 -1
  113. package/types/src/testing/proxy.d.ts +4 -4
@@ -0,0 +1,822 @@
1
+ // The Durable Object an application environment runs in (ADR-0065): it migrates its SQLite, boots the application,
2
+ // serves every request and repair tick its Worker forwards, and runs the jobs from its alarm, which it arms from their
3
+ // durable wakes in the transaction of every change. A plain class, so Bun and Lopata load it without `cloudflare:workers`.
4
+
5
+ import {
6
+ FABRIKA_APPLICATION_OBJECT,
7
+ FABRIKA_APPLICATION_OBJECTS,
8
+ FABRIKA_CONTRACT_DIGEST,
9
+ FABRIKA_PROXY_SECRET,
10
+ OPS_PREFIX,
11
+ type OpsBootFailureContent,
12
+ type OpsMode,
13
+ parseApplicationObjects,
14
+ PROXY_SECRET_HEADER,
15
+ } from '@fabrika/auth-core'
16
+ import { isScopedDatabase, type LeaseFence, type Session } from '@fabrika/db/framework'
17
+ import { SQLITE_FRAMEWORK_MIGRATIONS } from '@fabrika/db/migrations-sqlite'
18
+ import { openSqlite, type SqliteCursor, type SqliteStorage } from '@fabrika/db/sqlite'
19
+ import { migrateSqlite, type SqliteMigrationBundle, type SqliteMigrationTarget } from '@fabrika/db/sqlite-migrate'
20
+ import { AsyncLocalStorage } from 'node:async_hooks'
21
+ import { timingSafeEqual } from 'node:crypto'
22
+ import type { RequestExecutionContext } from '../fetch.js'
23
+ import { FILE_TOMBSTONE_RETENTION_MS } from '../files/kinds.js'
24
+ import { boot, type ContractApp } from '../implement/boot.js'
25
+ import { type CrashEvents, environmentCrashReporter, reportSurvivedCrashes } from '../implement/crashes.js'
26
+ import { MaintenanceFenceError, maintenanceRefusal, proxySecretDigest, runGated } from '../implement/dispatch.js'
27
+ import { type CapturedEnvironment, captureEnvironment } from '../implement/environment.js'
28
+ import type { ContractImplementation } from '../implement/implement.js'
29
+ import type { CrashReporter } from '../jobs/report.js'
30
+ import { type JobClock, type JobScheduler, systemClock } from '../jobs/runner.js'
31
+ import { environmentScrubber, type Scrubber } from '../jobs/scrub.js'
32
+ import { bindingAssets, bindingKeySource, bindingMembership, bindingTerminals, bindingWorkloadTokens, connectBindings } from './bindings.js'
33
+ import { importInProgress } from './import.js'
34
+ import { type BarrierEvent, MaintenanceBarrier, r2MaintenanceStore } from './maintenance.js'
35
+ import { CLOUDFLARE_ASSETS_BINDING, CLOUDFLARE_FILES_BINDING } from './names.js'
36
+ import { type OpsAccess, opsAccess, type OpsRecovery, type OpsStorage, serveOps } from './ops.js'
37
+ import { r2FileObjectStore } from './r2-files.js'
38
+ import { trackPending } from './tracker.js'
39
+
40
+ /** The application module, imported only after the environment was captured: the generated entrypoint's `import()`. */
41
+ export type ImplementationLoader = () => Promise<{ readonly default: ContractImplementation }>
42
+
43
+ /** One of the application's SQLite migrations, as `fabrika app build` bundles them from `db/migrations`. */
44
+ export interface ApplicationMigration {
45
+ readonly name: string
46
+ readonly sql: string
47
+ }
48
+
49
+ export interface ContractWorkerOptions {
50
+ /** The application's own SQLite migrations, applied as the bundle `app` after the framework's `fabrika`. */
51
+ readonly migrations?: readonly ApplicationMigration[]
52
+ /** Where the isolate hears of an unhandled rejection; the global `unhandledrejection` event by default. */
53
+ readonly rejections?: CrashEvents
54
+ /** What posts a report to Operations' ingest; `fetch` by default. */
55
+ readonly operationsFetch?: (request: Request) => Promise<Response>
56
+ /** The jobs' and the alarm's waits and an attempt's deadline; the system clock by default. */
57
+ readonly clock?: JobClock
58
+ }
59
+
60
+ /**
61
+ * Marks the Worker's repair tick, with its `scheduledTime`. The Worker's `fetch` removes it from every request it
62
+ * forwards, so only its own `scheduled` handler reaches the tick.
63
+ */
64
+ export const TICK_HEADER = 'X-Fabrika-Tick'
65
+
66
+ /** A wake runs at most this early: `Date.now()` is pinned between I/O, so an alarm may start a moment before it (I6). */
67
+ export const ALARM_SLACK_MS = 50
68
+ /** After an internal error the alarm is armed again this far ahead; the repair tick covers a re-arm that failed. */
69
+ export const ALARM_RETRY_MS = 30_000
70
+ /**
71
+ * The object's KV key that holds the name of the object it is, recorded at its first verified event: an alarm names no
72
+ * object, and the one ordinary traffic reaches may since have changed. Outside the SQLite, so no dump carries it.
73
+ */
74
+ export const OBJECT_NAME_KEY = 'fabrika:object'
75
+ /** A repair tick's time: the Worker's `scheduledTime`, digits only. */
76
+ const TICK_TIME = /^\d{1,16}$/
77
+
78
+ /** The object's alarm. On a SQLite-backed object `setAlarm` and `deleteAlarm` commit with the open transaction. */
79
+ export interface AlarmStorage {
80
+ setAlarm(scheduledTime: number): Promise<void>
81
+ deleteAlarm(): Promise<void>
82
+ }
83
+
84
+ /**
85
+ * What the object needs of its `ctx.storage`; a Durable Object's `DurableObjectStorage` fits it. A test double without
86
+ * an alarm arms none.
87
+ */
88
+ export type ApplicationStorage = SqliteStorage & OpsStorage & Partial<AlarmStorage>
89
+
90
+ /** What the object needs of its `DurableObjectState`. */
91
+ export interface ApplicationState {
92
+ readonly storage: ApplicationStorage & Partial<Omit<OpsRecovery, 'abort'>>
93
+ blockConcurrencyWhile<T>(callback: () => Promise<T>): Promise<T>
94
+ waitUntil(promise: Promise<unknown>): void
95
+ /** workerd resets the object; the test double has none. */
96
+ abort?(reason?: string): void
97
+ }
98
+
99
+ /** `state` with every `waitUntil` promise recorded by the isolate's pending-work tracker (the ops operation `timers`). */
100
+ export function trackedState(state: ApplicationState): ApplicationState {
101
+ const { abort } = state
102
+ return {
103
+ storage: state.storage,
104
+ blockConcurrencyWhile: (callback) => state.blockConcurrencyWhile(callback),
105
+ waitUntil: (promise) => state.waitUntil(trackPending(promise)),
106
+ ...(abort === undefined ? {} : { abort: (reason?: string) => abort.call(state, reason) }),
107
+ }
108
+ }
109
+
110
+ /** The object's recovery when its storage and state have every member PITR needs (ADR-0070), else `null`. */
111
+ function objectRecovery(state: ApplicationState): OpsRecovery | null {
112
+ const { storage } = state
113
+ const { getCurrentBookmark, getBookmarkForTime, onNextSessionRestoreBookmark } = storage
114
+ const { abort } = state
115
+ if (getCurrentBookmark === undefined || getBookmarkForTime === undefined || onNextSessionRestoreBookmark === undefined || abort === undefined) {
116
+ return null
117
+ }
118
+ return {
119
+ getCurrentBookmark: () => getCurrentBookmark.call(storage),
120
+ getBookmarkForTime: (time) => getBookmarkForTime.call(storage, time),
121
+ onNextSessionRestoreBookmark: (bookmark) => onNextSessionRestoreBookmark.call(storage, bookmark),
122
+ abort: (reason) => abort.call(state, reason),
123
+ }
124
+ }
125
+
126
+ /** The framework's bindings: IAM's application entrypoint and the Workers Assets of the SPA. No handler sees them. */
127
+ const FRAMEWORK_BINDINGS: ReadonlySet<string> = new Set(['IAM', CLOUDFLARE_ASSETS_BINDING])
128
+
129
+ /**
130
+ * The application this isolate booted, once: its module is loaded once, and its `scoped(tables)` layer binds once, to
131
+ * the storage of the object whose event runs (`objectStorage`), never to one database.
132
+ */
133
+ interface Booted {
134
+ readonly app: ContractApp
135
+ /** Whether the application has a data layer, which every object instance migrates in its own storage. */
136
+ readonly migrates: boolean
137
+ }
138
+
139
+ interface ApplicationSetup {
140
+ readonly load: ImplementationLoader
141
+ readonly options: ContractWorkerOptions
142
+ booted: Promise<Booted> | null
143
+ /** Whether the current streak of failed attachments was reported; an attachment that succeeds ends the streak. */
144
+ failureReported: boolean
145
+ watchingRejections: boolean
146
+ }
147
+
148
+ let registered: ApplicationSetup | null = null
149
+
150
+ /** The storage of the object whose event runs, and the barrier's view of that event; work it hands to `waitUntil` keeps both. */
151
+ interface ObjectScope {
152
+ readonly storage: ApplicationStorage
153
+ readonly event: BarrierEvent
154
+ }
155
+
156
+ const objectScope = new AsyncLocalStorage<ObjectScope>()
157
+
158
+ /** The cursors of the fenced transaction that runs: its commit checks whether any of them wrote. */
159
+ const openTransaction = new AsyncLocalStorage<SqliteCursor[]>()
160
+
161
+ function eventScope(): ObjectScope {
162
+ const scope = objectScope.getStore()
163
+ if (scope === undefined) {
164
+ throw new Error('a statement ran outside any event of an application object: the framework runs every event in its object scope')
165
+ }
166
+ return scope
167
+ }
168
+
169
+ /** The write fence (ADR-0069 §3): a write of an invocation admitted before a restrictive transition rolls back. */
170
+ function fence(event: BarrierEvent, cursors: readonly SqliteCursor[]): void {
171
+ if (cursors.some((cursor) => cursor.rowsWritten > 0) && !event.writable()) {
172
+ throw new MaintenanceFenceError()
173
+ }
174
+ }
175
+
176
+ /**
177
+ * The storage every framework and application write on SQLite reaches, bound to the object whose event runs. The
178
+ * fence runs inside each transaction before it commits; a statement outside one runs in a transaction of its own.
179
+ */
180
+ function fencedStorage(): SqliteStorage {
181
+ return {
182
+ sql: {
183
+ exec(query, ...bindings) {
184
+ const { storage, event } = eventScope()
185
+ const cursors = openTransaction.getStore()
186
+ if (cursors !== undefined) {
187
+ const cursor = storage.sql.exec(query, ...bindings)
188
+ cursors.push(cursor)
189
+ return cursor
190
+ }
191
+ return storage.transactionSync(() => {
192
+ const cursor = storage.sql.exec(query, ...bindings)
193
+ const rows = cursor.toArray()
194
+ fence(event, [cursor])
195
+ return { toArray: () => rows, rowsWritten: cursor.rowsWritten }
196
+ })
197
+ },
198
+ },
199
+ transaction(closure) {
200
+ const { storage, event } = eventScope()
201
+ // A nested transaction shares its parent's cursors, so the outer commit checks every write inside it.
202
+ const cursors = openTransaction.getStore() ?? []
203
+ return storage.transaction(() =>
204
+ openTransaction.run(cursors, async () => {
205
+ const result = await closure()
206
+ fence(event, cursors)
207
+ return result
208
+ })
209
+ )
210
+ },
211
+ transactionSync(closure) {
212
+ const { storage, event } = eventScope()
213
+ const cursors = openTransaction.getStore() ?? []
214
+ return storage.transactionSync(() =>
215
+ openTransaction.run(cursors, () => {
216
+ const result = closure()
217
+ fence(event, cursors)
218
+ return result
219
+ })
220
+ )
221
+ },
222
+ }
223
+ }
224
+
225
+ /** Whether migrations may be applied: in `open` and the migration phase; a restricted cold start only looks. */
226
+ type MigrationPolicy = 'apply' | 'refuse'
227
+
228
+ /** A restricted cold start found a migration to apply: the application does not boot until the operation's migrate. */
229
+ class MigrationsPending extends Error {
230
+ override readonly name = 'MigrationsPending'
231
+ }
232
+
233
+ /** The object holds an import marker (ADR-0070 §6): it neither boots nor migrates until the import completes or is discarded. */
234
+ class ImportInProgress extends Error {
235
+ override readonly name = 'ImportInProgress'
236
+ }
237
+
238
+ /** Whether the object holds an import marker; a storage without the synchronous KV (the bun:sqlite double) holds none. */
239
+ function importing(storage: ApplicationStorage): boolean {
240
+ const { kv } = storage
241
+ return kv !== undefined && importInProgress({ kv })
242
+ }
243
+
244
+ function isMigrationsPending(error: unknown): boolean {
245
+ return error instanceof MigrationsPending || (error instanceof Error && error.cause instanceof MigrationsPending)
246
+ }
247
+
248
+ /**
249
+ * The migration runner's target that applies nothing: it answers the runner's reads, and the first write it would make
250
+ * (the ledger's DDL, a migration, a digest) throws `MigrationsPending` instead.
251
+ */
252
+ function refusingTarget(storage: ApplicationStorage): SqliteMigrationTarget {
253
+ return {
254
+ exec(query, ...bindings) {
255
+ if (!/^\s*select\b/i.test(query)) {
256
+ throw new MigrationsPending('a migration is pending')
257
+ }
258
+ return storage.sql.exec(query, ...bindings)
259
+ },
260
+ transactionSync() {
261
+ throw new MigrationsPending('a migration is pending')
262
+ },
263
+ }
264
+ }
265
+
266
+ /** What the object classes of this isolate boot: the generated entrypoint's `createContractWorker` registers it. */
267
+ export function registerApplication(load: ImplementationLoader, options: ContractWorkerOptions): void {
268
+ registered = { load, options, booted: null, failureReported: false, watchingRejections: false }
269
+ }
270
+
271
+ /** The isolate's `unhandledrejection` event. Listening leaves the runtime's own handling, its log line, in place. */
272
+ function globalRejectionEvents(): CrashEvents {
273
+ return {
274
+ listen(listener) {
275
+ const rejection = (event: Event) => listener('unhandled_rejection', Reflect.get(event, 'reason'))
276
+ globalThis.addEventListener('unhandledrejection', rejection)
277
+ return () => globalThis.removeEventListener('unhandledrejection', rejection)
278
+ },
279
+ }
280
+ }
281
+
282
+ /**
283
+ * The framework's tables, then the application's, each migration once: a booted storage applies nothing. Under
284
+ * `refuse` a pending migration throws `MigrationsPending` before anything is written.
285
+ */
286
+ async function migrate(storage: ApplicationStorage, migrations: readonly ApplicationMigration[], policy: MigrationPolicy): Promise<void> {
287
+ const target: SqliteMigrationTarget = policy === 'apply'
288
+ ? { exec: (query, ...bindings) => storage.sql.exec(query, ...bindings), transactionSync: (fn) => storage.transactionSync(fn) }
289
+ : refusingTarget(storage)
290
+ await migrateSqlite(target, migrationBundles(migrations))
291
+ }
292
+
293
+ /** The bundles the object applies, in order: the framework's tables, then the application's. */
294
+ function migrationBundles(migrations: readonly ApplicationMigration[]): readonly SqliteMigrationBundle[] {
295
+ return [
296
+ { bundle: 'fabrika', migrations: SQLITE_FRAMEWORK_MIGRATIONS },
297
+ { bundle: 'app', migrations },
298
+ ]
299
+ }
300
+
301
+ async function bootApplication(
302
+ setup: ApplicationSetup,
303
+ state: ApplicationState,
304
+ env: Readonly<Record<string, unknown>>,
305
+ environment: CapturedEnvironment,
306
+ crashes: CrashReporter | null,
307
+ policy: MigrationPolicy,
308
+ ): Promise<Booted> {
309
+ const { default: implementation } = await setup.load()
310
+ const migrates = isScopedDatabase(implementation.database)
311
+ if (migrates) {
312
+ await migrate(state.storage, setup.options.migrations ?? [], policy)
313
+ }
314
+ const database = migrates ? openSqlite(fencedStorage()) : undefined
315
+ const app = boot(implementation, {
316
+ environment,
317
+ iam: bindingKeySource(env['IAM']),
318
+ assets: bindingAssets(env[CLOUDFLARE_ASSETS_BINDING]),
319
+ membership: bindingMembership(env['IAM']),
320
+ terminals: bindingTerminals(env['IAM']),
321
+ workloadTokens: bindingWorkloadTokens(env['IAM']),
322
+ connectBindings: connectBindings(env),
323
+ database,
324
+ files: r2FileObjectStore(env[CLOUDFLARE_FILES_BINDING]),
325
+ // A removed file's object outlives its row for the window, so a side restore finds its bytes (ADR-0070).
326
+ fileRetentionMs: FILE_TOMBSTONE_RETENTION_MS,
327
+ // The Worker forwards only with the proxy secret, which the object checked; `boot` checks it again.
328
+ requireProxySecret: true,
329
+ // The proxy Worker sets `X-Fabrika-Client-Ip` from `CF-Connecting-IP`, the visitor Cloudflare saw.
330
+ clientAddress: 'vouched',
331
+ projectWakes: (transaction) => projectWakes(setup, transaction),
332
+ ...(setup.options.operationsFetch === undefined ? {} : { operationsFetch: setup.options.operationsFetch }),
333
+ })
334
+ const booted: Booted = { app, migrates }
335
+ if (crashes !== null && !setup.watchingRejections) {
336
+ setup.watchingRejections = true
337
+ reportSurvivedCrashes(setup.options.rejections ?? globalRejectionEvents(), crashes)
338
+ }
339
+ return booted
340
+ }
341
+
342
+ /**
343
+ * The isolate's booted application with this instance's storage migrated, or the first boot. Reports the first failure
344
+ * of a streak as `boot_failed`, whichever instance or step it comes from; an attachment that succeeds ends the streak.
345
+ */
346
+ async function attach(
347
+ setup: ApplicationSetup | null,
348
+ state: ApplicationState,
349
+ env: Readonly<Record<string, unknown>>,
350
+ policy: MigrationPolicy,
351
+ failed: (failure: OpsBootFailureContent) => void,
352
+ ): Promise<Booted> {
353
+ if (setup === null) {
354
+ throw new Error('no contract application is registered: the generated entrypoint calls createContractWorker before an object starts')
355
+ }
356
+ // Before anything loads or migrates, and no boot failure: a half-loaded import never boots.
357
+ if (importing(state.storage)) {
358
+ throw new ImportInProgress('import_in_progress')
359
+ }
360
+ const environment = captureEnvironment(Object.fromEntries(Object.entries(env).filter(([name]) => !FRAMEWORK_BINDINGS.has(name))))
361
+ const crashes = environmentCrashReporter(environment, setup.options.operationsFetch)
362
+ try {
363
+ const booted = await attachOnce(setup, state, env, environment, crashes, policy)
364
+ setup.failureReported = false
365
+ return booted
366
+ } catch (error) {
367
+ if (isMigrationsPending(error)) {
368
+ throw error
369
+ }
370
+ failed(bootFailure(error, environmentScrubber(environment)))
371
+ if (crashes !== null && !setup.failureReported) {
372
+ setup.failureReported = true
373
+ state.waitUntil(crashes.report('boot_failed', error))
374
+ }
375
+ throw error
376
+ }
377
+ }
378
+
379
+ async function attachOnce(
380
+ setup: ApplicationSetup,
381
+ state: ApplicationState,
382
+ env: Readonly<Record<string, unknown>>,
383
+ environment: CapturedEnvironment,
384
+ crashes: CrashReporter | null,
385
+ policy: MigrationPolicy,
386
+ ): Promise<Booted> {
387
+ if (setup.booted === null) {
388
+ const booting = bootApplication(setup, state, env, environment, crashes, policy)
389
+ setup.booted = booting
390
+ // A failed boot is not kept: the next object instance boots again and fails with the same problems.
391
+ booting.catch(() => {
392
+ setup.booted = null
393
+ })
394
+ return await booting
395
+ }
396
+ const booted = await setup.booted
397
+ if (booted.migrates) {
398
+ await migrate(state.storage, setup.options.migrations ?? [], policy)
399
+ }
400
+ return booted
401
+ }
402
+
403
+ /** A failed boot as the ops view shows it, its text scrubbed as a crash report's is. */
404
+ function bootFailure(error: unknown, scrubber: Scrubber): OpsBootFailureContent {
405
+ const at = new Date().toISOString()
406
+ if (error instanceof Error) {
407
+ return { at, message: scrubber.scrub(error.message), stack: typeof error.stack === 'string' ? scrubber.scrub(error.stack) : null }
408
+ }
409
+ const message = typeof error === 'string' ? scrubber.scrub(error) : `a thrown ${error === null ? 'null' : typeof error} that is not an Error`
410
+ return { at, message, stack: null }
411
+ }
412
+
413
+ /**
414
+ * `pending`: a restricted cold start found a migration to apply, or the object holds an import marker, so the
415
+ * application did not boot; that is no failure.
416
+ */
417
+ type Outcome = { readonly booted: Booted } | { readonly failure: unknown } | { readonly pending: true }
418
+
419
+ /** The wait after an instance's first failed boot before an event boots again; it doubles with each failure, to the cap. */
420
+ const BOOT_RETRY_MS = 1_000
421
+ const BOOT_RETRY_CAP_MS = 60_000
422
+
423
+ /** The alarm armed at `next`, or deleted when nothing is pending; inside a transaction it commits with it. */
424
+ async function armAlarm(storage: ApplicationStorage, next: Date | null): Promise<void> {
425
+ if (next === null) {
426
+ await storage.deleteAlarm?.()
427
+ } else {
428
+ await storage.setAlarm?.(next.getTime())
429
+ }
430
+ }
431
+
432
+ /**
433
+ * The runtime's projection of the jobs' wakes and leases (ADR-0065 §5, I1): the store runs it in the transaction of
434
+ * every change, and it arms the alarm of the object whose event runs from `next`. A throw rolls the change back.
435
+ */
436
+ async function projectWakes(setup: ApplicationSetup, transaction: Session): Promise<void> {
437
+ // A wake or lease changes only once the application booted.
438
+ const booted = setup.booted === null ? null : await setup.booted
439
+ const scheduler = booted?.app.jobs.scheduler ?? null
440
+ if (scheduler !== null) {
441
+ await armAlarm(eventScope().storage, await scheduler.next(transaction))
442
+ }
443
+ }
444
+
445
+ /** A run the alarm took the lease for, handed to `waitUntil`; the runner logs and reports its own failures. */
446
+ async function runScheduled(scheduler: JobScheduler, job: string, fence: LeaseFence, clock: JobClock): Promise<void> {
447
+ try {
448
+ await scheduler.run(job, fence, { clock })
449
+ } catch {
450
+ console.error(`fabrika: job '${job}' did not start: driver_failed`)
451
+ }
452
+ }
453
+
454
+ /** The application environment's one object. The Worker's generated entrypoint exports it as `FabrikaApplication`. */
455
+ export class FabrikaApplication {
456
+ readonly #state: ApplicationState
457
+ readonly #env: Readonly<Record<string, unknown>>
458
+ readonly #setup: ApplicationSetup | null
459
+ /** `proxySecretDigest` of the environment's proxy secret; `null` refuses every request. */
460
+ readonly #secret: Uint8Array | null
461
+ /** What the platform wrote for Control's ops channel; `null` refuses every ops request. */
462
+ readonly #ops: OpsAccess | null
463
+ /** The object ordinary traffic reaches, so the one this instance is when such traffic arrives; `null` when malformed. */
464
+ readonly #object: string | null
465
+ /** The object this instance is, once a verified event named it or its storage recorded it (`OBJECT_NAME_KEY`). */
466
+ #name: string | null = null
467
+ /** The maintenance barrier, read from R2 once per instance; a failed read is not kept, so the next request reads again. */
468
+ #barrier: Promise<MaintenanceBarrier> | null = null
469
+ #barrierObject: string | null = null
470
+ /** The instance's failed boot, kept in memory for the ops view until a boot succeeds. */
471
+ #bootFailure: OpsBootFailureContent | null = null
472
+ /** The instance's boot, `null` until the first event that needs the application: an ops request never starts it. */
473
+ #outcome: Promise<Outcome> | null = null
474
+ #failedBoots = 0
475
+ /** When an event may boot again after a failed boot; until then every request answers the failure. */
476
+ #retryAt = 0
477
+ /** Whether this instance ran the scheduler's startup repair, which its first open event does. */
478
+ #repaired = false
479
+ readonly #clock: JobClock
480
+
481
+ constructor(state: ApplicationState, env: Readonly<Record<string, unknown>>) {
482
+ this.#state = trackedState(state)
483
+ this.#env = env
484
+ this.#setup = registered
485
+ this.#clock = registered?.options.clock ?? systemClock
486
+ const secret = env[FABRIKA_PROXY_SECRET]
487
+ this.#secret = typeof secret === 'string' && secret !== '' ? proxySecretDigest(secret) : null
488
+ this.#ops = opsAccess(env)
489
+ const object = env[FABRIKA_APPLICATION_OBJECT]
490
+ const objects = env[FABRIKA_APPLICATION_OBJECTS]
491
+ this.#object = parseApplicationObjects(typeof object === 'string' ? object : undefined, typeof objects === 'string' ? objects : undefined)
492
+ ?.object ?? null
493
+ }
494
+
495
+ #contractDigest(): string | null {
496
+ const digest = this.#env[FABRIKA_CONTRACT_DIGEST]
497
+ return typeof digest === 'string' ? digest : null
498
+ }
499
+
500
+ async fetch(request: Request): Promise<Response> {
501
+ // Another Worker on the account that binds this class reaches nothing, not even an answer that the object exists.
502
+ if (!this.#fromWorker(request)) {
503
+ return new Response(null, { status: 404 })
504
+ }
505
+ if (new URL(request.url).pathname.startsWith(OPS_PREFIX)) {
506
+ // The mode before the request: a transition that opens the object arms its alarm, which repairs.
507
+ let before: OpsMode | null = null
508
+ // Served from storage alone: it never starts a boot or a retry, and waits for one under way only once authenticated.
509
+ const response = await serveOps(request, this.#ops, {
510
+ storage: this.#state.storage,
511
+ bucket: this.#env[CLOUDFLARE_FILES_BINDING],
512
+ recovery: objectRecovery(this.#state),
513
+ contractDigest: this.#contractDigest(),
514
+ migrations: migrationBundles(this.#setup?.options.migrations ?? []),
515
+ bootFailure: async () => {
516
+ await this.#outcome
517
+ return this.#bootFailure
518
+ },
519
+ maintenance: async (object) => {
520
+ const barrier = await this.#maintenance(object)
521
+ before ??= barrier.mode
522
+ return barrier
523
+ },
524
+ migrate: () => this.#migrated(),
525
+ })
526
+ if (before !== null && before !== 'open') {
527
+ await this.#reopened()
528
+ }
529
+ return response
530
+ }
531
+ let barrier: MaintenanceBarrier
532
+ try {
533
+ barrier = await this.#maintenance(this.#object)
534
+ } catch {
535
+ // Fails closed: a state R2 could not give is never read as `open` (ADR-0069 §3, cold starts).
536
+ return maintenanceRefusal()
537
+ }
538
+ if (importing(this.#state.storage)) {
539
+ // ADR-0070 §6: whatever the mode, a half-loaded import serves nothing; the tick repairs nothing.
540
+ return request.headers.get(TICK_HEADER) === null ? maintenanceRefusal() : new Response(null, { status: 204 })
541
+ }
542
+ if (barrier.mode !== 'open' && request.headers.get(TICK_HEADER) !== null) {
543
+ // The tick repairs nothing and writes nothing until the object opens.
544
+ return new Response(null, { status: 204 })
545
+ }
546
+ if (barrier.mode === 'closed' || barrier.mode === 'inspect') {
547
+ return maintenanceRefusal()
548
+ }
549
+ const outcome = await this.#current(barrier.mode === 'open' ? 'apply' : 'refuse')
550
+ if ('pending' in outcome) {
551
+ return maintenanceRefusal()
552
+ }
553
+ if ('failure' in outcome) {
554
+ throw outcome.failure
555
+ }
556
+ const event = barrier.event()
557
+ return await objectScope.run({ storage: this.#state.storage, event }, () => runGated(event, () => this.#serve(outcome.booted, request)))
558
+ }
559
+
560
+ /** The name of the object this instance is: the one its storage recorded, or `null` before a verified event named it. */
561
+ #recorded(): string | null {
562
+ if (this.#name === null) {
563
+ const stored = this.#state.storage.kv?.get(OBJECT_NAME_KEY)
564
+ if (typeof stored === 'string' && stored !== '') {
565
+ this.#name = stored
566
+ }
567
+ }
568
+ return this.#name
569
+ }
570
+
571
+ /**
572
+ * Whether this instance is the object a verified event names; the first such event records the name, which never
573
+ * changes after. A storage without KV (the bun:sqlite double) keeps it in memory only.
574
+ */
575
+ #named(object: string): boolean {
576
+ const recorded = this.#recorded()
577
+ if (recorded !== null) {
578
+ return recorded === object
579
+ }
580
+ this.#state.storage.kv?.put(OBJECT_NAME_KEY, object)
581
+ this.#name = object
582
+ return true
583
+ }
584
+
585
+ /** The barrier of the object this instance is; every name it is asked under must be the same one. */
586
+ #maintenance(object: string | null): Promise<MaintenanceBarrier> {
587
+ if (object === null) {
588
+ return Promise.reject(new Error(`${FABRIKA_APPLICATION_OBJECT} and ${FABRIKA_APPLICATION_OBJECTS} name no valid set of objects`))
589
+ }
590
+ if (!this.#named(object)) {
591
+ return Promise.reject(new Error('this object is another object than the one the event names'))
592
+ }
593
+ if (this.#barrier !== null && this.#barrierObject !== object) {
594
+ return Promise.reject(new Error('one object instance was asked under two object names'))
595
+ }
596
+ if (this.#barrier === null) {
597
+ const reading = (async () => await MaintenanceBarrier.load(r2MaintenanceStore(this.#env[CLOUDFLARE_FILES_BINDING], object)))()
598
+ this.#barrier = reading
599
+ this.#barrierObject = object
600
+ reading.catch(() => {
601
+ if (this.#barrier === reading) {
602
+ this.#barrier = null
603
+ this.#barrierObject = null
604
+ }
605
+ })
606
+ }
607
+ return this.#barrier
608
+ }
609
+
610
+ /** The migration phase's boot: applies the pending migrations whatever the mode; `false` when the boot failed. */
611
+ async #migrated(): Promise<boolean> {
612
+ const current = this.#outcome === null ? null : await this.#outcome
613
+ if (current !== null && 'booted' in current) {
614
+ return true
615
+ }
616
+ const outcome = this.#boot('apply')
617
+ this.#outcome = outcome
618
+ return 'booted' in this.#settled(outcome, await outcome)
619
+ }
620
+
621
+ /** Settles either way: a callback that rejects would reset the object, and every request answers the failure instead. */
622
+ #boot(policy: MigrationPolicy): Promise<Outcome> {
623
+ return this.#state.blockConcurrencyWhile(() =>
624
+ attach(this.#setup, this.#state, this.#env, policy, (failure) => {
625
+ this.#bootFailure = failure
626
+ }).then((booted): Outcome => {
627
+ // The view reports the current boot: a failure an earlier attempt left is no longer the object's state.
628
+ this.#bootFailure = null
629
+ return { booted }
630
+ }, (failure: unknown): Outcome => {
631
+ if (isMigrationsPending(failure) || failure instanceof ImportInProgress) {
632
+ return { pending: true }
633
+ }
634
+ this.#failedBoots++
635
+ this.#retryAt = Date.now() + Math.min(BOOT_RETRY_MS * 2 ** (this.#failedBoots - 1), BOOT_RETRY_CAP_MS)
636
+ return { failure }
637
+ })
638
+ )
639
+ }
640
+
641
+ /**
642
+ * The instance's boot, started by the first event that needs it and booted again once a failed boot's backoff has
643
+ * passed; concurrent events share one boot and one retry. A boot that found a migration pending is not kept, so the
644
+ * next event looks again.
645
+ */
646
+ async #current(policy: MigrationPolicy): Promise<Outcome> {
647
+ if (this.#outcome === null) {
648
+ this.#outcome = this.#boot(policy)
649
+ }
650
+ const pending = this.#outcome
651
+ const outcome = await pending
652
+ if ('failure' in outcome && Date.now() >= this.#retryAt) {
653
+ if (this.#outcome === pending) {
654
+ this.#outcome = this.#boot(policy)
655
+ }
656
+ return this.#settled(this.#outcome, await this.#outcome)
657
+ }
658
+ return this.#settled(pending, outcome)
659
+ }
660
+
661
+ /** Forgets a boot that found a migration pending, so the next event boots again. */
662
+ #settled(boot: Promise<Outcome>, outcome: Outcome): Outcome {
663
+ if ('pending' in outcome && this.#outcome === boot) {
664
+ this.#outcome = null
665
+ }
666
+ return outcome
667
+ }
668
+
669
+ #serve(booted: Booted, request: Request): Promise<Response> {
670
+ const scheduler = booted.app.jobs.scheduler
671
+ const tick = request.headers.get(TICK_HEADER)
672
+ if (tick !== null) {
673
+ return TICK_TIME.test(tick) && Number.isSafeInteger(Number(tick)) ? this.#tick(scheduler) : Promise.resolve(new Response(null, { status: 400 }))
674
+ }
675
+ if (scheduler !== null && !this.#repaired) {
676
+ this.#state.waitUntil(this.#repairOnce(scheduler))
677
+ }
678
+ const exec: RequestExecutionContext = { waitUntil: (promise) => this.#state.waitUntil(promise) }
679
+ return booted.app.fetch(request, exec)
680
+ }
681
+
682
+ /**
683
+ * workerd calls it when the alarm is due (ADR-0065 §5). Only the object ordinary traffic reaches runs jobs, and only
684
+ * in `open`; otherwise it runs nothing and arms nothing: the maintenance operation owns the object, reopening arms the
685
+ * alarm again, and the repair tick arms the serving object's. It never throws, because after six failures the platform
686
+ * stops retrying: an internal error in `open` arms it again in `ALARM_RETRY_MS` and is reported.
687
+ */
688
+ async alarm(): Promise<void> {
689
+ let scheduler: JobScheduler | null = null
690
+ let barrier: MaintenanceBarrier | null = null
691
+ try {
692
+ const object = this.#recorded()
693
+ if (object === null || object !== this.#object) {
694
+ // No verified event named this object yet, or it is not the one ordinary traffic reaches (a rehearsal, a
695
+ // former serving object after a deploy moved the traffic): it runs no job.
696
+ return
697
+ }
698
+ if (importing(this.#state.storage)) {
699
+ // A half-loaded import runs no job and arms nothing; the repair tick arms the alarm once the object boots.
700
+ return
701
+ }
702
+ barrier = await this.#maintenance(object)
703
+ if (barrier.mode !== 'open') {
704
+ return
705
+ }
706
+ const outcome = await this.#current('apply')
707
+ if (!('booted' in outcome)) {
708
+ throw new Error('the application did not boot')
709
+ }
710
+ const booted = outcome.booted.app.jobs.scheduler
711
+ scheduler = booted
712
+ if (booted === null) {
713
+ return
714
+ }
715
+ const event = barrier.event()
716
+ await objectScope.run({ storage: this.#state.storage, event }, () => runGated(event, () => this.#wake(booted, event)))
717
+ } catch (error) {
718
+ if (error instanceof MaintenanceFenceError || barrier === null || barrier.mode !== 'open') {
719
+ // A transition restricted the object meanwhile, or none was ever read: no failure, and nothing is armed outside open.
720
+ return
721
+ }
722
+ console.error('fabrika: the alarm failed: alarm_failed')
723
+ try {
724
+ await this.#state.storage.setAlarm?.(Date.now() + ALARM_RETRY_MS)
725
+ } catch {
726
+ console.error('fabrika: the alarm could not be armed again; the repair tick arms it')
727
+ }
728
+ if (scheduler !== null) {
729
+ this.#state.waitUntil(scheduler.failed('alarm', error))
730
+ }
731
+ }
732
+ }
733
+
734
+ /**
735
+ * One alarm: per job, in one transaction, a crashed holder reaped, the lease taken for a due wake (or the pending flag
736
+ * set for the run holding it) and the alarm armed from the projection. The runs go to `waitUntil`: one alarm runs at a
737
+ * time, and awaiting a long job would hold back every other wake. No wake is consumed here, only by its run's finish.
738
+ */
739
+ async #wake(scheduler: JobScheduler, event: BarrierEvent): Promise<void> {
740
+ await this.#repairOnce(scheduler)
741
+ const early = await scheduler.earlyBy(ALARM_SLACK_MS)
742
+ if (early > 0) {
743
+ await this.#clock.wait(early, new AbortController().signal)
744
+ }
745
+ const storage = fencedStorage()
746
+ // A job whose lease is held past its expiry but which the contract no longer declares is reaped too.
747
+ for (const job of [...scheduler.jobs, ...(await scheduler.orphans())]) {
748
+ if (!event.writable()) {
749
+ // A transition restricted the object since the alarm was admitted; reopening arms it again.
750
+ return
751
+ }
752
+ const taken = await storage.transaction(async () => {
753
+ const taken = await scheduler.take(job)
754
+ await armAlarm(eventScope().storage, await scheduler.next())
755
+ return taken
756
+ })
757
+ if (taken.reaped) {
758
+ this.#state.waitUntil(scheduler.crashed(job))
759
+ }
760
+ if (taken.fence !== null) {
761
+ this.#state.waitUntil(runScheduled(scheduler, job, taken.fence, this.#clock))
762
+ }
763
+ }
764
+ }
765
+
766
+ /** The startup repair once per instance, at its first open event; a failure is reported and the next event repairs. */
767
+ async #repairOnce(scheduler: JobScheduler): Promise<void> {
768
+ if (this.#repaired) {
769
+ return
770
+ }
771
+ try {
772
+ this.#repaired = await scheduler.repair()
773
+ } catch (error) {
774
+ this.#repairFailed(scheduler, error)
775
+ }
776
+ }
777
+
778
+ /** A repair the write fence refused is no failure: a transition closed the object, and reopening repairs. */
779
+ #repairFailed(scheduler: JobScheduler, error: unknown): void {
780
+ if (error instanceof MaintenanceFenceError) {
781
+ return
782
+ }
783
+ console.error("fabrika: the job scheduler's repair failed: repair_failed")
784
+ this.#state.waitUntil(scheduler.failed('repair', error))
785
+ }
786
+
787
+ /** The repair tick: the startup repair again, which also arms the first alarm of an environment nobody has called. */
788
+ async #tick(scheduler: JobScheduler | null): Promise<Response> {
789
+ if (scheduler === null) {
790
+ return new Response(null, { status: 204 })
791
+ }
792
+ try {
793
+ this.#repaired = (await scheduler.repair()) || this.#repaired
794
+ } catch (error) {
795
+ this.#repairFailed(scheduler, error)
796
+ return new Response(null, { status: error instanceof MaintenanceFenceError ? 503 : 500 })
797
+ }
798
+ return new Response(null, { status: 204 })
799
+ }
800
+
801
+ /** After a transition that opened this object, its alarm fires at once and repairs: an alarm outside `open` armed nothing. */
802
+ async #reopened(): Promise<void> {
803
+ if (this.#barrier === null || this.#barrierObject !== this.#object) {
804
+ return
805
+ }
806
+ try {
807
+ if ((await this.#barrier).mode !== 'open') {
808
+ return
809
+ }
810
+ this.#repaired = false
811
+ await this.#state.storage.setAlarm?.(Date.now())
812
+ } catch {
813
+ console.error('fabrika: the alarm could not be armed after the object opened; the repair tick arms it')
814
+ }
815
+ }
816
+
817
+ /** The fixed-size digests are compared in constant time, so neither the length nor a prefix of the secret leaks. */
818
+ #fromWorker(request: Request): boolean {
819
+ const presented = request.headers.get(PROXY_SECRET_HEADER) ?? ''
820
+ return this.#secret !== null && presented !== '' && timingSafeEqual(proxySecretDigest(presented), this.#secret)
821
+ }
822
+ }