@fabrika/app 0.9.0-dev.4 → 0.9.0-dev.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -9
- package/skill/SKILL.md +41 -14
- package/skill/references/app-setup.md +43 -12
- package/skill/references/dev.md +2 -1
- package/skill/references/endpoint.md +30 -24
- package/skill/references/errors.md +40 -35
- package/skill/references/files.md +4 -2
- package/skill/references/jobs.md +21 -4
- package/skill/references/table-and-migration.md +19 -2
- package/skill/references/tests.md +40 -1
- package/skill/references/webhooks-links-connectors.md +1 -1
- package/src/build-recipe.ts +198 -0
- package/src/build.ts +60 -4
- package/src/bun.ts +181 -13
- package/src/cloudflare/application.ts +822 -0
- package/src/cloudflare/bindings.ts +159 -0
- package/src/cloudflare/export.ts +621 -0
- package/src/cloudflare/import.ts +1013 -0
- package/src/cloudflare/maintenance.ts +518 -0
- package/src/cloudflare/names.ts +16 -0
- package/src/cloudflare/ops.ts +375 -0
- package/src/cloudflare/r2-files.ts +172 -0
- package/src/cloudflare/restore.ts +167 -0
- package/src/cloudflare/tracker.ts +192 -0
- package/src/cloudflare.ts +100 -146
- package/src/contract/app-id.ts +5 -0
- package/src/contract/artifact-schema.ts +9 -1
- package/src/contract/build.ts +40 -2
- package/src/contract/compile.ts +1 -0
- package/src/contract/context.ts +35 -9
- package/src/contract/declaration.ts +2 -0
- package/src/contract/defaults.ts +22 -2
- package/src/contract/define.ts +29 -3
- package/src/contract/kit.ts +4 -1
- package/src/contract/platform.ts +3 -2
- package/src/contract/vocabulary.ts +26 -8
- package/src/dev/launcher.ts +50 -4
- package/src/disk-assets.ts +8 -3
- package/src/files/archive.ts +2 -2
- package/src/files/job-names.ts +12 -0
- package/src/files/kinds.ts +73 -31
- package/src/files/scan.ts +3 -4
- package/src/files/streams.ts +3 -1
- package/src/implement/anonymous.ts +12 -11
- package/src/implement/boot.ts +75 -6
- package/src/implement/bounded.ts +21 -2
- package/src/implement/capability.ts +15 -3
- package/src/implement/dispatch.ts +419 -177
- package/src/implement/failures.ts +17 -1
- package/src/implement/faults.ts +1 -0
- package/src/implement/index.ts +1 -0
- package/src/implement/invocation.ts +58 -10
- package/src/implement/principal.ts +25 -2
- package/src/implement/terminals.ts +93 -0
- package/src/implement/workload.ts +42 -16
- package/src/jobs/mail.ts +8 -0
- package/src/jobs/report.ts +5 -1
- package/src/jobs/runner.ts +672 -113
- package/src/jobs/schedule.ts +7 -1
- package/src/jobs/scrub.ts +14 -1
- package/src/jobs/types.ts +6 -0
- package/src/spa-fallback.ts +12 -0
- package/src/testing/endpoint-access.ts +307 -152
- package/src/testing/isolation.ts +8 -1
- package/src/testing/kit.ts +80 -2
- package/src/testing/proxy.ts +15 -10
- package/types/src/build-recipe.d.ts +70 -0
- package/types/src/build.d.ts +19 -1
- package/types/src/bun.d.ts +34 -1
- package/types/src/cloudflare/application.d.ts +73 -0
- package/types/src/cloudflare/bindings.d.ts +33 -0
- package/types/src/cloudflare/export.d.ts +16 -0
- package/types/src/cloudflare/import.d.ts +51 -0
- package/types/src/cloudflare/maintenance.d.ts +117 -0
- package/types/src/cloudflare/names.d.ts +11 -0
- package/types/src/cloudflare/ops.d.ts +81 -0
- package/types/src/cloudflare/r2-files.d.ts +9 -0
- package/types/src/cloudflare/restore.d.ts +9 -0
- package/types/src/cloudflare/tracker.d.ts +29 -0
- package/types/src/cloudflare.d.ts +12 -18
- package/types/src/contract/app-id.d.ts +2 -0
- package/types/src/contract/artifact-schema.d.ts +2 -0
- package/types/src/contract/context.d.ts +35 -11
- package/types/src/contract/declaration.d.ts +4 -0
- package/types/src/contract/defaults.d.ts +19 -1
- package/types/src/contract/define.d.ts +20 -3
- package/types/src/contract/kit.d.ts +2 -1
- package/types/src/contract/platform.d.ts +2 -2
- package/types/src/contract/vocabulary.d.ts +9 -7
- package/types/src/disk-assets.d.ts +4 -1
- package/types/src/files/job-names.d.ts +9 -0
- package/types/src/files/kinds.d.ts +21 -14
- package/types/src/files/scan.d.ts +1 -3
- package/types/src/implement/anonymous.d.ts +2 -4
- package/types/src/implement/boot.d.ts +17 -1
- package/types/src/implement/bounded.d.ts +11 -1
- package/types/src/implement/capability.d.ts +4 -2
- package/types/src/implement/dispatch.d.ts +34 -1
- package/types/src/implement/failures.d.ts +5 -0
- package/types/src/implement/faults.d.ts +1 -0
- package/types/src/implement/index.d.ts +1 -0
- package/types/src/implement/invocation.d.ts +22 -7
- package/types/src/implement/principal.d.ts +6 -1
- package/types/src/implement/terminals.d.ts +33 -0
- package/types/src/implement/workload.d.ts +5 -0
- package/types/src/jobs/runner.d.ts +66 -2
- package/types/src/jobs/schedule.d.ts +2 -0
- package/types/src/jobs/types.d.ts +6 -0
- package/types/src/spa-fallback.d.ts +2 -0
- package/types/src/testing/endpoint-access.d.ts +37 -5
- package/types/src/testing/isolation.d.ts +1 -1
- package/types/src/testing/kit.d.ts +22 -1
- package/types/src/testing/proxy.d.ts +4 -4
|
@@ -0,0 +1,822 @@
|
|
|
1
|
+
// The Durable Object an application environment runs in (ADR-0065): it migrates its SQLite, boots the application,
|
|
2
|
+
// serves every request and repair tick its Worker forwards, and runs the jobs from its alarm, which it arms from their
|
|
3
|
+
// durable wakes in the transaction of every change. A plain class, so Bun and Lopata load it without `cloudflare:workers`.
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
FABRIKA_APPLICATION_OBJECT,
|
|
7
|
+
FABRIKA_APPLICATION_OBJECTS,
|
|
8
|
+
FABRIKA_CONTRACT_DIGEST,
|
|
9
|
+
FABRIKA_PROXY_SECRET,
|
|
10
|
+
OPS_PREFIX,
|
|
11
|
+
type OpsBootFailureContent,
|
|
12
|
+
type OpsMode,
|
|
13
|
+
parseApplicationObjects,
|
|
14
|
+
PROXY_SECRET_HEADER,
|
|
15
|
+
} from '@fabrika/auth-core'
|
|
16
|
+
import { isScopedDatabase, type LeaseFence, type Session } from '@fabrika/db/framework'
|
|
17
|
+
import { SQLITE_FRAMEWORK_MIGRATIONS } from '@fabrika/db/migrations-sqlite'
|
|
18
|
+
import { openSqlite, type SqliteCursor, type SqliteStorage } from '@fabrika/db/sqlite'
|
|
19
|
+
import { migrateSqlite, type SqliteMigrationBundle, type SqliteMigrationTarget } from '@fabrika/db/sqlite-migrate'
|
|
20
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
21
|
+
import { timingSafeEqual } from 'node:crypto'
|
|
22
|
+
import type { RequestExecutionContext } from '../fetch.js'
|
|
23
|
+
import { FILE_TOMBSTONE_RETENTION_MS } from '../files/kinds.js'
|
|
24
|
+
import { boot, type ContractApp } from '../implement/boot.js'
|
|
25
|
+
import { type CrashEvents, environmentCrashReporter, reportSurvivedCrashes } from '../implement/crashes.js'
|
|
26
|
+
import { MaintenanceFenceError, maintenanceRefusal, proxySecretDigest, runGated } from '../implement/dispatch.js'
|
|
27
|
+
import { type CapturedEnvironment, captureEnvironment } from '../implement/environment.js'
|
|
28
|
+
import type { ContractImplementation } from '../implement/implement.js'
|
|
29
|
+
import type { CrashReporter } from '../jobs/report.js'
|
|
30
|
+
import { type JobClock, type JobScheduler, systemClock } from '../jobs/runner.js'
|
|
31
|
+
import { environmentScrubber, type Scrubber } from '../jobs/scrub.js'
|
|
32
|
+
import { bindingAssets, bindingKeySource, bindingMembership, bindingTerminals, bindingWorkloadTokens, connectBindings } from './bindings.js'
|
|
33
|
+
import { importInProgress } from './import.js'
|
|
34
|
+
import { type BarrierEvent, MaintenanceBarrier, r2MaintenanceStore } from './maintenance.js'
|
|
35
|
+
import { CLOUDFLARE_ASSETS_BINDING, CLOUDFLARE_FILES_BINDING } from './names.js'
|
|
36
|
+
import { type OpsAccess, opsAccess, type OpsRecovery, type OpsStorage, serveOps } from './ops.js'
|
|
37
|
+
import { r2FileObjectStore } from './r2-files.js'
|
|
38
|
+
import { trackPending } from './tracker.js'
|
|
39
|
+
|
|
40
|
+
/** The application module, imported only after the environment was captured: the generated entrypoint's `import()`. */
|
|
41
|
+
export type ImplementationLoader = () => Promise<{ readonly default: ContractImplementation }>
|
|
42
|
+
|
|
43
|
+
/** One of the application's SQLite migrations, as `fabrika app build` bundles them from `db/migrations`. */
|
|
44
|
+
export interface ApplicationMigration {
|
|
45
|
+
readonly name: string
|
|
46
|
+
readonly sql: string
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface ContractWorkerOptions {
|
|
50
|
+
/** The application's own SQLite migrations, applied as the bundle `app` after the framework's `fabrika`. */
|
|
51
|
+
readonly migrations?: readonly ApplicationMigration[]
|
|
52
|
+
/** Where the isolate hears of an unhandled rejection; the global `unhandledrejection` event by default. */
|
|
53
|
+
readonly rejections?: CrashEvents
|
|
54
|
+
/** What posts a report to Operations' ingest; `fetch` by default. */
|
|
55
|
+
readonly operationsFetch?: (request: Request) => Promise<Response>
|
|
56
|
+
/** The jobs' and the alarm's waits and an attempt's deadline; the system clock by default. */
|
|
57
|
+
readonly clock?: JobClock
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Marks the Worker's repair tick, with its `scheduledTime`. The Worker's `fetch` removes it from every request it
|
|
62
|
+
* forwards, so only its own `scheduled` handler reaches the tick.
|
|
63
|
+
*/
|
|
64
|
+
export const TICK_HEADER = 'X-Fabrika-Tick'
|
|
65
|
+
|
|
66
|
+
/** A wake runs at most this early: `Date.now()` is pinned between I/O, so an alarm may start a moment before it (I6). */
|
|
67
|
+
export const ALARM_SLACK_MS = 50
|
|
68
|
+
/** After an internal error the alarm is armed again this far ahead; the repair tick covers a re-arm that failed. */
|
|
69
|
+
export const ALARM_RETRY_MS = 30_000
|
|
70
|
+
/**
|
|
71
|
+
* The object's KV key that holds the name of the object it is, recorded at its first verified event: an alarm names no
|
|
72
|
+
* object, and the one ordinary traffic reaches may since have changed. Outside the SQLite, so no dump carries it.
|
|
73
|
+
*/
|
|
74
|
+
export const OBJECT_NAME_KEY = 'fabrika:object'
|
|
75
|
+
/** A repair tick's time: the Worker's `scheduledTime`, digits only. */
|
|
76
|
+
const TICK_TIME = /^\d{1,16}$/
|
|
77
|
+
|
|
78
|
+
/** The object's alarm. On a SQLite-backed object `setAlarm` and `deleteAlarm` commit with the open transaction. */
|
|
79
|
+
export interface AlarmStorage {
|
|
80
|
+
setAlarm(scheduledTime: number): Promise<void>
|
|
81
|
+
deleteAlarm(): Promise<void>
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* What the object needs of its `ctx.storage`; a Durable Object's `DurableObjectStorage` fits it. A test double without
|
|
86
|
+
* an alarm arms none.
|
|
87
|
+
*/
|
|
88
|
+
export type ApplicationStorage = SqliteStorage & OpsStorage & Partial<AlarmStorage>
|
|
89
|
+
|
|
90
|
+
/** What the object needs of its `DurableObjectState`. */
|
|
91
|
+
export interface ApplicationState {
|
|
92
|
+
readonly storage: ApplicationStorage & Partial<Omit<OpsRecovery, 'abort'>>
|
|
93
|
+
blockConcurrencyWhile<T>(callback: () => Promise<T>): Promise<T>
|
|
94
|
+
waitUntil(promise: Promise<unknown>): void
|
|
95
|
+
/** workerd resets the object; the test double has none. */
|
|
96
|
+
abort?(reason?: string): void
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** `state` with every `waitUntil` promise recorded by the isolate's pending-work tracker (the ops operation `timers`). */
|
|
100
|
+
export function trackedState(state: ApplicationState): ApplicationState {
|
|
101
|
+
const { abort } = state
|
|
102
|
+
return {
|
|
103
|
+
storage: state.storage,
|
|
104
|
+
blockConcurrencyWhile: (callback) => state.blockConcurrencyWhile(callback),
|
|
105
|
+
waitUntil: (promise) => state.waitUntil(trackPending(promise)),
|
|
106
|
+
...(abort === undefined ? {} : { abort: (reason?: string) => abort.call(state, reason) }),
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The object's recovery when its storage and state have every member PITR needs (ADR-0070), else `null`. */
|
|
111
|
+
function objectRecovery(state: ApplicationState): OpsRecovery | null {
|
|
112
|
+
const { storage } = state
|
|
113
|
+
const { getCurrentBookmark, getBookmarkForTime, onNextSessionRestoreBookmark } = storage
|
|
114
|
+
const { abort } = state
|
|
115
|
+
if (getCurrentBookmark === undefined || getBookmarkForTime === undefined || onNextSessionRestoreBookmark === undefined || abort === undefined) {
|
|
116
|
+
return null
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
getCurrentBookmark: () => getCurrentBookmark.call(storage),
|
|
120
|
+
getBookmarkForTime: (time) => getBookmarkForTime.call(storage, time),
|
|
121
|
+
onNextSessionRestoreBookmark: (bookmark) => onNextSessionRestoreBookmark.call(storage, bookmark),
|
|
122
|
+
abort: (reason) => abort.call(state, reason),
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** The framework's bindings: IAM's application entrypoint and the Workers Assets of the SPA. No handler sees them. */
|
|
127
|
+
const FRAMEWORK_BINDINGS: ReadonlySet<string> = new Set(['IAM', CLOUDFLARE_ASSETS_BINDING])
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The application this isolate booted, once: its module is loaded once, and its `scoped(tables)` layer binds once, to
|
|
131
|
+
* the storage of the object whose event runs (`objectStorage`), never to one database.
|
|
132
|
+
*/
|
|
133
|
+
interface Booted {
|
|
134
|
+
readonly app: ContractApp
|
|
135
|
+
/** Whether the application has a data layer, which every object instance migrates in its own storage. */
|
|
136
|
+
readonly migrates: boolean
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
interface ApplicationSetup {
|
|
140
|
+
readonly load: ImplementationLoader
|
|
141
|
+
readonly options: ContractWorkerOptions
|
|
142
|
+
booted: Promise<Booted> | null
|
|
143
|
+
/** Whether the current streak of failed attachments was reported; an attachment that succeeds ends the streak. */
|
|
144
|
+
failureReported: boolean
|
|
145
|
+
watchingRejections: boolean
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
let registered: ApplicationSetup | null = null
|
|
149
|
+
|
|
150
|
+
/** The storage of the object whose event runs, and the barrier's view of that event; work it hands to `waitUntil` keeps both. */
|
|
151
|
+
interface ObjectScope {
|
|
152
|
+
readonly storage: ApplicationStorage
|
|
153
|
+
readonly event: BarrierEvent
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const objectScope = new AsyncLocalStorage<ObjectScope>()
|
|
157
|
+
|
|
158
|
+
/** The cursors of the fenced transaction that runs: its commit checks whether any of them wrote. */
|
|
159
|
+
const openTransaction = new AsyncLocalStorage<SqliteCursor[]>()
|
|
160
|
+
|
|
161
|
+
function eventScope(): ObjectScope {
|
|
162
|
+
const scope = objectScope.getStore()
|
|
163
|
+
if (scope === undefined) {
|
|
164
|
+
throw new Error('a statement ran outside any event of an application object: the framework runs every event in its object scope')
|
|
165
|
+
}
|
|
166
|
+
return scope
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** The write fence (ADR-0069 §3): a write of an invocation admitted before a restrictive transition rolls back. */
|
|
170
|
+
function fence(event: BarrierEvent, cursors: readonly SqliteCursor[]): void {
|
|
171
|
+
if (cursors.some((cursor) => cursor.rowsWritten > 0) && !event.writable()) {
|
|
172
|
+
throw new MaintenanceFenceError()
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* The storage every framework and application write on SQLite reaches, bound to the object whose event runs. The
|
|
178
|
+
* fence runs inside each transaction before it commits; a statement outside one runs in a transaction of its own.
|
|
179
|
+
*/
|
|
180
|
+
function fencedStorage(): SqliteStorage {
|
|
181
|
+
return {
|
|
182
|
+
sql: {
|
|
183
|
+
exec(query, ...bindings) {
|
|
184
|
+
const { storage, event } = eventScope()
|
|
185
|
+
const cursors = openTransaction.getStore()
|
|
186
|
+
if (cursors !== undefined) {
|
|
187
|
+
const cursor = storage.sql.exec(query, ...bindings)
|
|
188
|
+
cursors.push(cursor)
|
|
189
|
+
return cursor
|
|
190
|
+
}
|
|
191
|
+
return storage.transactionSync(() => {
|
|
192
|
+
const cursor = storage.sql.exec(query, ...bindings)
|
|
193
|
+
const rows = cursor.toArray()
|
|
194
|
+
fence(event, [cursor])
|
|
195
|
+
return { toArray: () => rows, rowsWritten: cursor.rowsWritten }
|
|
196
|
+
})
|
|
197
|
+
},
|
|
198
|
+
},
|
|
199
|
+
transaction(closure) {
|
|
200
|
+
const { storage, event } = eventScope()
|
|
201
|
+
// A nested transaction shares its parent's cursors, so the outer commit checks every write inside it.
|
|
202
|
+
const cursors = openTransaction.getStore() ?? []
|
|
203
|
+
return storage.transaction(() =>
|
|
204
|
+
openTransaction.run(cursors, async () => {
|
|
205
|
+
const result = await closure()
|
|
206
|
+
fence(event, cursors)
|
|
207
|
+
return result
|
|
208
|
+
})
|
|
209
|
+
)
|
|
210
|
+
},
|
|
211
|
+
transactionSync(closure) {
|
|
212
|
+
const { storage, event } = eventScope()
|
|
213
|
+
const cursors = openTransaction.getStore() ?? []
|
|
214
|
+
return storage.transactionSync(() =>
|
|
215
|
+
openTransaction.run(cursors, () => {
|
|
216
|
+
const result = closure()
|
|
217
|
+
fence(event, cursors)
|
|
218
|
+
return result
|
|
219
|
+
})
|
|
220
|
+
)
|
|
221
|
+
},
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Whether migrations may be applied: in `open` and the migration phase; a restricted cold start only looks. */
|
|
226
|
+
type MigrationPolicy = 'apply' | 'refuse'
|
|
227
|
+
|
|
228
|
+
/** A restricted cold start found a migration to apply: the application does not boot until the operation's migrate. */
|
|
229
|
+
class MigrationsPending extends Error {
|
|
230
|
+
override readonly name = 'MigrationsPending'
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** The object holds an import marker (ADR-0070 §6): it neither boots nor migrates until the import completes or is discarded. */
|
|
234
|
+
class ImportInProgress extends Error {
|
|
235
|
+
override readonly name = 'ImportInProgress'
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Whether the object holds an import marker; a storage without the synchronous KV (the bun:sqlite double) holds none. */
|
|
239
|
+
function importing(storage: ApplicationStorage): boolean {
|
|
240
|
+
const { kv } = storage
|
|
241
|
+
return kv !== undefined && importInProgress({ kv })
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function isMigrationsPending(error: unknown): boolean {
|
|
245
|
+
return error instanceof MigrationsPending || (error instanceof Error && error.cause instanceof MigrationsPending)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* The migration runner's target that applies nothing: it answers the runner's reads, and the first write it would make
|
|
250
|
+
* (the ledger's DDL, a migration, a digest) throws `MigrationsPending` instead.
|
|
251
|
+
*/
|
|
252
|
+
function refusingTarget(storage: ApplicationStorage): SqliteMigrationTarget {
|
|
253
|
+
return {
|
|
254
|
+
exec(query, ...bindings) {
|
|
255
|
+
if (!/^\s*select\b/i.test(query)) {
|
|
256
|
+
throw new MigrationsPending('a migration is pending')
|
|
257
|
+
}
|
|
258
|
+
return storage.sql.exec(query, ...bindings)
|
|
259
|
+
},
|
|
260
|
+
transactionSync() {
|
|
261
|
+
throw new MigrationsPending('a migration is pending')
|
|
262
|
+
},
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** What the object classes of this isolate boot: the generated entrypoint's `createContractWorker` registers it. */
|
|
267
|
+
export function registerApplication(load: ImplementationLoader, options: ContractWorkerOptions): void {
|
|
268
|
+
registered = { load, options, booted: null, failureReported: false, watchingRejections: false }
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** The isolate's `unhandledrejection` event. Listening leaves the runtime's own handling, its log line, in place. */
|
|
272
|
+
function globalRejectionEvents(): CrashEvents {
|
|
273
|
+
return {
|
|
274
|
+
listen(listener) {
|
|
275
|
+
const rejection = (event: Event) => listener('unhandled_rejection', Reflect.get(event, 'reason'))
|
|
276
|
+
globalThis.addEventListener('unhandledrejection', rejection)
|
|
277
|
+
return () => globalThis.removeEventListener('unhandledrejection', rejection)
|
|
278
|
+
},
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* The framework's tables, then the application's, each migration once: a booted storage applies nothing. Under
|
|
284
|
+
* `refuse` a pending migration throws `MigrationsPending` before anything is written.
|
|
285
|
+
*/
|
|
286
|
+
async function migrate(storage: ApplicationStorage, migrations: readonly ApplicationMigration[], policy: MigrationPolicy): Promise<void> {
|
|
287
|
+
const target: SqliteMigrationTarget = policy === 'apply'
|
|
288
|
+
? { exec: (query, ...bindings) => storage.sql.exec(query, ...bindings), transactionSync: (fn) => storage.transactionSync(fn) }
|
|
289
|
+
: refusingTarget(storage)
|
|
290
|
+
await migrateSqlite(target, migrationBundles(migrations))
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** The bundles the object applies, in order: the framework's tables, then the application's. */
|
|
294
|
+
function migrationBundles(migrations: readonly ApplicationMigration[]): readonly SqliteMigrationBundle[] {
|
|
295
|
+
return [
|
|
296
|
+
{ bundle: 'fabrika', migrations: SQLITE_FRAMEWORK_MIGRATIONS },
|
|
297
|
+
{ bundle: 'app', migrations },
|
|
298
|
+
]
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
async function bootApplication(
|
|
302
|
+
setup: ApplicationSetup,
|
|
303
|
+
state: ApplicationState,
|
|
304
|
+
env: Readonly<Record<string, unknown>>,
|
|
305
|
+
environment: CapturedEnvironment,
|
|
306
|
+
crashes: CrashReporter | null,
|
|
307
|
+
policy: MigrationPolicy,
|
|
308
|
+
): Promise<Booted> {
|
|
309
|
+
const { default: implementation } = await setup.load()
|
|
310
|
+
const migrates = isScopedDatabase(implementation.database)
|
|
311
|
+
if (migrates) {
|
|
312
|
+
await migrate(state.storage, setup.options.migrations ?? [], policy)
|
|
313
|
+
}
|
|
314
|
+
const database = migrates ? openSqlite(fencedStorage()) : undefined
|
|
315
|
+
const app = boot(implementation, {
|
|
316
|
+
environment,
|
|
317
|
+
iam: bindingKeySource(env['IAM']),
|
|
318
|
+
assets: bindingAssets(env[CLOUDFLARE_ASSETS_BINDING]),
|
|
319
|
+
membership: bindingMembership(env['IAM']),
|
|
320
|
+
terminals: bindingTerminals(env['IAM']),
|
|
321
|
+
workloadTokens: bindingWorkloadTokens(env['IAM']),
|
|
322
|
+
connectBindings: connectBindings(env),
|
|
323
|
+
database,
|
|
324
|
+
files: r2FileObjectStore(env[CLOUDFLARE_FILES_BINDING]),
|
|
325
|
+
// A removed file's object outlives its row for the window, so a side restore finds its bytes (ADR-0070).
|
|
326
|
+
fileRetentionMs: FILE_TOMBSTONE_RETENTION_MS,
|
|
327
|
+
// The Worker forwards only with the proxy secret, which the object checked; `boot` checks it again.
|
|
328
|
+
requireProxySecret: true,
|
|
329
|
+
// The proxy Worker sets `X-Fabrika-Client-Ip` from `CF-Connecting-IP`, the visitor Cloudflare saw.
|
|
330
|
+
clientAddress: 'vouched',
|
|
331
|
+
projectWakes: (transaction) => projectWakes(setup, transaction),
|
|
332
|
+
...(setup.options.operationsFetch === undefined ? {} : { operationsFetch: setup.options.operationsFetch }),
|
|
333
|
+
})
|
|
334
|
+
const booted: Booted = { app, migrates }
|
|
335
|
+
if (crashes !== null && !setup.watchingRejections) {
|
|
336
|
+
setup.watchingRejections = true
|
|
337
|
+
reportSurvivedCrashes(setup.options.rejections ?? globalRejectionEvents(), crashes)
|
|
338
|
+
}
|
|
339
|
+
return booted
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* The isolate's booted application with this instance's storage migrated, or the first boot. Reports the first failure
|
|
344
|
+
* of a streak as `boot_failed`, whichever instance or step it comes from; an attachment that succeeds ends the streak.
|
|
345
|
+
*/
|
|
346
|
+
async function attach(
|
|
347
|
+
setup: ApplicationSetup | null,
|
|
348
|
+
state: ApplicationState,
|
|
349
|
+
env: Readonly<Record<string, unknown>>,
|
|
350
|
+
policy: MigrationPolicy,
|
|
351
|
+
failed: (failure: OpsBootFailureContent) => void,
|
|
352
|
+
): Promise<Booted> {
|
|
353
|
+
if (setup === null) {
|
|
354
|
+
throw new Error('no contract application is registered: the generated entrypoint calls createContractWorker before an object starts')
|
|
355
|
+
}
|
|
356
|
+
// Before anything loads or migrates, and no boot failure: a half-loaded import never boots.
|
|
357
|
+
if (importing(state.storage)) {
|
|
358
|
+
throw new ImportInProgress('import_in_progress')
|
|
359
|
+
}
|
|
360
|
+
const environment = captureEnvironment(Object.fromEntries(Object.entries(env).filter(([name]) => !FRAMEWORK_BINDINGS.has(name))))
|
|
361
|
+
const crashes = environmentCrashReporter(environment, setup.options.operationsFetch)
|
|
362
|
+
try {
|
|
363
|
+
const booted = await attachOnce(setup, state, env, environment, crashes, policy)
|
|
364
|
+
setup.failureReported = false
|
|
365
|
+
return booted
|
|
366
|
+
} catch (error) {
|
|
367
|
+
if (isMigrationsPending(error)) {
|
|
368
|
+
throw error
|
|
369
|
+
}
|
|
370
|
+
failed(bootFailure(error, environmentScrubber(environment)))
|
|
371
|
+
if (crashes !== null && !setup.failureReported) {
|
|
372
|
+
setup.failureReported = true
|
|
373
|
+
state.waitUntil(crashes.report('boot_failed', error))
|
|
374
|
+
}
|
|
375
|
+
throw error
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
async function attachOnce(
|
|
380
|
+
setup: ApplicationSetup,
|
|
381
|
+
state: ApplicationState,
|
|
382
|
+
env: Readonly<Record<string, unknown>>,
|
|
383
|
+
environment: CapturedEnvironment,
|
|
384
|
+
crashes: CrashReporter | null,
|
|
385
|
+
policy: MigrationPolicy,
|
|
386
|
+
): Promise<Booted> {
|
|
387
|
+
if (setup.booted === null) {
|
|
388
|
+
const booting = bootApplication(setup, state, env, environment, crashes, policy)
|
|
389
|
+
setup.booted = booting
|
|
390
|
+
// A failed boot is not kept: the next object instance boots again and fails with the same problems.
|
|
391
|
+
booting.catch(() => {
|
|
392
|
+
setup.booted = null
|
|
393
|
+
})
|
|
394
|
+
return await booting
|
|
395
|
+
}
|
|
396
|
+
const booted = await setup.booted
|
|
397
|
+
if (booted.migrates) {
|
|
398
|
+
await migrate(state.storage, setup.options.migrations ?? [], policy)
|
|
399
|
+
}
|
|
400
|
+
return booted
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** A failed boot as the ops view shows it, its text scrubbed as a crash report's is. */
|
|
404
|
+
function bootFailure(error: unknown, scrubber: Scrubber): OpsBootFailureContent {
|
|
405
|
+
const at = new Date().toISOString()
|
|
406
|
+
if (error instanceof Error) {
|
|
407
|
+
return { at, message: scrubber.scrub(error.message), stack: typeof error.stack === 'string' ? scrubber.scrub(error.stack) : null }
|
|
408
|
+
}
|
|
409
|
+
const message = typeof error === 'string' ? scrubber.scrub(error) : `a thrown ${error === null ? 'null' : typeof error} that is not an Error`
|
|
410
|
+
return { at, message, stack: null }
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/**
|
|
414
|
+
* `pending`: a restricted cold start found a migration to apply, or the object holds an import marker, so the
|
|
415
|
+
* application did not boot; that is no failure.
|
|
416
|
+
*/
|
|
417
|
+
type Outcome = { readonly booted: Booted } | { readonly failure: unknown } | { readonly pending: true }
|
|
418
|
+
|
|
419
|
+
/** The wait after an instance's first failed boot before an event boots again; it doubles with each failure, to the cap. */
|
|
420
|
+
const BOOT_RETRY_MS = 1_000
|
|
421
|
+
const BOOT_RETRY_CAP_MS = 60_000
|
|
422
|
+
|
|
423
|
+
/** The alarm armed at `next`, or deleted when nothing is pending; inside a transaction it commits with it. */
|
|
424
|
+
async function armAlarm(storage: ApplicationStorage, next: Date | null): Promise<void> {
|
|
425
|
+
if (next === null) {
|
|
426
|
+
await storage.deleteAlarm?.()
|
|
427
|
+
} else {
|
|
428
|
+
await storage.setAlarm?.(next.getTime())
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* The runtime's projection of the jobs' wakes and leases (ADR-0065 §5, I1): the store runs it in the transaction of
|
|
434
|
+
* every change, and it arms the alarm of the object whose event runs from `next`. A throw rolls the change back.
|
|
435
|
+
*/
|
|
436
|
+
async function projectWakes(setup: ApplicationSetup, transaction: Session): Promise<void> {
|
|
437
|
+
// A wake or lease changes only once the application booted.
|
|
438
|
+
const booted = setup.booted === null ? null : await setup.booted
|
|
439
|
+
const scheduler = booted?.app.jobs.scheduler ?? null
|
|
440
|
+
if (scheduler !== null) {
|
|
441
|
+
await armAlarm(eventScope().storage, await scheduler.next(transaction))
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/** A run the alarm took the lease for, handed to `waitUntil`; the runner logs and reports its own failures. */
|
|
446
|
+
async function runScheduled(scheduler: JobScheduler, job: string, fence: LeaseFence, clock: JobClock): Promise<void> {
|
|
447
|
+
try {
|
|
448
|
+
await scheduler.run(job, fence, { clock })
|
|
449
|
+
} catch {
|
|
450
|
+
console.error(`fabrika: job '${job}' did not start: driver_failed`)
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/** The application environment's one object. The Worker's generated entrypoint exports it as `FabrikaApplication`. */
|
|
455
|
+
export class FabrikaApplication {
|
|
456
|
+
readonly #state: ApplicationState
|
|
457
|
+
readonly #env: Readonly<Record<string, unknown>>
|
|
458
|
+
readonly #setup: ApplicationSetup | null
|
|
459
|
+
/** `proxySecretDigest` of the environment's proxy secret; `null` refuses every request. */
|
|
460
|
+
readonly #secret: Uint8Array | null
|
|
461
|
+
/** What the platform wrote for Control's ops channel; `null` refuses every ops request. */
|
|
462
|
+
readonly #ops: OpsAccess | null
|
|
463
|
+
/** The object ordinary traffic reaches, so the one this instance is when such traffic arrives; `null` when malformed. */
|
|
464
|
+
readonly #object: string | null
|
|
465
|
+
/** The object this instance is, once a verified event named it or its storage recorded it (`OBJECT_NAME_KEY`). */
|
|
466
|
+
#name: string | null = null
|
|
467
|
+
/** The maintenance barrier, read from R2 once per instance; a failed read is not kept, so the next request reads again. */
|
|
468
|
+
#barrier: Promise<MaintenanceBarrier> | null = null
|
|
469
|
+
#barrierObject: string | null = null
|
|
470
|
+
/** The instance's failed boot, kept in memory for the ops view until a boot succeeds. */
|
|
471
|
+
#bootFailure: OpsBootFailureContent | null = null
|
|
472
|
+
/** The instance's boot, `null` until the first event that needs the application: an ops request never starts it. */
|
|
473
|
+
#outcome: Promise<Outcome> | null = null
|
|
474
|
+
#failedBoots = 0
|
|
475
|
+
/** When an event may boot again after a failed boot; until then every request answers the failure. */
|
|
476
|
+
#retryAt = 0
|
|
477
|
+
/** Whether this instance ran the scheduler's startup repair, which its first open event does. */
|
|
478
|
+
#repaired = false
|
|
479
|
+
readonly #clock: JobClock
|
|
480
|
+
|
|
481
|
+
constructor(state: ApplicationState, env: Readonly<Record<string, unknown>>) {
|
|
482
|
+
this.#state = trackedState(state)
|
|
483
|
+
this.#env = env
|
|
484
|
+
this.#setup = registered
|
|
485
|
+
this.#clock = registered?.options.clock ?? systemClock
|
|
486
|
+
const secret = env[FABRIKA_PROXY_SECRET]
|
|
487
|
+
this.#secret = typeof secret === 'string' && secret !== '' ? proxySecretDigest(secret) : null
|
|
488
|
+
this.#ops = opsAccess(env)
|
|
489
|
+
const object = env[FABRIKA_APPLICATION_OBJECT]
|
|
490
|
+
const objects = env[FABRIKA_APPLICATION_OBJECTS]
|
|
491
|
+
this.#object = parseApplicationObjects(typeof object === 'string' ? object : undefined, typeof objects === 'string' ? objects : undefined)
|
|
492
|
+
?.object ?? null
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
#contractDigest(): string | null {
|
|
496
|
+
const digest = this.#env[FABRIKA_CONTRACT_DIGEST]
|
|
497
|
+
return typeof digest === 'string' ? digest : null
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
async fetch(request: Request): Promise<Response> {
|
|
501
|
+
// Another Worker on the account that binds this class reaches nothing, not even an answer that the object exists.
|
|
502
|
+
if (!this.#fromWorker(request)) {
|
|
503
|
+
return new Response(null, { status: 404 })
|
|
504
|
+
}
|
|
505
|
+
if (new URL(request.url).pathname.startsWith(OPS_PREFIX)) {
|
|
506
|
+
// The mode before the request: a transition that opens the object arms its alarm, which repairs.
|
|
507
|
+
let before: OpsMode | null = null
|
|
508
|
+
// Served from storage alone: it never starts a boot or a retry, and waits for one under way only once authenticated.
|
|
509
|
+
const response = await serveOps(request, this.#ops, {
|
|
510
|
+
storage: this.#state.storage,
|
|
511
|
+
bucket: this.#env[CLOUDFLARE_FILES_BINDING],
|
|
512
|
+
recovery: objectRecovery(this.#state),
|
|
513
|
+
contractDigest: this.#contractDigest(),
|
|
514
|
+
migrations: migrationBundles(this.#setup?.options.migrations ?? []),
|
|
515
|
+
bootFailure: async () => {
|
|
516
|
+
await this.#outcome
|
|
517
|
+
return this.#bootFailure
|
|
518
|
+
},
|
|
519
|
+
maintenance: async (object) => {
|
|
520
|
+
const barrier = await this.#maintenance(object)
|
|
521
|
+
before ??= barrier.mode
|
|
522
|
+
return barrier
|
|
523
|
+
},
|
|
524
|
+
migrate: () => this.#migrated(),
|
|
525
|
+
})
|
|
526
|
+
if (before !== null && before !== 'open') {
|
|
527
|
+
await this.#reopened()
|
|
528
|
+
}
|
|
529
|
+
return response
|
|
530
|
+
}
|
|
531
|
+
let barrier: MaintenanceBarrier
|
|
532
|
+
try {
|
|
533
|
+
barrier = await this.#maintenance(this.#object)
|
|
534
|
+
} catch {
|
|
535
|
+
// Fails closed: a state R2 could not give is never read as `open` (ADR-0069 §3, cold starts).
|
|
536
|
+
return maintenanceRefusal()
|
|
537
|
+
}
|
|
538
|
+
if (importing(this.#state.storage)) {
|
|
539
|
+
// ADR-0070 §6: whatever the mode, a half-loaded import serves nothing; the tick repairs nothing.
|
|
540
|
+
return request.headers.get(TICK_HEADER) === null ? maintenanceRefusal() : new Response(null, { status: 204 })
|
|
541
|
+
}
|
|
542
|
+
if (barrier.mode !== 'open' && request.headers.get(TICK_HEADER) !== null) {
|
|
543
|
+
// The tick repairs nothing and writes nothing until the object opens.
|
|
544
|
+
return new Response(null, { status: 204 })
|
|
545
|
+
}
|
|
546
|
+
if (barrier.mode === 'closed' || barrier.mode === 'inspect') {
|
|
547
|
+
return maintenanceRefusal()
|
|
548
|
+
}
|
|
549
|
+
const outcome = await this.#current(barrier.mode === 'open' ? 'apply' : 'refuse')
|
|
550
|
+
if ('pending' in outcome) {
|
|
551
|
+
return maintenanceRefusal()
|
|
552
|
+
}
|
|
553
|
+
if ('failure' in outcome) {
|
|
554
|
+
throw outcome.failure
|
|
555
|
+
}
|
|
556
|
+
const event = barrier.event()
|
|
557
|
+
return await objectScope.run({ storage: this.#state.storage, event }, () => runGated(event, () => this.#serve(outcome.booted, request)))
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** The name of the object this instance is: the one its storage recorded, or `null` before a verified event named it. */
|
|
561
|
+
#recorded(): string | null {
|
|
562
|
+
if (this.#name === null) {
|
|
563
|
+
const stored = this.#state.storage.kv?.get(OBJECT_NAME_KEY)
|
|
564
|
+
if (typeof stored === 'string' && stored !== '') {
|
|
565
|
+
this.#name = stored
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
return this.#name
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* Whether this instance is the object a verified event names; the first such event records the name, which never
|
|
573
|
+
* changes after. A storage without KV (the bun:sqlite double) keeps it in memory only.
|
|
574
|
+
*/
|
|
575
|
+
#named(object: string): boolean {
|
|
576
|
+
const recorded = this.#recorded()
|
|
577
|
+
if (recorded !== null) {
|
|
578
|
+
return recorded === object
|
|
579
|
+
}
|
|
580
|
+
this.#state.storage.kv?.put(OBJECT_NAME_KEY, object)
|
|
581
|
+
this.#name = object
|
|
582
|
+
return true
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/** The barrier of the object this instance is; every name it is asked under must be the same one. */
|
|
586
|
+
#maintenance(object: string | null): Promise<MaintenanceBarrier> {
|
|
587
|
+
if (object === null) {
|
|
588
|
+
return Promise.reject(new Error(`${FABRIKA_APPLICATION_OBJECT} and ${FABRIKA_APPLICATION_OBJECTS} name no valid set of objects`))
|
|
589
|
+
}
|
|
590
|
+
if (!this.#named(object)) {
|
|
591
|
+
return Promise.reject(new Error('this object is another object than the one the event names'))
|
|
592
|
+
}
|
|
593
|
+
if (this.#barrier !== null && this.#barrierObject !== object) {
|
|
594
|
+
return Promise.reject(new Error('one object instance was asked under two object names'))
|
|
595
|
+
}
|
|
596
|
+
if (this.#barrier === null) {
|
|
597
|
+
const reading = (async () => await MaintenanceBarrier.load(r2MaintenanceStore(this.#env[CLOUDFLARE_FILES_BINDING], object)))()
|
|
598
|
+
this.#barrier = reading
|
|
599
|
+
this.#barrierObject = object
|
|
600
|
+
reading.catch(() => {
|
|
601
|
+
if (this.#barrier === reading) {
|
|
602
|
+
this.#barrier = null
|
|
603
|
+
this.#barrierObject = null
|
|
604
|
+
}
|
|
605
|
+
})
|
|
606
|
+
}
|
|
607
|
+
return this.#barrier
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/** The migration phase's boot: applies the pending migrations whatever the mode; `false` when the boot failed. */
|
|
611
|
+
async #migrated(): Promise<boolean> {
|
|
612
|
+
const current = this.#outcome === null ? null : await this.#outcome
|
|
613
|
+
if (current !== null && 'booted' in current) {
|
|
614
|
+
return true
|
|
615
|
+
}
|
|
616
|
+
const outcome = this.#boot('apply')
|
|
617
|
+
this.#outcome = outcome
|
|
618
|
+
return 'booted' in this.#settled(outcome, await outcome)
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
/** Settles either way: a callback that rejects would reset the object, and every request answers the failure instead. */
|
|
622
|
+
#boot(policy: MigrationPolicy): Promise<Outcome> {
|
|
623
|
+
return this.#state.blockConcurrencyWhile(() =>
|
|
624
|
+
attach(this.#setup, this.#state, this.#env, policy, (failure) => {
|
|
625
|
+
this.#bootFailure = failure
|
|
626
|
+
}).then((booted): Outcome => {
|
|
627
|
+
// The view reports the current boot: a failure an earlier attempt left is no longer the object's state.
|
|
628
|
+
this.#bootFailure = null
|
|
629
|
+
return { booted }
|
|
630
|
+
}, (failure: unknown): Outcome => {
|
|
631
|
+
if (isMigrationsPending(failure) || failure instanceof ImportInProgress) {
|
|
632
|
+
return { pending: true }
|
|
633
|
+
}
|
|
634
|
+
this.#failedBoots++
|
|
635
|
+
this.#retryAt = Date.now() + Math.min(BOOT_RETRY_MS * 2 ** (this.#failedBoots - 1), BOOT_RETRY_CAP_MS)
|
|
636
|
+
return { failure }
|
|
637
|
+
})
|
|
638
|
+
)
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* The instance's boot, started by the first event that needs it and booted again once a failed boot's backoff has
|
|
643
|
+
* passed; concurrent events share one boot and one retry. A boot that found a migration pending is not kept, so the
|
|
644
|
+
* next event looks again.
|
|
645
|
+
*/
|
|
646
|
+
async #current(policy: MigrationPolicy): Promise<Outcome> {
|
|
647
|
+
if (this.#outcome === null) {
|
|
648
|
+
this.#outcome = this.#boot(policy)
|
|
649
|
+
}
|
|
650
|
+
const pending = this.#outcome
|
|
651
|
+
const outcome = await pending
|
|
652
|
+
if ('failure' in outcome && Date.now() >= this.#retryAt) {
|
|
653
|
+
if (this.#outcome === pending) {
|
|
654
|
+
this.#outcome = this.#boot(policy)
|
|
655
|
+
}
|
|
656
|
+
return this.#settled(this.#outcome, await this.#outcome)
|
|
657
|
+
}
|
|
658
|
+
return this.#settled(pending, outcome)
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/** Forgets a boot that found a migration pending, so the next event boots again. */
|
|
662
|
+
#settled(boot: Promise<Outcome>, outcome: Outcome): Outcome {
|
|
663
|
+
if ('pending' in outcome && this.#outcome === boot) {
|
|
664
|
+
this.#outcome = null
|
|
665
|
+
}
|
|
666
|
+
return outcome
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
#serve(booted: Booted, request: Request): Promise<Response> {
|
|
670
|
+
const scheduler = booted.app.jobs.scheduler
|
|
671
|
+
const tick = request.headers.get(TICK_HEADER)
|
|
672
|
+
if (tick !== null) {
|
|
673
|
+
return TICK_TIME.test(tick) && Number.isSafeInteger(Number(tick)) ? this.#tick(scheduler) : Promise.resolve(new Response(null, { status: 400 }))
|
|
674
|
+
}
|
|
675
|
+
if (scheduler !== null && !this.#repaired) {
|
|
676
|
+
this.#state.waitUntil(this.#repairOnce(scheduler))
|
|
677
|
+
}
|
|
678
|
+
const exec: RequestExecutionContext = { waitUntil: (promise) => this.#state.waitUntil(promise) }
|
|
679
|
+
return booted.app.fetch(request, exec)
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/**
|
|
683
|
+
* workerd calls it when the alarm is due (ADR-0065 §5). Only the object ordinary traffic reaches runs jobs, and only
|
|
684
|
+
* in `open`; otherwise it runs nothing and arms nothing: the maintenance operation owns the object, reopening arms the
|
|
685
|
+
* alarm again, and the repair tick arms the serving object's. It never throws, because after six failures the platform
|
|
686
|
+
* stops retrying: an internal error in `open` arms it again in `ALARM_RETRY_MS` and is reported.
|
|
687
|
+
*/
|
|
688
|
+
async alarm(): Promise<void> {
|
|
689
|
+
let scheduler: JobScheduler | null = null
|
|
690
|
+
let barrier: MaintenanceBarrier | null = null
|
|
691
|
+
try {
|
|
692
|
+
const object = this.#recorded()
|
|
693
|
+
if (object === null || object !== this.#object) {
|
|
694
|
+
// No verified event named this object yet, or it is not the one ordinary traffic reaches (a rehearsal, a
|
|
695
|
+
// former serving object after a deploy moved the traffic): it runs no job.
|
|
696
|
+
return
|
|
697
|
+
}
|
|
698
|
+
if (importing(this.#state.storage)) {
|
|
699
|
+
// A half-loaded import runs no job and arms nothing; the repair tick arms the alarm once the object boots.
|
|
700
|
+
return
|
|
701
|
+
}
|
|
702
|
+
barrier = await this.#maintenance(object)
|
|
703
|
+
if (barrier.mode !== 'open') {
|
|
704
|
+
return
|
|
705
|
+
}
|
|
706
|
+
const outcome = await this.#current('apply')
|
|
707
|
+
if (!('booted' in outcome)) {
|
|
708
|
+
throw new Error('the application did not boot')
|
|
709
|
+
}
|
|
710
|
+
const booted = outcome.booted.app.jobs.scheduler
|
|
711
|
+
scheduler = booted
|
|
712
|
+
if (booted === null) {
|
|
713
|
+
return
|
|
714
|
+
}
|
|
715
|
+
const event = barrier.event()
|
|
716
|
+
await objectScope.run({ storage: this.#state.storage, event }, () => runGated(event, () => this.#wake(booted, event)))
|
|
717
|
+
} catch (error) {
|
|
718
|
+
if (error instanceof MaintenanceFenceError || barrier === null || barrier.mode !== 'open') {
|
|
719
|
+
// A transition restricted the object meanwhile, or none was ever read: no failure, and nothing is armed outside open.
|
|
720
|
+
return
|
|
721
|
+
}
|
|
722
|
+
console.error('fabrika: the alarm failed: alarm_failed')
|
|
723
|
+
try {
|
|
724
|
+
await this.#state.storage.setAlarm?.(Date.now() + ALARM_RETRY_MS)
|
|
725
|
+
} catch {
|
|
726
|
+
console.error('fabrika: the alarm could not be armed again; the repair tick arms it')
|
|
727
|
+
}
|
|
728
|
+
if (scheduler !== null) {
|
|
729
|
+
this.#state.waitUntil(scheduler.failed('alarm', error))
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
/**
|
|
735
|
+
* One alarm: per job, in one transaction, a crashed holder reaped, the lease taken for a due wake (or the pending flag
|
|
736
|
+
* set for the run holding it) and the alarm armed from the projection. The runs go to `waitUntil`: one alarm runs at a
|
|
737
|
+
* time, and awaiting a long job would hold back every other wake. No wake is consumed here, only by its run's finish.
|
|
738
|
+
*/
|
|
739
|
+
async #wake(scheduler: JobScheduler, event: BarrierEvent): Promise<void> {
|
|
740
|
+
await this.#repairOnce(scheduler)
|
|
741
|
+
const early = await scheduler.earlyBy(ALARM_SLACK_MS)
|
|
742
|
+
if (early > 0) {
|
|
743
|
+
await this.#clock.wait(early, new AbortController().signal)
|
|
744
|
+
}
|
|
745
|
+
const storage = fencedStorage()
|
|
746
|
+
// A job whose lease is held past its expiry but which the contract no longer declares is reaped too.
|
|
747
|
+
for (const job of [...scheduler.jobs, ...(await scheduler.orphans())]) {
|
|
748
|
+
if (!event.writable()) {
|
|
749
|
+
// A transition restricted the object since the alarm was admitted; reopening arms it again.
|
|
750
|
+
return
|
|
751
|
+
}
|
|
752
|
+
const taken = await storage.transaction(async () => {
|
|
753
|
+
const taken = await scheduler.take(job)
|
|
754
|
+
await armAlarm(eventScope().storage, await scheduler.next())
|
|
755
|
+
return taken
|
|
756
|
+
})
|
|
757
|
+
if (taken.reaped) {
|
|
758
|
+
this.#state.waitUntil(scheduler.crashed(job))
|
|
759
|
+
}
|
|
760
|
+
if (taken.fence !== null) {
|
|
761
|
+
this.#state.waitUntil(runScheduled(scheduler, job, taken.fence, this.#clock))
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
/** The startup repair once per instance, at its first open event; a failure is reported and the next event repairs. */
|
|
767
|
+
async #repairOnce(scheduler: JobScheduler): Promise<void> {
|
|
768
|
+
if (this.#repaired) {
|
|
769
|
+
return
|
|
770
|
+
}
|
|
771
|
+
try {
|
|
772
|
+
this.#repaired = await scheduler.repair()
|
|
773
|
+
} catch (error) {
|
|
774
|
+
this.#repairFailed(scheduler, error)
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
/** A repair the write fence refused is no failure: a transition closed the object, and reopening repairs. */
|
|
779
|
+
#repairFailed(scheduler: JobScheduler, error: unknown): void {
|
|
780
|
+
if (error instanceof MaintenanceFenceError) {
|
|
781
|
+
return
|
|
782
|
+
}
|
|
783
|
+
console.error("fabrika: the job scheduler's repair failed: repair_failed")
|
|
784
|
+
this.#state.waitUntil(scheduler.failed('repair', error))
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/** The repair tick: the startup repair again, which also arms the first alarm of an environment nobody has called. */
|
|
788
|
+
async #tick(scheduler: JobScheduler | null): Promise<Response> {
|
|
789
|
+
if (scheduler === null) {
|
|
790
|
+
return new Response(null, { status: 204 })
|
|
791
|
+
}
|
|
792
|
+
try {
|
|
793
|
+
this.#repaired = (await scheduler.repair()) || this.#repaired
|
|
794
|
+
} catch (error) {
|
|
795
|
+
this.#repairFailed(scheduler, error)
|
|
796
|
+
return new Response(null, { status: error instanceof MaintenanceFenceError ? 503 : 500 })
|
|
797
|
+
}
|
|
798
|
+
return new Response(null, { status: 204 })
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/** After a transition that opened this object, its alarm fires at once and repairs: an alarm outside `open` armed nothing. */
|
|
802
|
+
async #reopened(): Promise<void> {
|
|
803
|
+
if (this.#barrier === null || this.#barrierObject !== this.#object) {
|
|
804
|
+
return
|
|
805
|
+
}
|
|
806
|
+
try {
|
|
807
|
+
if ((await this.#barrier).mode !== 'open') {
|
|
808
|
+
return
|
|
809
|
+
}
|
|
810
|
+
this.#repaired = false
|
|
811
|
+
await this.#state.storage.setAlarm?.(Date.now())
|
|
812
|
+
} catch {
|
|
813
|
+
console.error('fabrika: the alarm could not be armed after the object opened; the repair tick arms it')
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
/** The fixed-size digests are compared in constant time, so neither the length nor a prefix of the secret leaks. */
|
|
818
|
+
#fromWorker(request: Request): boolean {
|
|
819
|
+
const presented = request.headers.get(PROXY_SECRET_HEADER) ?? ''
|
|
820
|
+
return this.#secret !== null && presented !== '' && timingSafeEqual(proxySecretDigest(presented), this.#secret)
|
|
821
|
+
}
|
|
822
|
+
}
|