@fabrika/app 0.9.0-cloudflare.2 → 0.9.0-cloudflare.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -9
- package/skill/SKILL.md +20 -18
- package/skill/references/app-setup.md +29 -16
- package/skill/references/endpoint.md +24 -22
- package/skill/references/errors.md +3 -2
- package/skill/references/jobs.md +13 -3
- package/skill/references/tests.md +7 -0
- package/src/bun.ts +159 -6
- package/src/cloudflare/application.ts +543 -73
- package/src/cloudflare/export.ts +621 -0
- package/src/cloudflare/import.ts +1097 -0
- package/src/cloudflare/maintenance.ts +518 -0
- package/src/cloudflare/ops.ts +147 -18
- package/src/cloudflare/restore.ts +167 -0
- package/src/cloudflare/tracker.ts +192 -0
- package/src/cloudflare.ts +3 -2
- package/src/contract/artifact-schema.ts +7 -1
- package/src/contract/build.ts +6 -2
- package/src/contract/context.ts +26 -9
- package/src/contract/defaults.ts +22 -2
- package/src/contract/define.ts +1 -1
- package/src/contract/group.ts +1 -1
- package/src/files/kinds.ts +54 -24
- package/src/implement/anonymous.ts +1 -10
- package/src/implement/boot.ts +21 -9
- package/src/implement/dispatch.ts +171 -33
- package/src/implement/egress.ts +34 -8
- package/src/implement/invocation.ts +50 -10
- package/src/implement/workload.ts +18 -3
- package/src/jobs/runner.ts +639 -96
- package/src/testing/kit.ts +60 -0
- package/types/src/bun.d.ts +34 -1
- package/types/src/cloudflare/application.d.ts +36 -5
- package/types/src/cloudflare/export.d.ts +16 -0
- package/types/src/cloudflare/import.d.ts +58 -0
- package/types/src/cloudflare/maintenance.d.ts +117 -0
- package/types/src/cloudflare/ops.d.ts +39 -4
- package/types/src/cloudflare/restore.d.ts +9 -0
- package/types/src/cloudflare/tracker.d.ts +29 -0
- package/types/src/cloudflare.d.ts +2 -1
- package/types/src/contract/artifact-schema.d.ts +2 -0
- package/types/src/contract/context.d.ts +24 -9
- package/types/src/contract/defaults.d.ts +19 -1
- package/types/src/contract/define.d.ts +1 -1
- package/types/src/contract/group.d.ts +1 -1
- package/types/src/files/kinds.d.ts +17 -6
- package/types/src/implement/anonymous.d.ts +0 -4
- package/types/src/implement/boot.d.ts +11 -0
- package/types/src/implement/dispatch.d.ts +31 -0
- package/types/src/implement/invocation.d.ts +17 -7
- package/types/src/implement/workload.d.ts +5 -0
- package/types/src/jobs/runner.d.ts +60 -2
- package/types/src/testing/kit.d.ts +16 -0
|
@@ -1,27 +1,41 @@
|
|
|
1
|
-
// The Durable Object an application environment runs in (ADR-0065): it migrates its SQLite, boots the application
|
|
2
|
-
// serves every request and
|
|
3
|
-
// `cloudflare:workers`.
|
|
1
|
+
// The Durable Object an application environment runs in (ADR-0065): it migrates its SQLite, boots the application,
|
|
2
|
+
// serves every request and repair tick its Worker forwards, and runs the jobs from its alarm, which it arms from their
|
|
3
|
+
// durable wakes in the transaction of every change. A plain class, so Bun and Lopata load it without `cloudflare:workers`.
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
|
|
5
|
+
import {
|
|
6
|
+
FABRIKA_APPLICATION_OBJECT,
|
|
7
|
+
FABRIKA_APPLICATION_OBJECTS,
|
|
8
|
+
FABRIKA_CONTRACT_DIGEST,
|
|
9
|
+
FABRIKA_PROXY_SECRET,
|
|
10
|
+
OPS_PREFIX,
|
|
11
|
+
type OpsBootFailureContent,
|
|
12
|
+
type OpsMode,
|
|
13
|
+
parseApplicationObjects,
|
|
14
|
+
PROXY_SECRET_HEADER,
|
|
15
|
+
} from '@fabrika/auth-core'
|
|
16
|
+
import { isScopedDatabase, type LeaseFence, type Session } from '@fabrika/db/framework'
|
|
7
17
|
import { SQLITE_FRAMEWORK_MIGRATIONS } from '@fabrika/db/migrations-sqlite'
|
|
8
|
-
import { openSqlite, type SqliteStorage } from '@fabrika/db/sqlite'
|
|
9
|
-
import { migrateSqlite } from '@fabrika/db/sqlite-migrate'
|
|
18
|
+
import { openSqlite, type SqliteCursor, type SqliteStorage } from '@fabrika/db/sqlite'
|
|
19
|
+
import { migrateSqlite, type SqliteMigrationBundle, type SqliteMigrationTarget } from '@fabrika/db/sqlite-migrate'
|
|
10
20
|
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
11
21
|
import { timingSafeEqual } from 'node:crypto'
|
|
12
22
|
import type { RequestExecutionContext } from '../fetch.js'
|
|
23
|
+
import { FILE_TOMBSTONE_RETENTION_MS } from '../files/kinds.js'
|
|
13
24
|
import { boot, type ContractApp } from '../implement/boot.js'
|
|
14
25
|
import { type CrashEvents, environmentCrashReporter, reportSurvivedCrashes } from '../implement/crashes.js'
|
|
15
|
-
import { proxySecretDigest } from '../implement/dispatch.js'
|
|
26
|
+
import { MaintenanceFenceError, maintenanceRefusal, proxySecretDigest, runGated } from '../implement/dispatch.js'
|
|
16
27
|
import { type CapturedEnvironment, captureEnvironment } from '../implement/environment.js'
|
|
17
28
|
import type { ContractImplementation } from '../implement/implement.js'
|
|
18
29
|
import type { CrashReporter } from '../jobs/report.js'
|
|
19
|
-
import {
|
|
30
|
+
import { type JobClock, type JobScheduler, systemClock } from '../jobs/runner.js'
|
|
20
31
|
import { environmentScrubber, type Scrubber } from '../jobs/scrub.js'
|
|
21
32
|
import { bindingAssets, bindingKeySource, bindingMembership, bindingTerminals, bindingWorkloadTokens, connectBindings } from './bindings.js'
|
|
33
|
+
import { importInProgress } from './import.js'
|
|
34
|
+
import { type BarrierEvent, MaintenanceBarrier, r2MaintenanceStore } from './maintenance.js'
|
|
22
35
|
import { CLOUDFLARE_ASSETS_BINDING, CLOUDFLARE_FILES_BINDING } from './names.js'
|
|
23
|
-
import { type OpsAccess, opsAccess, type OpsStorage, serveOps } from './ops.js'
|
|
36
|
+
import { type OpsAccess, opsAccess, type OpsRecovery, type OpsStorage, serveOps } from './ops.js'
|
|
24
37
|
import { r2FileObjectStore } from './r2-files.js'
|
|
38
|
+
import { trackPending } from './tracker.js'
|
|
25
39
|
|
|
26
40
|
/** The application module, imported only after the environment was captured: the generated entrypoint's `import()`. */
|
|
27
41
|
export type ImplementationLoader = () => Promise<{ readonly default: ContractImplementation }>
|
|
@@ -39,22 +53,74 @@ export interface ContractWorkerOptions {
|
|
|
39
53
|
readonly rejections?: CrashEvents
|
|
40
54
|
/** What posts a report to Operations' ingest; `fetch` by default. */
|
|
41
55
|
readonly operationsFetch?: (request: Request) => Promise<Response>
|
|
56
|
+
/** The jobs' and the alarm's waits and an attempt's deadline; the system clock by default. */
|
|
57
|
+
readonly clock?: JobClock
|
|
42
58
|
}
|
|
43
59
|
|
|
44
60
|
/**
|
|
45
|
-
* Marks the Worker's
|
|
61
|
+
* Marks the Worker's repair tick, with its `scheduledTime`. The Worker's `fetch` removes it from every request it
|
|
46
62
|
* forwards, so only its own `scheduled` handler reaches the tick.
|
|
47
63
|
*/
|
|
48
64
|
export const TICK_HEADER = 'X-Fabrika-Tick'
|
|
49
65
|
|
|
50
|
-
/**
|
|
51
|
-
export
|
|
66
|
+
/** A wake runs at most this early: `Date.now()` is pinned between I/O, so an alarm may start a moment before it (I6). */
|
|
67
|
+
export const ALARM_SLACK_MS = 50
|
|
68
|
+
/** After an internal error the alarm is armed again this far ahead; the repair tick covers a re-arm that failed. */
|
|
69
|
+
export const ALARM_RETRY_MS = 30_000
|
|
70
|
+
/**
|
|
71
|
+
* The object's KV key that holds the name of the object it is, recorded at its first verified event: an alarm names no
|
|
72
|
+
* object, and the one ordinary traffic reaches may since have changed. Outside the SQLite, so no dump carries it.
|
|
73
|
+
*/
|
|
74
|
+
export const OBJECT_NAME_KEY = 'fabrika:object'
|
|
75
|
+
/** A repair tick's time: the Worker's `scheduledTime`, digits only. */
|
|
76
|
+
const TICK_TIME = /^\d{1,16}$/
|
|
77
|
+
|
|
78
|
+
/** The object's alarm. On a SQLite-backed object `setAlarm` and `deleteAlarm` commit with the open transaction. */
|
|
79
|
+
export interface AlarmStorage {
|
|
80
|
+
setAlarm(scheduledTime: number): Promise<void>
|
|
81
|
+
deleteAlarm(): Promise<void>
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* What the object needs of its `ctx.storage`; a Durable Object's `DurableObjectStorage` fits it. A test double without
|
|
86
|
+
* an alarm arms none.
|
|
87
|
+
*/
|
|
88
|
+
export type ApplicationStorage = SqliteStorage & OpsStorage & Partial<AlarmStorage>
|
|
52
89
|
|
|
53
90
|
/** What the object needs of its `DurableObjectState`. */
|
|
54
91
|
export interface ApplicationState {
|
|
55
|
-
readonly storage: ApplicationStorage
|
|
92
|
+
readonly storage: ApplicationStorage & Partial<Omit<OpsRecovery, 'abort'>>
|
|
56
93
|
blockConcurrencyWhile<T>(callback: () => Promise<T>): Promise<T>
|
|
57
94
|
waitUntil(promise: Promise<unknown>): void
|
|
95
|
+
/** workerd resets the object; the test double has none. */
|
|
96
|
+
abort?(reason?: string): void
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** `state` with every `waitUntil` promise recorded by the isolate's pending-work tracker (the ops operation `timers`). */
|
|
100
|
+
export function trackedState(state: ApplicationState): ApplicationState {
|
|
101
|
+
const { abort } = state
|
|
102
|
+
return {
|
|
103
|
+
storage: state.storage,
|
|
104
|
+
blockConcurrencyWhile: (callback) => state.blockConcurrencyWhile(callback),
|
|
105
|
+
waitUntil: (promise) => state.waitUntil(trackPending(promise)),
|
|
106
|
+
...(abort === undefined ? {} : { abort: (reason?: string) => abort.call(state, reason) }),
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The object's recovery when its storage and state have every member PITR needs (ADR-0070), else `null`. */
|
|
111
|
+
function objectRecovery(state: ApplicationState): OpsRecovery | null {
|
|
112
|
+
const { storage } = state
|
|
113
|
+
const { getCurrentBookmark, getBookmarkForTime, onNextSessionRestoreBookmark } = storage
|
|
114
|
+
const { abort } = state
|
|
115
|
+
if (getCurrentBookmark === undefined || getBookmarkForTime === undefined || onNextSessionRestoreBookmark === undefined || abort === undefined) {
|
|
116
|
+
return null
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
getCurrentBookmark: () => getCurrentBookmark.call(storage),
|
|
120
|
+
getBookmarkForTime: (time) => getBookmarkForTime.call(storage, time),
|
|
121
|
+
onNextSessionRestoreBookmark: (bookmark) => onNextSessionRestoreBookmark.call(storage, bookmark),
|
|
122
|
+
abort: (reason) => abort.call(state, reason),
|
|
123
|
+
}
|
|
58
124
|
}
|
|
59
125
|
|
|
60
126
|
/** The framework's bindings: IAM's application entrypoint and the Workers Assets of the SPA. No handler sees them. */
|
|
@@ -81,15 +147,120 @@ interface ApplicationSetup {
|
|
|
81
147
|
|
|
82
148
|
let registered: ApplicationSetup | null = null
|
|
83
149
|
|
|
84
|
-
/** The storage of the object whose event runs
|
|
85
|
-
|
|
150
|
+
/** The storage of the object whose event runs, and the barrier's view of that event; work it hands to `waitUntil` keeps both. */
|
|
151
|
+
interface ObjectScope {
|
|
152
|
+
readonly storage: ApplicationStorage
|
|
153
|
+
readonly event: BarrierEvent
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const objectScope = new AsyncLocalStorage<ObjectScope>()
|
|
157
|
+
|
|
158
|
+
/** The cursors of the fenced transaction that runs: its commit checks whether any of them wrote. */
|
|
159
|
+
const openTransaction = new AsyncLocalStorage<SqliteCursor[]>()
|
|
86
160
|
|
|
87
|
-
function
|
|
88
|
-
const
|
|
89
|
-
if (
|
|
161
|
+
function eventScope(): ObjectScope {
|
|
162
|
+
const scope = objectScope.getStore()
|
|
163
|
+
if (scope === undefined) {
|
|
90
164
|
throw new Error('a statement ran outside any event of an application object: the framework runs every event in its object scope')
|
|
91
165
|
}
|
|
92
|
-
return
|
|
166
|
+
return scope
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** The write fence (ADR-0069 §3): a write of an invocation admitted before a restrictive transition rolls back. */
|
|
170
|
+
function fence(event: BarrierEvent, cursors: readonly SqliteCursor[]): void {
|
|
171
|
+
if (cursors.some((cursor) => cursor.rowsWritten > 0) && !event.writable()) {
|
|
172
|
+
throw new MaintenanceFenceError()
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* The storage every framework and application write on SQLite reaches, bound to the object whose event runs. The
|
|
178
|
+
* fence runs inside each transaction before it commits; a statement outside one runs in a transaction of its own.
|
|
179
|
+
*/
|
|
180
|
+
function fencedStorage(): SqliteStorage {
|
|
181
|
+
return {
|
|
182
|
+
sql: {
|
|
183
|
+
exec(query, ...bindings) {
|
|
184
|
+
const { storage, event } = eventScope()
|
|
185
|
+
const cursors = openTransaction.getStore()
|
|
186
|
+
if (cursors !== undefined) {
|
|
187
|
+
const cursor = storage.sql.exec(query, ...bindings)
|
|
188
|
+
cursors.push(cursor)
|
|
189
|
+
return cursor
|
|
190
|
+
}
|
|
191
|
+
return storage.transactionSync(() => {
|
|
192
|
+
const cursor = storage.sql.exec(query, ...bindings)
|
|
193
|
+
const rows = cursor.toArray()
|
|
194
|
+
fence(event, [cursor])
|
|
195
|
+
return { toArray: () => rows, rowsWritten: cursor.rowsWritten }
|
|
196
|
+
})
|
|
197
|
+
},
|
|
198
|
+
},
|
|
199
|
+
transaction(closure) {
|
|
200
|
+
const { storage, event } = eventScope()
|
|
201
|
+
// A nested transaction shares its parent's cursors, so the outer commit checks every write inside it.
|
|
202
|
+
const cursors = openTransaction.getStore() ?? []
|
|
203
|
+
return storage.transaction(() =>
|
|
204
|
+
openTransaction.run(cursors, async () => {
|
|
205
|
+
const result = await closure()
|
|
206
|
+
fence(event, cursors)
|
|
207
|
+
return result
|
|
208
|
+
})
|
|
209
|
+
)
|
|
210
|
+
},
|
|
211
|
+
transactionSync(closure) {
|
|
212
|
+
const { storage, event } = eventScope()
|
|
213
|
+
const cursors = openTransaction.getStore() ?? []
|
|
214
|
+
return storage.transactionSync(() =>
|
|
215
|
+
openTransaction.run(cursors, () => {
|
|
216
|
+
const result = closure()
|
|
217
|
+
fence(event, cursors)
|
|
218
|
+
return result
|
|
219
|
+
})
|
|
220
|
+
)
|
|
221
|
+
},
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Whether migrations may be applied: in `open` and the migration phase; a restricted cold start only looks. */
|
|
226
|
+
type MigrationPolicy = 'apply' | 'refuse'
|
|
227
|
+
|
|
228
|
+
/** A restricted cold start found a migration to apply: the application does not boot until the operation's migrate. */
|
|
229
|
+
class MigrationsPending extends Error {
|
|
230
|
+
override readonly name = 'MigrationsPending'
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** The object holds an import marker (ADR-0070 §6): it neither boots nor migrates until the import completes or is discarded. */
|
|
234
|
+
class ImportInProgress extends Error {
|
|
235
|
+
override readonly name = 'ImportInProgress'
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Whether the object holds an import marker; a storage without the synchronous KV (the bun:sqlite double) holds none. */
|
|
239
|
+
function importing(storage: ApplicationStorage): boolean {
|
|
240
|
+
const { kv } = storage
|
|
241
|
+
return kv !== undefined && importInProgress({ kv })
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function isMigrationsPending(error: unknown): boolean {
|
|
245
|
+
return error instanceof MigrationsPending || (error instanceof Error && error.cause instanceof MigrationsPending)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* The migration runner's target that applies nothing: it answers the runner's reads, and the first write it would make
|
|
250
|
+
* (the ledger's DDL, a migration, a digest) throws `MigrationsPending` instead.
|
|
251
|
+
*/
|
|
252
|
+
function refusingTarget(storage: ApplicationStorage): SqliteMigrationTarget {
|
|
253
|
+
return {
|
|
254
|
+
exec(query, ...bindings) {
|
|
255
|
+
if (!/^\s*select\b/i.test(query)) {
|
|
256
|
+
throw new MigrationsPending('a migration is pending')
|
|
257
|
+
}
|
|
258
|
+
return storage.sql.exec(query, ...bindings)
|
|
259
|
+
},
|
|
260
|
+
transactionSync() {
|
|
261
|
+
throw new MigrationsPending('a migration is pending')
|
|
262
|
+
},
|
|
263
|
+
}
|
|
93
264
|
}
|
|
94
265
|
|
|
95
266
|
/** What the object classes of this isolate boot: the generated entrypoint's `createContractWorker` registers it. */
|
|
@@ -108,12 +279,23 @@ function globalRejectionEvents(): CrashEvents {
|
|
|
108
279
|
}
|
|
109
280
|
}
|
|
110
281
|
|
|
111
|
-
/**
|
|
112
|
-
|
|
113
|
-
|
|
282
|
+
/**
|
|
283
|
+
* The framework's tables, then the application's, each migration once: a booted storage applies nothing. Under
|
|
284
|
+
* `refuse` a pending migration throws `MigrationsPending` before anything is written.
|
|
285
|
+
*/
|
|
286
|
+
async function migrate(storage: ApplicationStorage, migrations: readonly ApplicationMigration[], policy: MigrationPolicy): Promise<void> {
|
|
287
|
+
const target: SqliteMigrationTarget = policy === 'apply'
|
|
288
|
+
? { exec: (query, ...bindings) => storage.sql.exec(query, ...bindings), transactionSync: (fn) => storage.transactionSync(fn) }
|
|
289
|
+
: refusingTarget(storage)
|
|
290
|
+
await migrateSqlite(target, migrationBundles(migrations))
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** The bundles the object applies, in order: the framework's tables, then the application's. */
|
|
294
|
+
function migrationBundles(migrations: readonly ApplicationMigration[]): readonly SqliteMigrationBundle[] {
|
|
295
|
+
return [
|
|
114
296
|
{ bundle: 'fabrika', migrations: SQLITE_FRAMEWORK_MIGRATIONS },
|
|
115
297
|
{ bundle: 'app', migrations },
|
|
116
|
-
]
|
|
298
|
+
]
|
|
117
299
|
}
|
|
118
300
|
|
|
119
301
|
async function bootApplication(
|
|
@@ -122,19 +304,14 @@ async function bootApplication(
|
|
|
122
304
|
env: Readonly<Record<string, unknown>>,
|
|
123
305
|
environment: CapturedEnvironment,
|
|
124
306
|
crashes: CrashReporter | null,
|
|
307
|
+
policy: MigrationPolicy,
|
|
125
308
|
): Promise<Booted> {
|
|
126
309
|
const { default: implementation } = await setup.load()
|
|
127
310
|
const migrates = isScopedDatabase(implementation.database)
|
|
128
311
|
if (migrates) {
|
|
129
|
-
await migrate(state.storage, setup.options.migrations ?? [])
|
|
312
|
+
await migrate(state.storage, setup.options.migrations ?? [], policy)
|
|
130
313
|
}
|
|
131
|
-
const database = migrates
|
|
132
|
-
? openSqlite({
|
|
133
|
-
sql: { exec: (query, ...bindings) => eventStorage().sql.exec(query, ...bindings) },
|
|
134
|
-
transaction: (closure) => eventStorage().transaction(closure),
|
|
135
|
-
transactionSync: (closure) => eventStorage().transactionSync(closure),
|
|
136
|
-
})
|
|
137
|
-
: undefined
|
|
314
|
+
const database = migrates ? openSqlite(fencedStorage()) : undefined
|
|
138
315
|
const app = boot(implementation, {
|
|
139
316
|
environment,
|
|
140
317
|
iam: bindingKeySource(env['IAM']),
|
|
@@ -145,10 +322,13 @@ async function bootApplication(
|
|
|
145
322
|
connectBindings: connectBindings(env),
|
|
146
323
|
database,
|
|
147
324
|
files: r2FileObjectStore(env[CLOUDFLARE_FILES_BINDING]),
|
|
325
|
+
// A removed file's object outlives its row for the window, so a side restore finds its bytes (ADR-0070).
|
|
326
|
+
fileRetentionMs: FILE_TOMBSTONE_RETENTION_MS,
|
|
148
327
|
// The Worker forwards only with the proxy secret, which the object checked; `boot` checks it again.
|
|
149
328
|
requireProxySecret: true,
|
|
150
329
|
// The proxy Worker sets `X-Fabrika-Client-Ip` from `CF-Connecting-IP`, the visitor Cloudflare saw.
|
|
151
330
|
clientAddress: 'vouched',
|
|
331
|
+
projectWakes: (transaction) => projectWakes(setup, transaction),
|
|
152
332
|
...(setup.options.operationsFetch === undefined ? {} : { operationsFetch: setup.options.operationsFetch }),
|
|
153
333
|
})
|
|
154
334
|
const booted: Booted = { app, migrates }
|
|
@@ -167,18 +347,26 @@ async function attach(
|
|
|
167
347
|
setup: ApplicationSetup | null,
|
|
168
348
|
state: ApplicationState,
|
|
169
349
|
env: Readonly<Record<string, unknown>>,
|
|
350
|
+
policy: MigrationPolicy,
|
|
170
351
|
failed: (failure: OpsBootFailureContent) => void,
|
|
171
352
|
): Promise<Booted> {
|
|
172
353
|
if (setup === null) {
|
|
173
354
|
throw new Error('no contract application is registered: the generated entrypoint calls createContractWorker before an object starts')
|
|
174
355
|
}
|
|
356
|
+
// Before anything loads or migrates, and no boot failure: a half-loaded import never boots.
|
|
357
|
+
if (importing(state.storage)) {
|
|
358
|
+
throw new ImportInProgress('import_in_progress')
|
|
359
|
+
}
|
|
175
360
|
const environment = captureEnvironment(Object.fromEntries(Object.entries(env).filter(([name]) => !FRAMEWORK_BINDINGS.has(name))))
|
|
176
361
|
const crashes = environmentCrashReporter(environment, setup.options.operationsFetch)
|
|
177
362
|
try {
|
|
178
|
-
const booted = await attachOnce(setup, state, env, environment, crashes)
|
|
363
|
+
const booted = await attachOnce(setup, state, env, environment, crashes, policy)
|
|
179
364
|
setup.failureReported = false
|
|
180
365
|
return booted
|
|
181
366
|
} catch (error) {
|
|
367
|
+
if (isMigrationsPending(error)) {
|
|
368
|
+
throw error
|
|
369
|
+
}
|
|
182
370
|
failed(bootFailure(error, environmentScrubber(environment)))
|
|
183
371
|
if (crashes !== null && !setup.failureReported) {
|
|
184
372
|
setup.failureReported = true
|
|
@@ -194,9 +382,10 @@ async function attachOnce(
|
|
|
194
382
|
env: Readonly<Record<string, unknown>>,
|
|
195
383
|
environment: CapturedEnvironment,
|
|
196
384
|
crashes: CrashReporter | null,
|
|
385
|
+
policy: MigrationPolicy,
|
|
197
386
|
): Promise<Booted> {
|
|
198
387
|
if (setup.booted === null) {
|
|
199
|
-
const booting = bootApplication(setup, state, env, environment, crashes)
|
|
388
|
+
const booting = bootApplication(setup, state, env, environment, crashes, policy)
|
|
200
389
|
setup.booted = booting
|
|
201
390
|
// A failed boot is not kept: the next object instance boots again and fails with the same problems.
|
|
202
391
|
booting.catch(() => {
|
|
@@ -206,7 +395,7 @@ async function attachOnce(
|
|
|
206
395
|
}
|
|
207
396
|
const booted = await setup.booted
|
|
208
397
|
if (booted.migrates) {
|
|
209
|
-
await migrate(state.storage, setup.options.migrations ?? [])
|
|
398
|
+
await migrate(state.storage, setup.options.migrations ?? [], policy)
|
|
210
399
|
}
|
|
211
400
|
return booted
|
|
212
401
|
}
|
|
@@ -221,18 +410,44 @@ function bootFailure(error: unknown, scrubber: Scrubber): OpsBootFailureContent
|
|
|
221
410
|
return { at, message, stack: null }
|
|
222
411
|
}
|
|
223
412
|
|
|
224
|
-
|
|
413
|
+
/**
|
|
414
|
+
* `pending`: a restricted cold start found a migration to apply, or the object holds an import marker, so the
|
|
415
|
+
* application did not boot; that is no failure.
|
|
416
|
+
*/
|
|
417
|
+
type Outcome = { readonly booted: Booted } | { readonly failure: unknown } | { readonly pending: true }
|
|
225
418
|
|
|
226
419
|
/** The wait after an instance's first failed boot before an event boots again; it doubles with each failure, to the cap. */
|
|
227
420
|
const BOOT_RETRY_MS = 1_000
|
|
228
421
|
const BOOT_RETRY_CAP_MS = 60_000
|
|
229
422
|
|
|
230
|
-
/**
|
|
231
|
-
async function
|
|
423
|
+
/** The alarm armed at `next`, or deleted when nothing is pending; inside a transaction it commits with it. */
|
|
424
|
+
async function armAlarm(storage: ApplicationStorage, next: Date | null): Promise<void> {
|
|
425
|
+
if (next === null) {
|
|
426
|
+
await storage.deleteAlarm?.()
|
|
427
|
+
} else {
|
|
428
|
+
await storage.setAlarm?.(next.getTime())
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* The runtime's projection of the jobs' wakes and leases (ADR-0065 §5, I1): the store runs it in the transaction of
|
|
434
|
+
* every change, and it arms the alarm of the object whose event runs from `next`. A throw rolls the change back.
|
|
435
|
+
*/
|
|
436
|
+
async function projectWakes(setup: ApplicationSetup, transaction: Session): Promise<void> {
|
|
437
|
+
// A wake or lease changes only once the application booted.
|
|
438
|
+
const booted = setup.booted === null ? null : await setup.booted
|
|
439
|
+
const scheduler = booted?.app.jobs.scheduler ?? null
|
|
440
|
+
if (scheduler !== null) {
|
|
441
|
+
await armAlarm(eventScope().storage, await scheduler.next(transaction))
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/** A run the alarm took the lease for, handed to `waitUntil`; the runner logs and reports its own failures. */
|
|
446
|
+
async function runScheduled(scheduler: JobScheduler, job: string, fence: LeaseFence, clock: JobClock): Promise<void> {
|
|
232
447
|
try {
|
|
233
|
-
await
|
|
448
|
+
await scheduler.run(job, fence, { clock })
|
|
234
449
|
} catch {
|
|
235
|
-
console.error(`fabrika: job '${
|
|
450
|
+
console.error(`fabrika: job '${job}' did not start: driver_failed`)
|
|
236
451
|
}
|
|
237
452
|
}
|
|
238
453
|
|
|
@@ -245,21 +460,41 @@ export class FabrikaApplication {
|
|
|
245
460
|
readonly #secret: Uint8Array | null
|
|
246
461
|
/** What the platform wrote for Control's ops channel; `null` refuses every ops request. */
|
|
247
462
|
readonly #ops: OpsAccess | null
|
|
248
|
-
/** The
|
|
463
|
+
/** The object ordinary traffic reaches, so the one this instance is when such traffic arrives; `null` when malformed. */
|
|
464
|
+
readonly #object: string | null
|
|
465
|
+
/** The object this instance is, once a verified event named it or its storage recorded it (`OBJECT_NAME_KEY`). */
|
|
466
|
+
#name: string | null = null
|
|
467
|
+
/** The maintenance barrier, read from R2 once per instance; a failed read is not kept, so the next request reads again. */
|
|
468
|
+
#barrier: Promise<MaintenanceBarrier> | null = null
|
|
469
|
+
#barrierObject: string | null = null
|
|
470
|
+
/** The instance's failed boot, kept in memory for the ops view until a boot succeeds. */
|
|
249
471
|
#bootFailure: OpsBootFailureContent | null = null
|
|
250
472
|
/** The instance's boot, `null` until the first event that needs the application: an ops request never starts it. */
|
|
251
473
|
#outcome: Promise<Outcome> | null = null
|
|
252
474
|
#failedBoots = 0
|
|
253
475
|
/** When an event may boot again after a failed boot; until then every request answers the failure. */
|
|
254
476
|
#retryAt = 0
|
|
477
|
+
/** Whether this instance ran the scheduler's startup repair, which its first open event does. */
|
|
478
|
+
#repaired = false
|
|
479
|
+
readonly #clock: JobClock
|
|
255
480
|
|
|
256
481
|
constructor(state: ApplicationState, env: Readonly<Record<string, unknown>>) {
|
|
257
|
-
this.#state = state
|
|
482
|
+
this.#state = trackedState(state)
|
|
258
483
|
this.#env = env
|
|
259
484
|
this.#setup = registered
|
|
485
|
+
this.#clock = registered?.options.clock ?? systemClock
|
|
260
486
|
const secret = env[FABRIKA_PROXY_SECRET]
|
|
261
487
|
this.#secret = typeof secret === 'string' && secret !== '' ? proxySecretDigest(secret) : null
|
|
262
488
|
this.#ops = opsAccess(env)
|
|
489
|
+
const object = env[FABRIKA_APPLICATION_OBJECT]
|
|
490
|
+
const objects = env[FABRIKA_APPLICATION_OBJECTS]
|
|
491
|
+
this.#object = parseApplicationObjects(typeof object === 'string' ? object : undefined, typeof objects === 'string' ? objects : undefined)
|
|
492
|
+
?.object ?? null
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
#contractDigest(): string | null {
|
|
496
|
+
const digest = this.#env[FABRIKA_CONTRACT_DIGEST]
|
|
497
|
+
return typeof digest === 'string' ? digest : null
|
|
263
498
|
}
|
|
264
499
|
|
|
265
500
|
async fetch(request: Request): Promise<Response> {
|
|
@@ -268,28 +503,134 @@ export class FabrikaApplication {
|
|
|
268
503
|
return new Response(null, { status: 404 })
|
|
269
504
|
}
|
|
270
505
|
if (new URL(request.url).pathname.startsWith(OPS_PREFIX)) {
|
|
506
|
+
// The mode before the request: a transition that opens the object arms its alarm, which repairs.
|
|
507
|
+
let before: OpsMode | null = null
|
|
271
508
|
// Served from storage alone: it never starts a boot or a retry, and waits for one under way only once authenticated.
|
|
272
|
-
|
|
509
|
+
const response = await serveOps(request, this.#ops, {
|
|
273
510
|
storage: this.#state.storage,
|
|
511
|
+
bucket: this.#env[CLOUDFLARE_FILES_BINDING],
|
|
512
|
+
recovery: objectRecovery(this.#state),
|
|
513
|
+
contractDigest: this.#contractDigest(),
|
|
514
|
+
migrations: migrationBundles(this.#setup?.options.migrations ?? []),
|
|
274
515
|
bootFailure: async () => {
|
|
275
516
|
await this.#outcome
|
|
276
517
|
return this.#bootFailure
|
|
277
518
|
},
|
|
519
|
+
maintenance: async (object) => {
|
|
520
|
+
const barrier = await this.#maintenance(object)
|
|
521
|
+
before ??= barrier.mode
|
|
522
|
+
return barrier
|
|
523
|
+
},
|
|
524
|
+
migrate: () => this.#migrated(),
|
|
278
525
|
})
|
|
526
|
+
if (before !== null && before !== 'open') {
|
|
527
|
+
await this.#reopened()
|
|
528
|
+
}
|
|
529
|
+
return response
|
|
530
|
+
}
|
|
531
|
+
let barrier: MaintenanceBarrier
|
|
532
|
+
try {
|
|
533
|
+
barrier = await this.#maintenance(this.#object)
|
|
534
|
+
} catch {
|
|
535
|
+
// Fails closed: a state R2 could not give is never read as `open` (ADR-0069 §3, cold starts).
|
|
536
|
+
return maintenanceRefusal()
|
|
537
|
+
}
|
|
538
|
+
if (importing(this.#state.storage)) {
|
|
539
|
+
// ADR-0070 §6: whatever the mode, a half-loaded import serves nothing; the tick repairs nothing.
|
|
540
|
+
return request.headers.get(TICK_HEADER) === null ? maintenanceRefusal() : new Response(null, { status: 204 })
|
|
541
|
+
}
|
|
542
|
+
if (barrier.mode !== 'open' && request.headers.get(TICK_HEADER) !== null) {
|
|
543
|
+
// The tick repairs nothing and writes nothing until the object opens.
|
|
544
|
+
return new Response(null, { status: 204 })
|
|
545
|
+
}
|
|
546
|
+
if (barrier.mode === 'closed' || barrier.mode === 'inspect') {
|
|
547
|
+
return maintenanceRefusal()
|
|
548
|
+
}
|
|
549
|
+
const outcome = await this.#current(barrier.mode === 'open' ? 'apply' : 'refuse')
|
|
550
|
+
if ('pending' in outcome) {
|
|
551
|
+
return maintenanceRefusal()
|
|
279
552
|
}
|
|
280
|
-
const outcome = await this.#current()
|
|
281
553
|
if ('failure' in outcome) {
|
|
282
554
|
throw outcome.failure
|
|
283
555
|
}
|
|
284
|
-
|
|
556
|
+
const event = barrier.event()
|
|
557
|
+
return await objectScope.run({ storage: this.#state.storage, event }, () => runGated(event, () => this.#serve(outcome.booted, request)))
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** The name of the object this instance is: the one its storage recorded, or `null` before a verified event named it. */
|
|
561
|
+
#recorded(): string | null {
|
|
562
|
+
if (this.#name === null) {
|
|
563
|
+
const stored = this.#state.storage.kv?.get(OBJECT_NAME_KEY)
|
|
564
|
+
if (typeof stored === 'string' && stored !== '') {
|
|
565
|
+
this.#name = stored
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
return this.#name
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* Whether this instance is the object a verified event names; the first such event records the name, which never
|
|
573
|
+
* changes after. A storage without KV (the bun:sqlite double) keeps it in memory only.
|
|
574
|
+
*/
|
|
575
|
+
#named(object: string): boolean {
|
|
576
|
+
const recorded = this.#recorded()
|
|
577
|
+
if (recorded !== null) {
|
|
578
|
+
return recorded === object
|
|
579
|
+
}
|
|
580
|
+
this.#state.storage.kv?.put(OBJECT_NAME_KEY, object)
|
|
581
|
+
this.#name = object
|
|
582
|
+
return true
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/** The barrier of the object this instance is; every name it is asked under must be the same one. */
|
|
586
|
+
#maintenance(object: string | null): Promise<MaintenanceBarrier> {
|
|
587
|
+
if (object === null) {
|
|
588
|
+
return Promise.reject(new Error(`${FABRIKA_APPLICATION_OBJECT} and ${FABRIKA_APPLICATION_OBJECTS} name no valid set of objects`))
|
|
589
|
+
}
|
|
590
|
+
if (!this.#named(object)) {
|
|
591
|
+
return Promise.reject(new Error('this object is another object than the one the event names'))
|
|
592
|
+
}
|
|
593
|
+
if (this.#barrier !== null && this.#barrierObject !== object) {
|
|
594
|
+
return Promise.reject(new Error('one object instance was asked under two object names'))
|
|
595
|
+
}
|
|
596
|
+
if (this.#barrier === null) {
|
|
597
|
+
const reading = (async () => await MaintenanceBarrier.load(r2MaintenanceStore(this.#env[CLOUDFLARE_FILES_BINDING], object)))()
|
|
598
|
+
this.#barrier = reading
|
|
599
|
+
this.#barrierObject = object
|
|
600
|
+
reading.catch(() => {
|
|
601
|
+
if (this.#barrier === reading) {
|
|
602
|
+
this.#barrier = null
|
|
603
|
+
this.#barrierObject = null
|
|
604
|
+
}
|
|
605
|
+
})
|
|
606
|
+
}
|
|
607
|
+
return this.#barrier
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/** The migration phase's boot: applies the pending migrations whatever the mode; `false` when the boot failed. */
|
|
611
|
+
async #migrated(): Promise<boolean> {
|
|
612
|
+
const current = this.#outcome === null ? null : await this.#outcome
|
|
613
|
+
if (current !== null && 'booted' in current) {
|
|
614
|
+
return true
|
|
615
|
+
}
|
|
616
|
+
const outcome = this.#boot('apply')
|
|
617
|
+
this.#outcome = outcome
|
|
618
|
+
return 'booted' in this.#settled(outcome, await outcome)
|
|
285
619
|
}
|
|
286
620
|
|
|
287
621
|
/** Settles either way: a callback that rejects would reset the object, and every request answers the failure instead. */
|
|
288
|
-
#boot(): Promise<Outcome> {
|
|
622
|
+
#boot(policy: MigrationPolicy): Promise<Outcome> {
|
|
289
623
|
return this.#state.blockConcurrencyWhile(() =>
|
|
290
|
-
attach(this.#setup, this.#state, this.#env, (failure) => {
|
|
624
|
+
attach(this.#setup, this.#state, this.#env, policy, (failure) => {
|
|
291
625
|
this.#bootFailure = failure
|
|
292
|
-
}).then((booted): Outcome =>
|
|
626
|
+
}).then((booted): Outcome => {
|
|
627
|
+
// The view reports the current boot: a failure an earlier attempt left is no longer the object's state.
|
|
628
|
+
this.#bootFailure = null
|
|
629
|
+
return { booted }
|
|
630
|
+
}, (failure: unknown): Outcome => {
|
|
631
|
+
if (isMigrationsPending(failure) || failure instanceof ImportInProgress) {
|
|
632
|
+
return { pending: true }
|
|
633
|
+
}
|
|
293
634
|
this.#failedBoots++
|
|
294
635
|
this.#retryAt = Date.now() + Math.min(BOOT_RETRY_MS * 2 ** (this.#failedBoots - 1), BOOT_RETRY_CAP_MS)
|
|
295
636
|
return { failure }
|
|
@@ -299,54 +640,183 @@ export class FabrikaApplication {
|
|
|
299
640
|
|
|
300
641
|
/**
|
|
301
642
|
* The instance's boot, started by the first event that needs it and booted again once a failed boot's backoff has
|
|
302
|
-
* passed; concurrent events share one boot and one retry.
|
|
643
|
+
* passed; concurrent events share one boot and one retry. A boot that found a migration pending is not kept, so the
|
|
644
|
+
* next event looks again.
|
|
303
645
|
*/
|
|
304
|
-
async #current(): Promise<Outcome> {
|
|
646
|
+
async #current(policy: MigrationPolicy): Promise<Outcome> {
|
|
305
647
|
if (this.#outcome === null) {
|
|
306
|
-
this.#outcome = this.#boot()
|
|
648
|
+
this.#outcome = this.#boot(policy)
|
|
307
649
|
}
|
|
308
650
|
const pending = this.#outcome
|
|
309
651
|
const outcome = await pending
|
|
310
|
-
if (
|
|
311
|
-
|
|
652
|
+
if ('failure' in outcome && Date.now() >= this.#retryAt) {
|
|
653
|
+
if (this.#outcome === pending) {
|
|
654
|
+
this.#outcome = this.#boot(policy)
|
|
655
|
+
}
|
|
656
|
+
return this.#settled(this.#outcome, await this.#outcome)
|
|
312
657
|
}
|
|
313
|
-
|
|
314
|
-
|
|
658
|
+
return this.#settled(pending, outcome)
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/** Forgets a boot that found a migration pending, so the next event boots again. */
|
|
662
|
+
#settled(boot: Promise<Outcome>, outcome: Outcome): Outcome {
|
|
663
|
+
if ('pending' in outcome && this.#outcome === boot) {
|
|
664
|
+
this.#outcome = null
|
|
315
665
|
}
|
|
316
|
-
return
|
|
666
|
+
return outcome
|
|
317
667
|
}
|
|
318
668
|
|
|
319
669
|
#serve(booted: Booted, request: Request): Promise<Response> {
|
|
670
|
+
const scheduler = booted.app.jobs.scheduler
|
|
320
671
|
const tick = request.headers.get(TICK_HEADER)
|
|
321
672
|
if (tick !== null) {
|
|
322
|
-
return
|
|
673
|
+
return TICK_TIME.test(tick) && Number.isSafeInteger(Number(tick)) ? this.#tick(scheduler) : Promise.resolve(new Response(null, { status: 400 }))
|
|
674
|
+
}
|
|
675
|
+
if (scheduler !== null && !this.#repaired) {
|
|
676
|
+
this.#state.waitUntil(this.#repairOnce(scheduler))
|
|
323
677
|
}
|
|
324
678
|
const exec: RequestExecutionContext = { waitUntil: (promise) => this.#state.waitUntil(promise) }
|
|
325
679
|
return booted.app.fetch(request, exec)
|
|
326
680
|
}
|
|
327
681
|
|
|
328
|
-
/**
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
682
|
+
/**
|
|
683
|
+
* workerd calls it when the alarm is due (ADR-0065 §5). Only the object ordinary traffic reaches runs jobs, and only
|
|
684
|
+
* in `open`; otherwise it runs nothing and arms nothing: the maintenance operation owns the object, reopening arms the
|
|
685
|
+
* alarm again, and the repair tick arms the serving object's. It never throws, because after six failures the platform
|
|
686
|
+
* stops retrying: an internal error in `open` arms it again in `ALARM_RETRY_MS` and is reported.
|
|
687
|
+
*/
|
|
688
|
+
async alarm(): Promise<void> {
|
|
689
|
+
let scheduler: JobScheduler | null = null
|
|
690
|
+
let barrier: MaintenanceBarrier | null = null
|
|
691
|
+
try {
|
|
692
|
+
const object = this.#recorded()
|
|
693
|
+
if (object === null || object !== this.#object) {
|
|
694
|
+
// No verified event named this object yet, or it is not the one ordinary traffic reaches (a rehearsal, a
|
|
695
|
+
// former serving object after a deploy moved the traffic): it runs no job.
|
|
696
|
+
return
|
|
697
|
+
}
|
|
698
|
+
if (importing(this.#state.storage)) {
|
|
699
|
+
// A half-loaded import runs no job and arms nothing; the repair tick arms the alarm once the object boots.
|
|
700
|
+
return
|
|
701
|
+
}
|
|
702
|
+
barrier = await this.#maintenance(object)
|
|
703
|
+
if (barrier.mode !== 'open') {
|
|
704
|
+
return
|
|
705
|
+
}
|
|
706
|
+
const outcome = await this.#current('apply')
|
|
707
|
+
if (!('booted' in outcome)) {
|
|
708
|
+
throw new Error('the application did not boot')
|
|
709
|
+
}
|
|
710
|
+
const booted = outcome.booted.app.jobs.scheduler
|
|
711
|
+
scheduler = booted
|
|
712
|
+
if (booted === null) {
|
|
713
|
+
return
|
|
714
|
+
}
|
|
715
|
+
const event = barrier.event()
|
|
716
|
+
await objectScope.run({ storage: this.#state.storage, event }, () => runGated(event, () => this.#wake(booted, event)))
|
|
717
|
+
} catch (error) {
|
|
718
|
+
if (error instanceof MaintenanceFenceError || barrier === null || barrier.mode !== 'open') {
|
|
719
|
+
// A transition restricted the object meanwhile, or none was ever read: no failure, and nothing is armed outside open.
|
|
720
|
+
return
|
|
721
|
+
}
|
|
722
|
+
console.error('fabrika: the alarm failed: alarm_failed')
|
|
723
|
+
try {
|
|
724
|
+
await this.#state.storage.setAlarm?.(Date.now() + ALARM_RETRY_MS)
|
|
725
|
+
} catch {
|
|
726
|
+
console.error('fabrika: the alarm could not be armed again; the repair tick arms it')
|
|
727
|
+
}
|
|
728
|
+
if (scheduler !== null) {
|
|
729
|
+
this.#state.waitUntil(scheduler.failed('alarm', error))
|
|
730
|
+
}
|
|
731
|
+
}
|
|
332
732
|
}
|
|
333
733
|
|
|
334
|
-
/**
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
734
|
+
/**
|
|
735
|
+
* One alarm: per job, in one transaction, a crashed holder reaped, the lease taken for a due wake (or the pending flag
|
|
736
|
+
* set for the run holding it) and the alarm armed from the projection. The runs go to `waitUntil`: one alarm runs at a
|
|
737
|
+
* time, and awaiting a long job would hold back every other wake. No wake is consumed here, only by its run's finish.
|
|
738
|
+
*/
|
|
739
|
+
async #wake(scheduler: JobScheduler, event: BarrierEvent): Promise<void> {
|
|
740
|
+
await this.#repairOnce(scheduler)
|
|
741
|
+
const early = await scheduler.earlyBy(ALARM_SLACK_MS)
|
|
742
|
+
if (early > 0) {
|
|
743
|
+
await this.#clock.wait(early, new AbortController().signal)
|
|
339
744
|
}
|
|
340
|
-
const
|
|
341
|
-
|
|
342
|
-
for (const job of jobs) {
|
|
343
|
-
if (
|
|
344
|
-
|
|
745
|
+
const storage = fencedStorage()
|
|
746
|
+
// A job whose lease is held past its expiry but which the contract no longer declares is reaped too.
|
|
747
|
+
for (const job of [...scheduler.jobs, ...(await scheduler.orphans())]) {
|
|
748
|
+
if (!event.writable()) {
|
|
749
|
+
// A transition restricted the object since the alarm was admitted; reopening arms it again.
|
|
750
|
+
return
|
|
751
|
+
}
|
|
752
|
+
const taken = await storage.transaction(async () => {
|
|
753
|
+
const taken = await scheduler.take(job)
|
|
754
|
+
await armAlarm(eventScope().storage, await scheduler.next())
|
|
755
|
+
return taken
|
|
756
|
+
})
|
|
757
|
+
if (taken.reaped) {
|
|
758
|
+
this.#state.waitUntil(scheduler.crashed(job))
|
|
759
|
+
}
|
|
760
|
+
if (taken.fence !== null) {
|
|
761
|
+
this.#state.waitUntil(runScheduled(scheduler, job, taken.fence, this.#clock))
|
|
345
762
|
}
|
|
346
763
|
}
|
|
347
|
-
|
|
348
|
-
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
/** The startup repair once per instance, at its first open event; a failure is reported and the next event repairs. */
|
|
767
|
+
async #repairOnce(scheduler: JobScheduler): Promise<void> {
|
|
768
|
+
if (this.#repaired) {
|
|
769
|
+
return
|
|
770
|
+
}
|
|
771
|
+
try {
|
|
772
|
+
this.#repaired = await scheduler.repair()
|
|
773
|
+
} catch (error) {
|
|
774
|
+
this.#repairFailed(scheduler, error)
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
/** A repair the write fence refused is no failure: a transition closed the object, and reopening repairs. */
|
|
779
|
+
#repairFailed(scheduler: JobScheduler, error: unknown): void {
|
|
780
|
+
if (error instanceof MaintenanceFenceError) {
|
|
781
|
+
return
|
|
782
|
+
}
|
|
783
|
+
console.error("fabrika: the job scheduler's repair failed: repair_failed")
|
|
784
|
+
this.#state.waitUntil(scheduler.failed('repair', error))
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
/** The repair tick: the startup repair again, which also arms the first alarm of an environment nobody has called. */
|
|
788
|
+
async #tick(scheduler: JobScheduler | null): Promise<Response> {
|
|
789
|
+
if (scheduler === null) {
|
|
790
|
+
return new Response(null, { status: 204 })
|
|
791
|
+
}
|
|
792
|
+
try {
|
|
793
|
+
this.#repaired = (await scheduler.repair()) || this.#repaired
|
|
794
|
+
} catch (error) {
|
|
795
|
+
this.#repairFailed(scheduler, error)
|
|
796
|
+
return new Response(null, { status: error instanceof MaintenanceFenceError ? 503 : 500 })
|
|
349
797
|
}
|
|
350
798
|
return new Response(null, { status: 204 })
|
|
351
799
|
}
|
|
800
|
+
|
|
801
|
+
/** After a transition that opened this object, its alarm fires at once and repairs: an alarm outside `open` armed nothing. */
|
|
802
|
+
async #reopened(): Promise<void> {
|
|
803
|
+
if (this.#barrier === null || this.#barrierObject !== this.#object) {
|
|
804
|
+
return
|
|
805
|
+
}
|
|
806
|
+
try {
|
|
807
|
+
if ((await this.#barrier).mode !== 'open') {
|
|
808
|
+
return
|
|
809
|
+
}
|
|
810
|
+
this.#repaired = false
|
|
811
|
+
await this.#state.storage.setAlarm?.(Date.now())
|
|
812
|
+
} catch {
|
|
813
|
+
console.error('fabrika: the alarm could not be armed after the object opened; the repair tick arms it')
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
/** The fixed-size digests are compared in constant time, so neither the length nor a prefix of the secret leaks. */
|
|
818
|
+
#fromWorker(request: Request): boolean {
|
|
819
|
+
const presented = request.headers.get(PROXY_SECRET_HEADER) ?? ''
|
|
820
|
+
return this.#secret !== null && presented !== '' && timingSafeEqual(proxySecretDigest(presented), this.#secret)
|
|
821
|
+
}
|
|
352
822
|
}
|