@fabrika/app 0.9.0-cloudflare.1 → 0.9.0-cloudflare.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -9
- package/skill/SKILL.md +21 -17
- package/skill/references/app-setup.md +42 -11
- package/skill/references/endpoint.md +29 -22
- package/skill/references/errors.md +32 -31
- package/skill/references/jobs.md +22 -3
- package/skill/references/tests.md +29 -1
- package/src/build.ts +6 -1
- package/src/bun.ts +168 -5
- package/src/cloudflare/application.ts +546 -76
- package/src/cloudflare/bindings.ts +26 -5
- package/src/cloudflare/export.ts +621 -0
- package/src/cloudflare/import.ts +1097 -0
- package/src/cloudflare/maintenance.ts +518 -0
- package/src/cloudflare/names.ts +6 -0
- package/src/cloudflare/ops.ts +147 -18
- package/src/cloudflare/restore.ts +167 -0
- package/src/cloudflare/tracker.ts +192 -0
- package/src/cloudflare.ts +3 -2
- package/src/contract/artifact-schema.ts +13 -2
- package/src/contract/build.ts +38 -11
- package/src/contract/context.ts +26 -9
- package/src/contract/defaults.ts +22 -2
- package/src/contract/define.ts +1 -1
- package/src/contract/group.ts +10 -1
- package/src/contract/kit.ts +2 -1
- package/src/disk-assets.ts +8 -3
- package/src/files/kinds.ts +54 -24
- package/src/implement/anonymous.ts +1 -10
- package/src/implement/boot.ts +30 -12
- package/src/implement/capability.ts +3 -0
- package/src/implement/dispatch.ts +175 -37
- package/src/implement/egress.ts +367 -8
- package/src/implement/invocation.ts +60 -19
- package/src/implement/workload.ts +18 -3
- package/src/jobs/runner.ts +646 -98
- package/src/spa-fallback.ts +12 -0
- package/src/testing/endpoint-access.ts +135 -23
- package/src/testing/isolation.ts +8 -1
- package/src/testing/kit.ts +60 -0
- package/types/src/build.d.ts +1 -1
- package/types/src/bun.d.ts +34 -1
- package/types/src/cloudflare/application.d.ts +36 -5
- package/types/src/cloudflare/bindings.d.ts +6 -0
- package/types/src/cloudflare/export.d.ts +16 -0
- package/types/src/cloudflare/import.d.ts +58 -0
- package/types/src/cloudflare/maintenance.d.ts +117 -0
- package/types/src/cloudflare/names.d.ts +5 -0
- package/types/src/cloudflare/ops.d.ts +39 -4
- package/types/src/cloudflare/restore.d.ts +9 -0
- package/types/src/cloudflare/tracker.d.ts +29 -0
- package/types/src/cloudflare.d.ts +2 -1
- package/types/src/contract/artifact-schema.d.ts +2 -0
- package/types/src/contract/context.d.ts +24 -9
- package/types/src/contract/defaults.d.ts +19 -1
- package/types/src/contract/define.d.ts +1 -1
- package/types/src/contract/group.d.ts +9 -1
- package/types/src/contract/kit.d.ts +2 -2
- package/types/src/disk-assets.d.ts +4 -1
- package/types/src/files/kinds.d.ts +17 -6
- package/types/src/implement/anonymous.d.ts +0 -4
- package/types/src/implement/boot.d.ts +17 -0
- package/types/src/implement/capability.d.ts +2 -0
- package/types/src/implement/dispatch.d.ts +33 -1
- package/types/src/implement/egress.d.ts +36 -1
- package/types/src/implement/invocation.d.ts +20 -10
- package/types/src/implement/workload.d.ts +5 -0
- package/types/src/jobs/runner.d.ts +63 -2
- package/types/src/spa-fallback.d.ts +2 -0
- package/types/src/testing/endpoint-access.d.ts +25 -1
- package/types/src/testing/isolation.d.ts +1 -1
- package/types/src/testing/kit.d.ts +16 -0
package/src/cloudflare/ops.ts
CHANGED
|
@@ -9,31 +9,47 @@ import {
|
|
|
9
9
|
FABRIKA_APPLICATION_OBJECTS,
|
|
10
10
|
FABRIKA_OPS_KEY_DIGESTS,
|
|
11
11
|
formatEndpointHeader,
|
|
12
|
+
OPS_DATA_REFUSED_STATUS,
|
|
12
13
|
OPS_METHOD,
|
|
13
14
|
OPS_ROW_COUNT_CAP,
|
|
15
|
+
OPS_STALE_STATUS,
|
|
14
16
|
type OpsBootFailureContent,
|
|
15
17
|
type OpsContentView,
|
|
18
|
+
type OpsDataRefusal,
|
|
16
19
|
opsKeyDigest,
|
|
17
20
|
type OpsOperation,
|
|
21
|
+
type OpsPublishedState,
|
|
18
22
|
opsRefusal,
|
|
23
|
+
type OpsStaleRefusal,
|
|
19
24
|
type OpsView,
|
|
20
25
|
parseApplicationObjects,
|
|
21
26
|
parseOpsContentView,
|
|
27
|
+
parseOpsExportRequest,
|
|
28
|
+
parseOpsImportRequest,
|
|
22
29
|
parseOpsInspectRequest,
|
|
23
30
|
parseOpsKeyDigests,
|
|
31
|
+
parseOpsMigrateAck,
|
|
24
32
|
parseOpsMigrateRequest,
|
|
25
33
|
parseOpsOperation,
|
|
34
|
+
parseOpsRestoreRequest,
|
|
26
35
|
parseOpsStateRequest,
|
|
36
|
+
parseOpsTimersRequest,
|
|
27
37
|
parseOpsView,
|
|
28
38
|
} from '@fabrika/auth-core'
|
|
29
39
|
import { FILES_TABLE, JOB_LEASES_TABLE, WEBHOOK_BINDINGS_TABLE, WEBHOOK_DELIVERIES_TABLE } from '@fabrika/db/migrations'
|
|
30
|
-
import type { SqliteCursor, SqliteValue } from '@fabrika/db/sqlite'
|
|
40
|
+
import type { SqliteCursor, SqliteStorage, SqliteValue } from '@fabrika/db/sqlite'
|
|
41
|
+
import type { SqliteMigrationBundle } from '@fabrika/db/sqlite-migrate'
|
|
31
42
|
import { opsProxyRow } from '@fabrika/proxy-core/manifest'
|
|
32
43
|
import { timingSafeEqual } from 'node:crypto'
|
|
33
44
|
import appPackage from '../../package.json' with { type: 'json' }
|
|
34
45
|
import { jsonResponse } from '../errors.js'
|
|
46
|
+
import { serveExport } from './export.js'
|
|
47
|
+
import { importInProgress, serveImport } from './import.js'
|
|
48
|
+
import type { MaintenanceBarrier, TransitionResult } from './maintenance.js'
|
|
49
|
+
import { serveRestore } from './restore.js'
|
|
50
|
+
import { serveTimers } from './tracker.js'
|
|
35
51
|
|
|
36
|
-
/** What the view reads of the object's storage; workerd's `DurableObjectStorage` has
|
|
52
|
+
/** What the view reads of the object's storage; workerd's `DurableObjectStorage` has every optional member. */
|
|
37
53
|
export interface OpsStorage {
|
|
38
54
|
readonly sql: {
|
|
39
55
|
exec(query: string, ...bindings: SqliteValue[]): SqliteCursor
|
|
@@ -42,6 +58,28 @@ export interface OpsStorage {
|
|
|
42
58
|
}
|
|
43
59
|
/** The bun:sqlite test double has no alarm. */
|
|
44
60
|
getAlarm?(): Promise<number | null>
|
|
61
|
+
/** The object's KV storage, which point-in-time recovery rewinds with its SQLite; the bun:sqlite double has none. */
|
|
62
|
+
get?(key: string): Promise<unknown>
|
|
63
|
+
put?(key: string, value: unknown): Promise<void>
|
|
64
|
+
/** The synchronous KV of a SQLite-backed object, in the same database; the bun:sqlite test double has none. */
|
|
65
|
+
readonly kv?: {
|
|
66
|
+
get(key: string): unknown
|
|
67
|
+
put(key: string, value: unknown): void
|
|
68
|
+
delete(key: string): boolean
|
|
69
|
+
}
|
|
70
|
+
/** Wipes the object's database, its KV and its alarm; the bun:sqlite test double has none. */
|
|
71
|
+
deleteAll?(): Promise<void>
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The object's point-in-time recovery: workerd's storage bookmarks and the abort that resets the object. `null` where the
|
|
76
|
+
* storage has none (the bun:sqlite double, a runtime without PITR).
|
|
77
|
+
*/
|
|
78
|
+
export interface OpsRecovery {
|
|
79
|
+
getCurrentBookmark(): Promise<string>
|
|
80
|
+
getBookmarkForTime(time: number): Promise<string>
|
|
81
|
+
onNextSessionRestoreBookmark(bookmark: string): Promise<string>
|
|
82
|
+
abort(reason: string): void
|
|
45
83
|
}
|
|
46
84
|
|
|
47
85
|
const OPS_ROW = opsProxyRow()
|
|
@@ -82,17 +120,29 @@ export function opsAccess(env: Readonly<Record<string, unknown>>): OpsAccess | n
|
|
|
82
120
|
|
|
83
121
|
/** What an ops request reads of the object it reached. */
|
|
84
122
|
export interface OpsTarget {
|
|
85
|
-
readonly storage: OpsStorage
|
|
123
|
+
readonly storage: OpsStorage & SqliteStorage
|
|
124
|
+
/** The environment's bucket (the `FABRIKA_FILES` R2 binding), where dumps and the maintenance state live. */
|
|
125
|
+
readonly bucket: unknown
|
|
126
|
+
readonly recovery: OpsRecovery | null
|
|
127
|
+
/** The digest of the contract this release was deployed with (`FABRIKA_CONTRACT_DIGEST`); a dump's manifest records it. */
|
|
128
|
+
readonly contractDigest: string | null
|
|
129
|
+
/** The release's migration bundles in the order the object applies them: the framework's, then the application's. */
|
|
130
|
+
readonly migrations: readonly SqliteMigrationBundle[]
|
|
86
131
|
/**
|
|
87
132
|
* The instance's last failed boot, the application's error text included. Asked only after authentication: it
|
|
88
133
|
* waits for a boot under way and never starts one.
|
|
89
134
|
*/
|
|
90
135
|
bootFailure(): Promise<OpsBootFailureContent | null>
|
|
136
|
+
/** The barrier of the object named `object`, read from R2 on a cold start; rejects while R2 cannot be read. */
|
|
137
|
+
maintenance(object: string): Promise<MaintenanceBarrier>
|
|
138
|
+
/** Runs the pending migrations and boots the application; `false` when the boot failed. */
|
|
139
|
+
migrate(): Promise<boolean>
|
|
91
140
|
}
|
|
92
141
|
|
|
93
142
|
/**
|
|
94
143
|
* Answers an ops request the Worker forwarded with the proxy secret, which the object checked before. Authentication
|
|
95
|
-
* runs first; an unknown operation, a malformed body and an object name the list lacks are refused after it.
|
|
144
|
+
* runs first; an unknown operation, a malformed body and an object name the list lacks are refused after it. Every
|
|
145
|
+
* operation needs the barrier, so while R2 cannot be read each answers 503.
|
|
96
146
|
*/
|
|
97
147
|
export async function serveOps(request: Request, access: OpsAccess | null, target: OpsTarget): Promise<Response> {
|
|
98
148
|
if (access === null || !opsAuthenticated(request, access)) {
|
|
@@ -103,19 +153,87 @@ export async function serveOps(request: Request, access: OpsAccess | null, targe
|
|
|
103
153
|
return opsRefusal()
|
|
104
154
|
}
|
|
105
155
|
const body = await jsonBody(request)
|
|
106
|
-
if (operation === 'inspect') {
|
|
107
|
-
const inspect = parseOpsInspectRequest(body)
|
|
108
|
-
if (inspect === null || !access.objects.objects.includes(inspect.object)) {
|
|
109
|
-
return opsRefusal()
|
|
110
|
-
}
|
|
111
|
-
return jsonResponse(await inspectView(target, inspect.content), 200)
|
|
112
|
-
}
|
|
113
156
|
const object = objectNamed(operation, body)
|
|
114
157
|
if (object === null || !access.objects.objects.includes(object)) {
|
|
115
158
|
return opsRefusal()
|
|
116
159
|
}
|
|
117
|
-
|
|
118
|
-
|
|
160
|
+
let barrier: MaintenanceBarrier
|
|
161
|
+
try {
|
|
162
|
+
barrier = await target.maintenance(object)
|
|
163
|
+
} catch {
|
|
164
|
+
return new Response(null, { status: 503 })
|
|
165
|
+
}
|
|
166
|
+
switch (operation) {
|
|
167
|
+
case 'inspect': {
|
|
168
|
+
const inspect = parseOpsInspectRequest(body)
|
|
169
|
+
return inspect === null ? opsRefusal() : jsonResponse(await inspectView(target, barrier, inspect.content), 200)
|
|
170
|
+
}
|
|
171
|
+
case 'state': {
|
|
172
|
+
const transition = parseOpsStateRequest(body)
|
|
173
|
+
const { kv } = target.storage
|
|
174
|
+
// ADR-0070 §6: an object that holds an import marker never leaves `inspect`.
|
|
175
|
+
const frozen = () => kv !== undefined && importInProgress({ kv })
|
|
176
|
+
return transition === null ? opsRefusal() : transitionAnswer(await barrier.transition(transition, frozen))
|
|
177
|
+
}
|
|
178
|
+
case 'migrate': {
|
|
179
|
+
const migration = parseOpsMigrateRequest(body)
|
|
180
|
+
return migration === null ? opsRefusal() : await migrateAnswer(target, barrier, migration.generation)
|
|
181
|
+
}
|
|
182
|
+
case 'export': {
|
|
183
|
+
const step = parseOpsExportRequest(body)
|
|
184
|
+
return step === null ? opsRefusal() : await serveExport(target, barrier, step)
|
|
185
|
+
}
|
|
186
|
+
case 'import': {
|
|
187
|
+
const step = parseOpsImportRequest(body)
|
|
188
|
+
return step === null ? opsRefusal() : await serveImport(target, barrier, step, access.objects.object)
|
|
189
|
+
}
|
|
190
|
+
case 'restore': {
|
|
191
|
+
const step = parseOpsRestoreRequest(body)
|
|
192
|
+
return step === null ? opsRefusal() : await serveRestore(target, barrier, step, object === access.objects.object)
|
|
193
|
+
}
|
|
194
|
+
case 'timers': {
|
|
195
|
+
const timers = parseOpsTimersRequest(body)
|
|
196
|
+
return timers === null ? opsRefusal() : serveTimers(timers)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** A transition's answer: the acknowledgement, the 409 naming what is published, or a 503 that Control retries. */
|
|
202
|
+
function transitionAnswer(result: TransitionResult): Response {
|
|
203
|
+
switch (result.kind) {
|
|
204
|
+
case 'acknowledged':
|
|
205
|
+
return jsonResponse(result.ack, 200)
|
|
206
|
+
case 'stale':
|
|
207
|
+
return staleRefusal(result.published)
|
|
208
|
+
case 'failed':
|
|
209
|
+
return new Response(null, { status: 503 })
|
|
210
|
+
case 'frozen': {
|
|
211
|
+
const refusal: OpsDataRefusal = { refused: 'wrong_mode', table: null }
|
|
212
|
+
return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Runs the migrations under the published generation and acknowledges with the new ledger; a failed boot is a 500. */
|
|
218
|
+
async function migrateAnswer(target: OpsTarget, barrier: MaintenanceBarrier, generation: number): Promise<Response> {
|
|
219
|
+
const result = await barrier.migrate(generation, () => target.migrate())
|
|
220
|
+
if (result.kind === 'stale') {
|
|
221
|
+
return staleRefusal(result.published)
|
|
222
|
+
}
|
|
223
|
+
if (!result.value) {
|
|
224
|
+
return new Response(null, { status: 500 })
|
|
225
|
+
}
|
|
226
|
+
const migrations = tableNames(target.storage).includes(MIGRATION_LEDGER) ? migrationLedger(target.storage) : []
|
|
227
|
+
const ack = parseOpsMigrateAck({ generation, migrations })
|
|
228
|
+
if (ack === null) {
|
|
229
|
+
throw new Error("the object's migration ledger does not parse as the wire's acknowledgement")
|
|
230
|
+
}
|
|
231
|
+
return jsonResponse(ack, 200)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function staleRefusal(published: OpsPublishedState | null): Response {
|
|
235
|
+
const refusal: OpsStaleRefusal = { refused: 'stale_generation', published }
|
|
236
|
+
return jsonResponse(refusal, OPS_STALE_STATUS)
|
|
119
237
|
}
|
|
120
238
|
|
|
121
239
|
/** The object an ops request names, read with its operation's strict parser; `null` for a body that does not parse. */
|
|
@@ -127,6 +245,14 @@ export function objectNamed(operation: OpsOperation, body: unknown): string | nu
|
|
|
127
245
|
return parseOpsStateRequest(body)?.object ?? null
|
|
128
246
|
case 'migrate':
|
|
129
247
|
return parseOpsMigrateRequest(body)?.object ?? null
|
|
248
|
+
case 'export':
|
|
249
|
+
return parseOpsExportRequest(body)?.object ?? null
|
|
250
|
+
case 'import':
|
|
251
|
+
return parseOpsImportRequest(body)?.object ?? null
|
|
252
|
+
case 'restore':
|
|
253
|
+
return parseOpsRestoreRequest(body)?.object ?? null
|
|
254
|
+
case 'timers':
|
|
255
|
+
return parseOpsTimersRequest(body)?.object ?? null
|
|
130
256
|
}
|
|
131
257
|
}
|
|
132
258
|
|
|
@@ -167,7 +293,7 @@ export function opsAuthenticated(request: Request, access: OpsAccess): boolean {
|
|
|
167
293
|
* The view of the object's storage, read and never written. A table the framework has not created yet (a boot that
|
|
168
294
|
* failed before its migrations) reads as empty. What the storage holds is parsed by the wire's own parser.
|
|
169
295
|
*/
|
|
170
|
-
async function inspectView(target: OpsTarget, content: boolean): Promise<OpsView | OpsContentView> {
|
|
296
|
+
async function inspectView(target: OpsTarget, barrier: MaintenanceBarrier, content: boolean): Promise<OpsView | OpsContentView> {
|
|
171
297
|
const { storage } = target
|
|
172
298
|
const bootFailure = await target.bootFailure()
|
|
173
299
|
const tables = tableNames(storage)
|
|
@@ -175,14 +301,12 @@ async function inspectView(target: OpsTarget, content: boolean): Promise<OpsView
|
|
|
175
301
|
const alarm = storage.getAlarm === undefined ? missing('getAlarm') : await storage.getAlarm()
|
|
176
302
|
const view = {
|
|
177
303
|
release: appPackage.version,
|
|
178
|
-
maintenance:
|
|
304
|
+
maintenance: barrier.view(),
|
|
179
305
|
database: {
|
|
180
306
|
sizeBytes: storage.sql.databaseSize ?? missing('sql.databaseSize'),
|
|
181
307
|
tables: tables.filter(admittedTableName).map((name) => ({ name, rows: cappedCount(storage, name) })),
|
|
182
308
|
},
|
|
183
|
-
migrations: has(MIGRATION_LEDGER)
|
|
184
|
-
? rows(storage, `select bundle, name, checksum, applied_at as appliedAt from ${MIGRATION_LEDGER} order by bundle, name`)
|
|
185
|
-
: [],
|
|
309
|
+
migrations: has(MIGRATION_LEDGER) ? migrationLedger(storage) : [],
|
|
186
310
|
scheduler: { alarm: alarm === null ? null : new Date(alarm).toISOString() },
|
|
187
311
|
jobs: has(JOB_LEASES_TABLE)
|
|
188
312
|
? rows(
|
|
@@ -211,6 +335,11 @@ async function inspectView(target: OpsTarget, content: boolean): Promise<OpsView
|
|
|
211
335
|
return parsed
|
|
212
336
|
}
|
|
213
337
|
|
|
338
|
+
/** The ledger `@fabrika/db/sqlite-migrate` keeps, as the wire names its rows; read only once the table exists. */
|
|
339
|
+
function migrationLedger(storage: OpsStorage): Record<string, SqliteValue>[] {
|
|
340
|
+
return rows(storage, `select bundle, name, checksum, applied_at as appliedAt from ${MIGRATION_LEDGER} order by bundle, name`)
|
|
341
|
+
}
|
|
342
|
+
|
|
214
343
|
/** Cloudflare's own `_cf_*` tables and SQLite's `sqlite_*` are no table of the application or the framework. */
|
|
215
344
|
function tableNames(storage: OpsStorage): string[] {
|
|
216
345
|
return storage.sql.exec(
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// One step of the original object's point-in-time restore (ADR-0070): the ops operation `restore`, served before the application boots.
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
OPS_DATA_REFUSED_STATUS,
|
|
5
|
+
OPS_STALE_STATUS,
|
|
6
|
+
type OpsDataRefusal,
|
|
7
|
+
type OpsDataRefusalReason,
|
|
8
|
+
type OpsMode,
|
|
9
|
+
type OpsRestoreArm,
|
|
10
|
+
type OpsRestorePrepare,
|
|
11
|
+
type OpsRestoreRequest,
|
|
12
|
+
type OpsRestoreVerify,
|
|
13
|
+
type OpsStaleRefusal,
|
|
14
|
+
parseOpsRestoreArmAck,
|
|
15
|
+
parseOpsRestorePrepareAck,
|
|
16
|
+
parseOpsRestoreVerifyAck,
|
|
17
|
+
} from '@fabrika/auth-core'
|
|
18
|
+
import { jsonResponse } from '../errors.js'
|
|
19
|
+
import type { MaintenanceBarrier, TurnStep } from './maintenance.js'
|
|
20
|
+
import type { OpsRecovery, OpsStorage, OpsTarget } from './ops.js'
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The KV key prefix of an operation's sentinel, followed by its id. KV, unlike R2, is rewound with the object's SQLite,
|
|
24
|
+
* so the absence of an operation's key proves its restore; no operation writes another's.
|
|
25
|
+
*/
|
|
26
|
+
const SENTINEL_PREFIX = '__fabrika/restore-sentinel/'
|
|
27
|
+
|
|
28
|
+
const ABORT_REASON = 'fabrika: point-in-time restore armed'
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* One step of the original object's point-in-time restore (ADR-0070 §9 step 6). `prepare` and `arm` run in the
|
|
32
|
+
* barrier's turn, only under the published generation, on an object that is not the one ordinary traffic reaches
|
|
33
|
+
* (`serving`), while it is `inspect`; `verify` writes nothing and runs in any mode.
|
|
34
|
+
*/
|
|
35
|
+
export async function serveRestore(target: OpsTarget, barrier: MaintenanceBarrier, step: OpsRestoreRequest, serving: boolean): Promise<Response> {
|
|
36
|
+
const { recovery } = target
|
|
37
|
+
if (recovery === null) {
|
|
38
|
+
return jsonResponse({ refused: 'not_implemented' }, 501)
|
|
39
|
+
}
|
|
40
|
+
const sentinels = kvSentinels(target.storage)
|
|
41
|
+
switch (step.step) {
|
|
42
|
+
case 'prepare':
|
|
43
|
+
return await inTurn(barrier, step.generation, serving, () => prepare(recovery, sentinels, step))
|
|
44
|
+
case 'arm':
|
|
45
|
+
return await inTurn(barrier, step.generation, serving, async () => {
|
|
46
|
+
await barrier.restrictedWrite()
|
|
47
|
+
return await arm(recovery, step)
|
|
48
|
+
})
|
|
49
|
+
case 'verify':
|
|
50
|
+
return await verify(recovery, sentinels, step)
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The first prepare of an operation reads the current bookmark, then the target's, and keeps both under its sentinel
|
|
56
|
+
* key; a retried one answers what it kept. A target not lexically before the current bookmark would keep the sentinel.
|
|
57
|
+
*/
|
|
58
|
+
async function prepare(recovery: OpsRecovery, sentinels: SentinelStore, step: OpsRestorePrepare): Promise<TurnStep<Response>> {
|
|
59
|
+
const kept = await sentinels.read(step.sentinel)
|
|
60
|
+
if (kept !== undefined) {
|
|
61
|
+
return answered(answer(parseOpsRestorePrepareAck(kept)))
|
|
62
|
+
}
|
|
63
|
+
const current = await recovery.getCurrentBookmark()
|
|
64
|
+
let target: string
|
|
65
|
+
try {
|
|
66
|
+
target = await recovery.getBookmarkForTime(Date.parse(step.at))
|
|
67
|
+
} catch {
|
|
68
|
+
return answered(dataRefusal('bookmark_unavailable'))
|
|
69
|
+
}
|
|
70
|
+
if (!(target < current)) {
|
|
71
|
+
return answered(dataRefusal('bookmark_unavailable'))
|
|
72
|
+
}
|
|
73
|
+
const ack = parseOpsRestorePrepareAck({ step: 'prepare', current, target })
|
|
74
|
+
if (ack !== null) {
|
|
75
|
+
await sentinels.write(step.sentinel, ack)
|
|
76
|
+
}
|
|
77
|
+
return answered(answer(ack))
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Arms the restore and resets the object once the answer is out, holding the barrier's turn until then. */
|
|
81
|
+
async function arm(recovery: OpsRecovery, step: OpsRestoreArm): Promise<TurnStep<Response>> {
|
|
82
|
+
let undo: string
|
|
83
|
+
try {
|
|
84
|
+
undo = await recovery.onNextSessionRestoreBookmark(step.target)
|
|
85
|
+
} catch {
|
|
86
|
+
return answered(dataRefusal('bookmark_unavailable'))
|
|
87
|
+
}
|
|
88
|
+
const response = answer(parseOpsRestoreArmAck({ step: 'arm', undo }))
|
|
89
|
+
// workerd's abort throws, so on a real object the turn is never released and no transition is acknowledged after it.
|
|
90
|
+
const reset = new Promise<void>((resolve) =>
|
|
91
|
+
setTimeout(() => {
|
|
92
|
+
recovery.abort(ABORT_REASON)
|
|
93
|
+
resolve()
|
|
94
|
+
}, 0)
|
|
95
|
+
)
|
|
96
|
+
return { value: response, holdUntil: reset }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Present while the KV holds this operation's key: a restore to a target before its `prepare` removed it. */
|
|
100
|
+
async function verify(recovery: OpsRecovery, sentinels: SentinelStore, step: OpsRestoreVerify): Promise<Response> {
|
|
101
|
+
const present = (await sentinels.read(step.sentinel)) !== undefined
|
|
102
|
+
return answer(parseOpsRestoreVerifyAck({ step: 'verify', sentinel: present ? 'present' : 'absent', bookmark: await recovery.getCurrentBookmark() }))
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* ADR-0070 §1 and §9: inside the barrier's turn, ADR-0069's stale refusal unless `generation` is the published one,
|
|
107
|
+
* then `serving_object` on the object of ordinary traffic, then `wrong_mode` unless `inspect`.
|
|
108
|
+
*/
|
|
109
|
+
async function inTurn(
|
|
110
|
+
barrier: MaintenanceBarrier,
|
|
111
|
+
generation: number,
|
|
112
|
+
serving: boolean,
|
|
113
|
+
work: () => Promise<TurnStep<Response>>,
|
|
114
|
+
): Promise<Response> {
|
|
115
|
+
const result = await barrier.step(generation, (mode: OpsMode) => {
|
|
116
|
+
if (serving) {
|
|
117
|
+
return Promise.resolve(answered(dataRefusal('serving_object')))
|
|
118
|
+
}
|
|
119
|
+
return mode === 'inspect' ? work() : Promise.resolve(answered(dataRefusal('wrong_mode')))
|
|
120
|
+
})
|
|
121
|
+
if (result.kind === 'ran') {
|
|
122
|
+
return result.value
|
|
123
|
+
}
|
|
124
|
+
const refusal: OpsStaleRefusal = { refused: 'stale_generation', published: result.published }
|
|
125
|
+
return jsonResponse(refusal, OPS_STALE_STATUS)
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function answered(response: Response): TurnStep<Response> {
|
|
129
|
+
return { value: response, holdUntil: null }
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function dataRefusal(reason: OpsDataRefusalReason): Response {
|
|
133
|
+
const refusal: OpsDataRefusal = { refused: reason, table: null }
|
|
134
|
+
return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** An answer the wire's own parser accepted; a bookmark the storage handed out that it refuses fails the step. */
|
|
138
|
+
function answer(parsed: object | null): Response {
|
|
139
|
+
if (parsed === null) {
|
|
140
|
+
throw new Error("the object's restore answer does not parse as the wire's acknowledgement")
|
|
141
|
+
}
|
|
142
|
+
return jsonResponse(parsed, 200)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** The operations' sentinels in the object's KV storage, each under its own key. */
|
|
146
|
+
interface SentinelStore {
|
|
147
|
+
/** What the operation's key holds, `undefined` when it holds nothing. */
|
|
148
|
+
read(sentinel: string): Promise<unknown>
|
|
149
|
+
write(sentinel: string, value: unknown): Promise<void>
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function kvSentinels(storage: OpsStorage): SentinelStore {
|
|
153
|
+
return {
|
|
154
|
+
async read(sentinel) {
|
|
155
|
+
if (storage.get === undefined) {
|
|
156
|
+
throw new Error("the object's storage has no get; a Durable Object's storage has it")
|
|
157
|
+
}
|
|
158
|
+
return await storage.get(`${SENTINEL_PREFIX}${sentinel}`)
|
|
159
|
+
},
|
|
160
|
+
async write(sentinel, value) {
|
|
161
|
+
if (storage.put === undefined) {
|
|
162
|
+
throw new Error("the object's storage has no put; a Durable Object's storage has it")
|
|
163
|
+
}
|
|
164
|
+
await storage.put(`${SENTINEL_PREFIX}${sentinel}`, value)
|
|
165
|
+
},
|
|
166
|
+
}
|
|
167
|
+
}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
// The pending-work tracker (ADR-0069 §2, ADR-0070 §10): the ops operation `timers`, diagnostics only. While it is on, it
|
|
2
|
+
// wraps the isolate's global timers and records each live timer and each `waitUntil` promise with the stack that
|
|
3
|
+
// registered it, scrubbed and bounded. Its scope is the isolate, not one object: every object the isolate hosts shares
|
|
4
|
+
// the switch and the records, and only the isolate's eviction resets them, so the next isolate's tracker starts off.
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
OPS_TIMER_STACK_MAX_BYTES,
|
|
8
|
+
OPS_TIMER_STACK_MAX_FRAMES,
|
|
9
|
+
OPS_TIMERS_CAP,
|
|
10
|
+
type OpsTimer,
|
|
11
|
+
type OpsTimersRequest,
|
|
12
|
+
type OpsTimersView,
|
|
13
|
+
} from '@fabrika/auth-core'
|
|
14
|
+
import { jsonResponse } from '../errors.js'
|
|
15
|
+
import { type Scrubber, secretScrubber } from '../jobs/scrub.js'
|
|
16
|
+
|
|
17
|
+
/** The global timers the tracker wraps while it is on; `globalThis` fits it. */
|
|
18
|
+
export interface TimerScope {
|
|
19
|
+
setTimeout(handler: unknown, delay?: unknown, ...args: unknown[]): unknown
|
|
20
|
+
setInterval(handler: unknown, delay?: unknown, ...args: unknown[]): unknown
|
|
21
|
+
clearTimeout(id?: unknown): void
|
|
22
|
+
clearInterval(id?: unknown): void
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface TimerTracker {
|
|
26
|
+
/** Switches tracking on (wrapping the scope's timers) or off (restoring them and forgetting every record). */
|
|
27
|
+
track(on: boolean): void
|
|
28
|
+
/**
|
|
29
|
+
* While tracking is on, records `promise` until it settles and answers a chain that settles as it does, rejection
|
|
30
|
+
* included, so the caller hands the chain to `waitUntil` and an unhandled rejection stays unhandled. Off, `promise`.
|
|
31
|
+
*/
|
|
32
|
+
pending<T>(promise: Promise<T>): Promise<T>
|
|
33
|
+
/** The live records, oldest first, at most `OPS_TIMERS_CAP` of them. */
|
|
34
|
+
view(): OpsTimersView
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
type Caller = (...args: never[]) => unknown
|
|
38
|
+
|
|
39
|
+
/** A tracker over `scope`'s timers, off until `track(true)`; each stack is scrubbed by `scrubber`. */
|
|
40
|
+
export function timerTracker(scope: TimerScope, now: () => number, scrubber: Scrubber): TimerTracker {
|
|
41
|
+
// Keyed by the timer's handle (`handleKey`), or by a fresh object for a promise; a Map keeps creation order.
|
|
42
|
+
const live = new Map<unknown, OpsTimer>()
|
|
43
|
+
/** The scope's own timers and the tracker's wrappers over them, while tracking is on. */
|
|
44
|
+
let installed: { readonly own: TimerScope; readonly wrapped: TimerScope } | null = null
|
|
45
|
+
|
|
46
|
+
const record = (kind: OpsTimer['kind'], delay: unknown, caller: Caller): OpsTimer => ({
|
|
47
|
+
kind,
|
|
48
|
+
since: new Date(now()).toISOString(),
|
|
49
|
+
delayMs: kind === 'waitUntil' ? null : delayOf(delay),
|
|
50
|
+
stack: stackBelow(caller, scrubber),
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
// A wrapper that other code kept after `track(false)` still works; it records only while tracking is on.
|
|
54
|
+
const wrap = (own: TimerScope): TimerScope => {
|
|
55
|
+
/** The record of a timer about to start, taken in the wrapper's own frame; `null` when nothing is recorded. */
|
|
56
|
+
const recorded = (kind: 'timeout' | 'interval', handler: unknown, delay: unknown, caller: Caller): OpsTimer | null =>
|
|
57
|
+
installed === null || typeof handler !== 'function' ? null : record(kind, delay, caller)
|
|
58
|
+
const schedule = (kind: 'timeout' | 'interval', entry: OpsTimer | null, handler: unknown, delay: unknown, args: readonly unknown[]): unknown => {
|
|
59
|
+
const start = kind === 'timeout' ? own.setTimeout : own.setInterval
|
|
60
|
+
if (entry === null || typeof handler !== 'function') {
|
|
61
|
+
return Reflect.apply(start, scope, [handler, delay, ...args])
|
|
62
|
+
}
|
|
63
|
+
let id: unknown
|
|
64
|
+
// An ordinary function, so the runtime's receiver (Bun passes the handle) reaches the handler.
|
|
65
|
+
const run = function(this: unknown, ...callArgs: unknown[]): unknown {
|
|
66
|
+
if (kind === 'timeout') {
|
|
67
|
+
live.delete(handleKey(id))
|
|
68
|
+
}
|
|
69
|
+
return Reflect.apply(handler, this, callArgs)
|
|
70
|
+
}
|
|
71
|
+
id = Reflect.apply(start, scope, [run, delay, ...args])
|
|
72
|
+
live.set(handleKey(id), entry)
|
|
73
|
+
return id
|
|
74
|
+
}
|
|
75
|
+
const clear = (stop: (id?: unknown) => void, id: unknown): void => {
|
|
76
|
+
// A timeout's and an interval's handles share one pool, so either clear ends either kind.
|
|
77
|
+
live.delete(handleKey(id))
|
|
78
|
+
Reflect.apply(stop, scope, [id])
|
|
79
|
+
}
|
|
80
|
+
// The stack is taken before `schedule`: a proper tail call (JavaScriptCore) would drop the frame `stackBelow` cuts at.
|
|
81
|
+
return {
|
|
82
|
+
setTimeout: function setTimeout(handler, delay, ...args) {
|
|
83
|
+
return schedule('timeout', recorded('timeout', handler, delay, setTimeout), handler, delay, args)
|
|
84
|
+
},
|
|
85
|
+
setInterval: function setInterval(handler, delay, ...args) {
|
|
86
|
+
return schedule('interval', recorded('interval', handler, delay, setInterval), handler, delay, args)
|
|
87
|
+
},
|
|
88
|
+
clearTimeout: (id) => clear(own.clearTimeout, id),
|
|
89
|
+
clearInterval: (id) => clear(own.clearInterval, id),
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const pending = <T>(promise: Promise<T>): Promise<T> => {
|
|
94
|
+
if (installed === null) {
|
|
95
|
+
return promise
|
|
96
|
+
}
|
|
97
|
+
const key = {}
|
|
98
|
+
live.set(key, record('waitUntil', null, pending))
|
|
99
|
+
return promise.then(
|
|
100
|
+
(value) => {
|
|
101
|
+
live.delete(key)
|
|
102
|
+
return value
|
|
103
|
+
},
|
|
104
|
+
(error: unknown) => {
|
|
105
|
+
live.delete(key)
|
|
106
|
+
throw error
|
|
107
|
+
},
|
|
108
|
+
)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return {
|
|
112
|
+
track(on) {
|
|
113
|
+
if (on && installed === null) {
|
|
114
|
+
const own = { setTimeout: scope.setTimeout, setInterval: scope.setInterval, clearTimeout: scope.clearTimeout, clearInterval: scope.clearInterval }
|
|
115
|
+
const wrapped = wrap(own)
|
|
116
|
+
installed = { own, wrapped }
|
|
117
|
+
scope.setTimeout = wrapped.setTimeout
|
|
118
|
+
scope.setInterval = wrapped.setInterval
|
|
119
|
+
scope.clearTimeout = wrapped.clearTimeout
|
|
120
|
+
scope.clearInterval = wrapped.clearInterval
|
|
121
|
+
} else if (!on && installed !== null) {
|
|
122
|
+
const { own, wrapped } = installed
|
|
123
|
+
// Restores only what is still the tracker's own, so a later wrapper by other code stays in place.
|
|
124
|
+
if (scope.setTimeout === wrapped.setTimeout) scope.setTimeout = own.setTimeout
|
|
125
|
+
if (scope.setInterval === wrapped.setInterval) scope.setInterval = own.setInterval
|
|
126
|
+
if (scope.clearTimeout === wrapped.clearTimeout) scope.clearTimeout = own.clearTimeout
|
|
127
|
+
if (scope.clearInterval === wrapped.clearInterval) scope.clearInterval = own.clearInterval
|
|
128
|
+
installed = null
|
|
129
|
+
live.clear()
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
pending,
|
|
133
|
+
view() {
|
|
134
|
+
const timers = [...live.values()]
|
|
135
|
+
return { tracking: installed !== null, timers: timers.slice(0, OPS_TIMERS_CAP), omitted: Math.max(0, timers.length - OPS_TIMERS_CAP) }
|
|
136
|
+
},
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* One key for every form of a handle a clear accepts: Bun's handle object also clears by its number, workerd's handle is
|
|
142
|
+
* that number already.
|
|
143
|
+
*/
|
|
144
|
+
function handleKey(id: unknown): unknown {
|
|
145
|
+
if (typeof id !== 'object' || id === null) {
|
|
146
|
+
return id
|
|
147
|
+
}
|
|
148
|
+
const numeric = Number(id)
|
|
149
|
+
return Number.isSafeInteger(numeric) ? numeric : id
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** The delay a timer was asked for, in whole milliseconds: what the runtime coerces to a number, else 0. */
|
|
153
|
+
function delayOf(delay: unknown): number {
|
|
154
|
+
const ms = typeof delay === 'number' || typeof delay === 'string' ? Number(delay) : 0
|
|
155
|
+
return Number.isFinite(ms) && ms > 0 ? Math.min(Math.trunc(ms), Number.MAX_SAFE_INTEGER) : 0
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** The frames below `caller`, without the tracker's own: at most `OPS_TIMER_STACK_MAX_FRAMES`, scrubbed, then cut to bytes. */
|
|
159
|
+
function stackBelow(caller: Caller, scrubber: Scrubber): string {
|
|
160
|
+
const holder: { stack?: string } = {}
|
|
161
|
+
Error.captureStackTrace(holder, caller)
|
|
162
|
+
const frames = (holder.stack ?? '').replace(/^Error\n/, '').split('\n').slice(0, OPS_TIMER_STACK_MAX_FRAMES)
|
|
163
|
+
return cutToBytes(scrubber.scrub(frames.join('\n')), OPS_TIMER_STACK_MAX_BYTES)
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** `text` cut at a character boundary so its UTF-8 encoding fits `bytes`. */
|
|
167
|
+
function cutToBytes(text: string, bytes: number): string {
|
|
168
|
+
const { read } = new TextEncoder().encodeInto(text, new Uint8Array(bytes))
|
|
169
|
+
return text.slice(0, read)
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* The isolate's tracker, shared by every object the isolate hosts. Its scrubber knows the credential shapes only: the
|
|
174
|
+
* tracker is reached before any object captures the environment's secret values.
|
|
175
|
+
*/
|
|
176
|
+
const isolateTracker = timerTracker(globalThis, () => Date.now(), secretScrubber([]))
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Records a promise an object hands to `waitUntil` while the tracker is on. Hand the answer, not `promise`, to
|
|
180
|
+
* `waitUntil`: it rejects as `promise` does, so an unhandled rejection still reaches the crash reporting.
|
|
181
|
+
*/
|
|
182
|
+
export function trackPending<T>(promise: Promise<T>): Promise<T> {
|
|
183
|
+
return isolateTracker.pending(promise)
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** The isolate's live timers and `waitUntil` promises; `track` switches the isolate's tracker on or off first. */
|
|
187
|
+
export function serveTimers(request: OpsTimersRequest): Response {
|
|
188
|
+
if (request.track !== null) {
|
|
189
|
+
isolateTracker.track(request.track)
|
|
190
|
+
}
|
|
191
|
+
return jsonResponse(isolateTracker.view(), 200)
|
|
192
|
+
}
|
package/src/cloudflare.ts
CHANGED
|
@@ -145,7 +145,8 @@ export interface ContractWorker {
|
|
|
145
145
|
/**
|
|
146
146
|
* The Worker module of a contract application, called only by the generated entrypoint, which also exports
|
|
147
147
|
* `FabrikaApplication`. The application runs in the environment's Durable Object (ADR-0065): `fetch` forwards every
|
|
148
|
-
* request to it with the proxy secret, the proxy's own headers unchanged, and `scheduled` forwards the
|
|
148
|
+
* request to it with the proxy secret, the proxy's own headers unchanged, and `scheduled` forwards the repair tick: the
|
|
149
|
+
* object's alarm runs the jobs, and the tick repairs its wakes and arms an alarm nothing armed yet.
|
|
149
150
|
* An ops request goes to the object its body names instead (ADR-0069). The object captures the environment, imports
|
|
150
151
|
* the application and boots it.
|
|
151
152
|
*/
|
|
@@ -169,7 +170,7 @@ export function createContractWorker(load: ImplementationLoader, options: Contra
|
|
|
169
170
|
})
|
|
170
171
|
const response = await applicationObject(env, applicationObjects(env).object)(tick)
|
|
171
172
|
if (!response.ok) {
|
|
172
|
-
throw new Error(`the application's object answered the
|
|
173
|
+
throw new Error(`the application's object answered the repair tick with ${response.status}`)
|
|
173
174
|
}
|
|
174
175
|
},
|
|
175
176
|
}
|