@fabrika/app 0.9.0-cloudflare.1 → 0.9.0-cloudflare.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/package.json +9 -9
  2. package/skill/SKILL.md +21 -17
  3. package/skill/references/app-setup.md +42 -11
  4. package/skill/references/endpoint.md +29 -22
  5. package/skill/references/errors.md +32 -31
  6. package/skill/references/jobs.md +22 -3
  7. package/skill/references/tests.md +29 -1
  8. package/src/build.ts +6 -1
  9. package/src/bun.ts +168 -5
  10. package/src/cloudflare/application.ts +546 -76
  11. package/src/cloudflare/bindings.ts +26 -5
  12. package/src/cloudflare/export.ts +621 -0
  13. package/src/cloudflare/import.ts +1097 -0
  14. package/src/cloudflare/maintenance.ts +518 -0
  15. package/src/cloudflare/names.ts +6 -0
  16. package/src/cloudflare/ops.ts +147 -18
  17. package/src/cloudflare/restore.ts +167 -0
  18. package/src/cloudflare/tracker.ts +192 -0
  19. package/src/cloudflare.ts +3 -2
  20. package/src/contract/artifact-schema.ts +13 -2
  21. package/src/contract/build.ts +38 -11
  22. package/src/contract/context.ts +26 -9
  23. package/src/contract/defaults.ts +22 -2
  24. package/src/contract/define.ts +1 -1
  25. package/src/contract/group.ts +10 -1
  26. package/src/contract/kit.ts +2 -1
  27. package/src/disk-assets.ts +8 -3
  28. package/src/files/kinds.ts +54 -24
  29. package/src/implement/anonymous.ts +1 -10
  30. package/src/implement/boot.ts +30 -12
  31. package/src/implement/capability.ts +3 -0
  32. package/src/implement/dispatch.ts +175 -37
  33. package/src/implement/egress.ts +367 -8
  34. package/src/implement/invocation.ts +60 -19
  35. package/src/implement/workload.ts +18 -3
  36. package/src/jobs/runner.ts +646 -98
  37. package/src/spa-fallback.ts +12 -0
  38. package/src/testing/endpoint-access.ts +135 -23
  39. package/src/testing/isolation.ts +8 -1
  40. package/src/testing/kit.ts +60 -0
  41. package/types/src/build.d.ts +1 -1
  42. package/types/src/bun.d.ts +34 -1
  43. package/types/src/cloudflare/application.d.ts +36 -5
  44. package/types/src/cloudflare/bindings.d.ts +6 -0
  45. package/types/src/cloudflare/export.d.ts +16 -0
  46. package/types/src/cloudflare/import.d.ts +58 -0
  47. package/types/src/cloudflare/maintenance.d.ts +117 -0
  48. package/types/src/cloudflare/names.d.ts +5 -0
  49. package/types/src/cloudflare/ops.d.ts +39 -4
  50. package/types/src/cloudflare/restore.d.ts +9 -0
  51. package/types/src/cloudflare/tracker.d.ts +29 -0
  52. package/types/src/cloudflare.d.ts +2 -1
  53. package/types/src/contract/artifact-schema.d.ts +2 -0
  54. package/types/src/contract/context.d.ts +24 -9
  55. package/types/src/contract/defaults.d.ts +19 -1
  56. package/types/src/contract/define.d.ts +1 -1
  57. package/types/src/contract/group.d.ts +9 -1
  58. package/types/src/contract/kit.d.ts +2 -2
  59. package/types/src/disk-assets.d.ts +4 -1
  60. package/types/src/files/kinds.d.ts +17 -6
  61. package/types/src/implement/anonymous.d.ts +0 -4
  62. package/types/src/implement/boot.d.ts +17 -0
  63. package/types/src/implement/capability.d.ts +2 -0
  64. package/types/src/implement/dispatch.d.ts +33 -1
  65. package/types/src/implement/egress.d.ts +36 -1
  66. package/types/src/implement/invocation.d.ts +20 -10
  67. package/types/src/implement/workload.d.ts +5 -0
  68. package/types/src/jobs/runner.d.ts +63 -2
  69. package/types/src/spa-fallback.d.ts +2 -0
  70. package/types/src/testing/endpoint-access.d.ts +25 -1
  71. package/types/src/testing/isolation.d.ts +1 -1
  72. package/types/src/testing/kit.d.ts +16 -0
@@ -9,31 +9,47 @@ import {
9
9
  FABRIKA_APPLICATION_OBJECTS,
10
10
  FABRIKA_OPS_KEY_DIGESTS,
11
11
  formatEndpointHeader,
12
+ OPS_DATA_REFUSED_STATUS,
12
13
  OPS_METHOD,
13
14
  OPS_ROW_COUNT_CAP,
15
+ OPS_STALE_STATUS,
14
16
  type OpsBootFailureContent,
15
17
  type OpsContentView,
18
+ type OpsDataRefusal,
16
19
  opsKeyDigest,
17
20
  type OpsOperation,
21
+ type OpsPublishedState,
18
22
  opsRefusal,
23
+ type OpsStaleRefusal,
19
24
  type OpsView,
20
25
  parseApplicationObjects,
21
26
  parseOpsContentView,
27
+ parseOpsExportRequest,
28
+ parseOpsImportRequest,
22
29
  parseOpsInspectRequest,
23
30
  parseOpsKeyDigests,
31
+ parseOpsMigrateAck,
24
32
  parseOpsMigrateRequest,
25
33
  parseOpsOperation,
34
+ parseOpsRestoreRequest,
26
35
  parseOpsStateRequest,
36
+ parseOpsTimersRequest,
27
37
  parseOpsView,
28
38
  } from '@fabrika/auth-core'
29
39
  import { FILES_TABLE, JOB_LEASES_TABLE, WEBHOOK_BINDINGS_TABLE, WEBHOOK_DELIVERIES_TABLE } from '@fabrika/db/migrations'
30
- import type { SqliteCursor, SqliteValue } from '@fabrika/db/sqlite'
40
+ import type { SqliteCursor, SqliteStorage, SqliteValue } from '@fabrika/db/sqlite'
41
+ import type { SqliteMigrationBundle } from '@fabrika/db/sqlite-migrate'
31
42
  import { opsProxyRow } from '@fabrika/proxy-core/manifest'
32
43
  import { timingSafeEqual } from 'node:crypto'
33
44
  import appPackage from '../../package.json' with { type: 'json' }
34
45
  import { jsonResponse } from '../errors.js'
46
+ import { serveExport } from './export.js'
47
+ import { importInProgress, serveImport } from './import.js'
48
+ import type { MaintenanceBarrier, TransitionResult } from './maintenance.js'
49
+ import { serveRestore } from './restore.js'
50
+ import { serveTimers } from './tracker.js'
35
51
 
36
- /** What the view reads of the object's storage; workerd's `DurableObjectStorage` has both optional members. */
52
+ /** What the view reads of the object's storage; workerd's `DurableObjectStorage` has every optional member. */
37
53
  export interface OpsStorage {
38
54
  readonly sql: {
39
55
  exec(query: string, ...bindings: SqliteValue[]): SqliteCursor
@@ -42,6 +58,28 @@ export interface OpsStorage {
42
58
  }
43
59
  /** The bun:sqlite test double has no alarm. */
44
60
  getAlarm?(): Promise<number | null>
61
+ /** The object's KV storage, which point-in-time recovery rewinds with its SQLite; the bun:sqlite double has none. */
62
+ get?(key: string): Promise<unknown>
63
+ put?(key: string, value: unknown): Promise<void>
64
+ /** The synchronous KV of a SQLite-backed object, in the same database; the bun:sqlite test double has none. */
65
+ readonly kv?: {
66
+ get(key: string): unknown
67
+ put(key: string, value: unknown): void
68
+ delete(key: string): boolean
69
+ }
70
+ /** Wipes the object's database, its KV and its alarm; the bun:sqlite test double has none. */
71
+ deleteAll?(): Promise<void>
72
+ }
73
+
74
+ /**
75
+ * The object's point-in-time recovery: workerd's storage bookmarks and the abort that resets the object. `null` where the
76
+ * storage has none (the bun:sqlite double, a runtime without PITR).
77
+ */
78
+ export interface OpsRecovery {
79
+ getCurrentBookmark(): Promise<string>
80
+ getBookmarkForTime(time: number): Promise<string>
81
+ onNextSessionRestoreBookmark(bookmark: string): Promise<string>
82
+ abort(reason: string): void
45
83
  }
46
84
 
47
85
  const OPS_ROW = opsProxyRow()
@@ -82,17 +120,29 @@ export function opsAccess(env: Readonly<Record<string, unknown>>): OpsAccess | n
82
120
 
83
121
  /** What an ops request reads of the object it reached. */
84
122
  export interface OpsTarget {
85
- readonly storage: OpsStorage
123
+ readonly storage: OpsStorage & SqliteStorage
124
+ /** The environment's bucket (the `FABRIKA_FILES` R2 binding), where dumps and the maintenance state live. */
125
+ readonly bucket: unknown
126
+ readonly recovery: OpsRecovery | null
127
+ /** The digest of the contract this release was deployed with (`FABRIKA_CONTRACT_DIGEST`); a dump's manifest records it. */
128
+ readonly contractDigest: string | null
129
+ /** The release's migration bundles in the order the object applies them: the framework's, then the application's. */
130
+ readonly migrations: readonly SqliteMigrationBundle[]
86
131
  /**
87
132
  * The instance's last failed boot, the application's error text included. Asked only after authentication: it
88
133
  * waits for a boot under way and never starts one.
89
134
  */
90
135
  bootFailure(): Promise<OpsBootFailureContent | null>
136
+ /** The barrier of the object named `object`, read from R2 on a cold start; rejects while R2 cannot be read. */
137
+ maintenance(object: string): Promise<MaintenanceBarrier>
138
+ /** Runs the pending migrations and boots the application; `false` when the boot failed. */
139
+ migrate(): Promise<boolean>
91
140
  }
92
141
 
93
142
  /**
94
143
  * Answers an ops request the Worker forwarded with the proxy secret, which the object checked before. Authentication
95
- * runs first; an unknown operation, a malformed body and an object name the list lacks are refused after it.
144
+ * runs first; an unknown operation, a malformed body and an object name the list lacks are refused after it. Every
145
+ * operation needs the barrier, so while R2 cannot be read each answers 503.
96
146
  */
97
147
  export async function serveOps(request: Request, access: OpsAccess | null, target: OpsTarget): Promise<Response> {
98
148
  if (access === null || !opsAuthenticated(request, access)) {
@@ -103,19 +153,87 @@ export async function serveOps(request: Request, access: OpsAccess | null, targe
103
153
  return opsRefusal()
104
154
  }
105
155
  const body = await jsonBody(request)
106
- if (operation === 'inspect') {
107
- const inspect = parseOpsInspectRequest(body)
108
- if (inspect === null || !access.objects.objects.includes(inspect.object)) {
109
- return opsRefusal()
110
- }
111
- return jsonResponse(await inspectView(target, inspect.content), 200)
112
- }
113
156
  const object = objectNamed(operation, body)
114
157
  if (object === null || !access.objects.objects.includes(object)) {
115
158
  return opsRefusal()
116
159
  }
117
- // `state` and `migrate` belong to the maintenance barrier (work unit 3.3b).
118
- return jsonResponse({ refused: 'not_implemented' }, 501)
160
+ let barrier: MaintenanceBarrier
161
+ try {
162
+ barrier = await target.maintenance(object)
163
+ } catch {
164
+ return new Response(null, { status: 503 })
165
+ }
166
+ switch (operation) {
167
+ case 'inspect': {
168
+ const inspect = parseOpsInspectRequest(body)
169
+ return inspect === null ? opsRefusal() : jsonResponse(await inspectView(target, barrier, inspect.content), 200)
170
+ }
171
+ case 'state': {
172
+ const transition = parseOpsStateRequest(body)
173
+ const { kv } = target.storage
174
+ // ADR-0070 §6: an object that holds an import marker never leaves `inspect`.
175
+ const frozen = () => kv !== undefined && importInProgress({ kv })
176
+ return transition === null ? opsRefusal() : transitionAnswer(await barrier.transition(transition, frozen))
177
+ }
178
+ case 'migrate': {
179
+ const migration = parseOpsMigrateRequest(body)
180
+ return migration === null ? opsRefusal() : await migrateAnswer(target, barrier, migration.generation)
181
+ }
182
+ case 'export': {
183
+ const step = parseOpsExportRequest(body)
184
+ return step === null ? opsRefusal() : await serveExport(target, barrier, step)
185
+ }
186
+ case 'import': {
187
+ const step = parseOpsImportRequest(body)
188
+ return step === null ? opsRefusal() : await serveImport(target, barrier, step, access.objects.object)
189
+ }
190
+ case 'restore': {
191
+ const step = parseOpsRestoreRequest(body)
192
+ return step === null ? opsRefusal() : await serveRestore(target, barrier, step, object === access.objects.object)
193
+ }
194
+ case 'timers': {
195
+ const timers = parseOpsTimersRequest(body)
196
+ return timers === null ? opsRefusal() : serveTimers(timers)
197
+ }
198
+ }
199
+ }
200
+
201
+ /** A transition's answer: the acknowledgement, the 409 naming what is published, or a 503 that Control retries. */
202
+ function transitionAnswer(result: TransitionResult): Response {
203
+ switch (result.kind) {
204
+ case 'acknowledged':
205
+ return jsonResponse(result.ack, 200)
206
+ case 'stale':
207
+ return staleRefusal(result.published)
208
+ case 'failed':
209
+ return new Response(null, { status: 503 })
210
+ case 'frozen': {
211
+ const refusal: OpsDataRefusal = { refused: 'wrong_mode', table: null }
212
+ return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
213
+ }
214
+ }
215
+ }
216
+
217
+ /** Runs the migrations under the published generation and acknowledges with the new ledger; a failed boot is a 500. */
218
+ async function migrateAnswer(target: OpsTarget, barrier: MaintenanceBarrier, generation: number): Promise<Response> {
219
+ const result = await barrier.migrate(generation, () => target.migrate())
220
+ if (result.kind === 'stale') {
221
+ return staleRefusal(result.published)
222
+ }
223
+ if (!result.value) {
224
+ return new Response(null, { status: 500 })
225
+ }
226
+ const migrations = tableNames(target.storage).includes(MIGRATION_LEDGER) ? migrationLedger(target.storage) : []
227
+ const ack = parseOpsMigrateAck({ generation, migrations })
228
+ if (ack === null) {
229
+ throw new Error("the object's migration ledger does not parse as the wire's acknowledgement")
230
+ }
231
+ return jsonResponse(ack, 200)
232
+ }
233
+
234
+ function staleRefusal(published: OpsPublishedState | null): Response {
235
+ const refusal: OpsStaleRefusal = { refused: 'stale_generation', published }
236
+ return jsonResponse(refusal, OPS_STALE_STATUS)
119
237
  }
120
238
 
121
239
  /** The object an ops request names, read with its operation's strict parser; `null` for a body that does not parse. */
@@ -127,6 +245,14 @@ export function objectNamed(operation: OpsOperation, body: unknown): string | nu
127
245
  return parseOpsStateRequest(body)?.object ?? null
128
246
  case 'migrate':
129
247
  return parseOpsMigrateRequest(body)?.object ?? null
248
+ case 'export':
249
+ return parseOpsExportRequest(body)?.object ?? null
250
+ case 'import':
251
+ return parseOpsImportRequest(body)?.object ?? null
252
+ case 'restore':
253
+ return parseOpsRestoreRequest(body)?.object ?? null
254
+ case 'timers':
255
+ return parseOpsTimersRequest(body)?.object ?? null
130
256
  }
131
257
  }
132
258
 
@@ -167,7 +293,7 @@ export function opsAuthenticated(request: Request, access: OpsAccess): boolean {
167
293
  * The view of the object's storage, read and never written. A table the framework has not created yet (a boot that
168
294
  * failed before its migrations) reads as empty. What the storage holds is parsed by the wire's own parser.
169
295
  */
170
- async function inspectView(target: OpsTarget, content: boolean): Promise<OpsView | OpsContentView> {
296
+ async function inspectView(target: OpsTarget, barrier: MaintenanceBarrier, content: boolean): Promise<OpsView | OpsContentView> {
171
297
  const { storage } = target
172
298
  const bootFailure = await target.bootFailure()
173
299
  const tables = tableNames(storage)
@@ -175,14 +301,12 @@ async function inspectView(target: OpsTarget, content: boolean): Promise<OpsView
175
301
  const alarm = storage.getAlarm === undefined ? missing('getAlarm') : await storage.getAlarm()
176
302
  const view = {
177
303
  release: appPackage.version,
178
- maintenance: { mode: 'open', generation: null, published: false },
304
+ maintenance: barrier.view(),
179
305
  database: {
180
306
  sizeBytes: storage.sql.databaseSize ?? missing('sql.databaseSize'),
181
307
  tables: tables.filter(admittedTableName).map((name) => ({ name, rows: cappedCount(storage, name) })),
182
308
  },
183
- migrations: has(MIGRATION_LEDGER)
184
- ? rows(storage, `select bundle, name, checksum, applied_at as appliedAt from ${MIGRATION_LEDGER} order by bundle, name`)
185
- : [],
309
+ migrations: has(MIGRATION_LEDGER) ? migrationLedger(storage) : [],
186
310
  scheduler: { alarm: alarm === null ? null : new Date(alarm).toISOString() },
187
311
  jobs: has(JOB_LEASES_TABLE)
188
312
  ? rows(
@@ -211,6 +335,11 @@ async function inspectView(target: OpsTarget, content: boolean): Promise<OpsView
211
335
  return parsed
212
336
  }
213
337
 
338
+ /** The ledger `@fabrika/db/sqlite-migrate` keeps, as the wire names its rows; read only once the table exists. */
339
+ function migrationLedger(storage: OpsStorage): Record<string, SqliteValue>[] {
340
+ return rows(storage, `select bundle, name, checksum, applied_at as appliedAt from ${MIGRATION_LEDGER} order by bundle, name`)
341
+ }
342
+
214
343
  /** Cloudflare's own `_cf_*` tables and SQLite's `sqlite_*` are no table of the application or the framework. */
215
344
  function tableNames(storage: OpsStorage): string[] {
216
345
  return storage.sql.exec(
@@ -0,0 +1,167 @@
1
+ // One step of the original object's point-in-time restore (ADR-0070): the ops operation `restore`, served before the application boots.
2
+
3
+ import {
4
+ OPS_DATA_REFUSED_STATUS,
5
+ OPS_STALE_STATUS,
6
+ type OpsDataRefusal,
7
+ type OpsDataRefusalReason,
8
+ type OpsMode,
9
+ type OpsRestoreArm,
10
+ type OpsRestorePrepare,
11
+ type OpsRestoreRequest,
12
+ type OpsRestoreVerify,
13
+ type OpsStaleRefusal,
14
+ parseOpsRestoreArmAck,
15
+ parseOpsRestorePrepareAck,
16
+ parseOpsRestoreVerifyAck,
17
+ } from '@fabrika/auth-core'
18
+ import { jsonResponse } from '../errors.js'
19
+ import type { MaintenanceBarrier, TurnStep } from './maintenance.js'
20
+ import type { OpsRecovery, OpsStorage, OpsTarget } from './ops.js'
21
+
22
+ /**
23
+ * The KV key prefix of an operation's sentinel, followed by its id. KV, unlike R2, is rewound with the object's SQLite,
24
+ * so the absence of an operation's key proves its restore; no operation writes another's.
25
+ */
26
+ const SENTINEL_PREFIX = '__fabrika/restore-sentinel/'
27
+
28
+ const ABORT_REASON = 'fabrika: point-in-time restore armed'
29
+
30
+ /**
31
+ * One step of the original object's point-in-time restore (ADR-0070 §9 step 6). `prepare` and `arm` run in the
32
+ * barrier's turn, only under the published generation, on an object that is not the one ordinary traffic reaches
33
+ * (`serving`), while it is `inspect`; `verify` writes nothing and runs in any mode.
34
+ */
35
+ export async function serveRestore(target: OpsTarget, barrier: MaintenanceBarrier, step: OpsRestoreRequest, serving: boolean): Promise<Response> {
36
+ const { recovery } = target
37
+ if (recovery === null) {
38
+ return jsonResponse({ refused: 'not_implemented' }, 501)
39
+ }
40
+ const sentinels = kvSentinels(target.storage)
41
+ switch (step.step) {
42
+ case 'prepare':
43
+ return await inTurn(barrier, step.generation, serving, () => prepare(recovery, sentinels, step))
44
+ case 'arm':
45
+ return await inTurn(barrier, step.generation, serving, async () => {
46
+ await barrier.restrictedWrite()
47
+ return await arm(recovery, step)
48
+ })
49
+ case 'verify':
50
+ return await verify(recovery, sentinels, step)
51
+ }
52
+ }
53
+
54
+ /**
55
+ * The first prepare of an operation reads the current bookmark, then the target's, and keeps both under its sentinel
56
+ * key; a retried one answers what it kept. A target not lexically before the current bookmark would keep the sentinel.
57
+ */
58
+ async function prepare(recovery: OpsRecovery, sentinels: SentinelStore, step: OpsRestorePrepare): Promise<TurnStep<Response>> {
59
+ const kept = await sentinels.read(step.sentinel)
60
+ if (kept !== undefined) {
61
+ return answered(answer(parseOpsRestorePrepareAck(kept)))
62
+ }
63
+ const current = await recovery.getCurrentBookmark()
64
+ let target: string
65
+ try {
66
+ target = await recovery.getBookmarkForTime(Date.parse(step.at))
67
+ } catch {
68
+ return answered(dataRefusal('bookmark_unavailable'))
69
+ }
70
+ if (!(target < current)) {
71
+ return answered(dataRefusal('bookmark_unavailable'))
72
+ }
73
+ const ack = parseOpsRestorePrepareAck({ step: 'prepare', current, target })
74
+ if (ack !== null) {
75
+ await sentinels.write(step.sentinel, ack)
76
+ }
77
+ return answered(answer(ack))
78
+ }
79
+
80
+ /** Arms the restore and resets the object once the answer is out, holding the barrier's turn until then. */
81
+ async function arm(recovery: OpsRecovery, step: OpsRestoreArm): Promise<TurnStep<Response>> {
82
+ let undo: string
83
+ try {
84
+ undo = await recovery.onNextSessionRestoreBookmark(step.target)
85
+ } catch {
86
+ return answered(dataRefusal('bookmark_unavailable'))
87
+ }
88
+ const response = answer(parseOpsRestoreArmAck({ step: 'arm', undo }))
89
+ // workerd's abort throws, so on a real object the turn is never released and no transition is acknowledged after it.
90
+ const reset = new Promise<void>((resolve) =>
91
+ setTimeout(() => {
92
+ recovery.abort(ABORT_REASON)
93
+ resolve()
94
+ }, 0)
95
+ )
96
+ return { value: response, holdUntil: reset }
97
+ }
98
+
99
+ /** Present while the KV holds this operation's key: a restore to a target before its `prepare` removed it. */
100
+ async function verify(recovery: OpsRecovery, sentinels: SentinelStore, step: OpsRestoreVerify): Promise<Response> {
101
+ const present = (await sentinels.read(step.sentinel)) !== undefined
102
+ return answer(parseOpsRestoreVerifyAck({ step: 'verify', sentinel: present ? 'present' : 'absent', bookmark: await recovery.getCurrentBookmark() }))
103
+ }
104
+
105
+ /**
106
+ * ADR-0070 §1 and §9: inside the barrier's turn, ADR-0069's stale refusal unless `generation` is the published one,
107
+ * then `serving_object` on the object of ordinary traffic, then `wrong_mode` unless `inspect`.
108
+ */
109
+ async function inTurn(
110
+ barrier: MaintenanceBarrier,
111
+ generation: number,
112
+ serving: boolean,
113
+ work: () => Promise<TurnStep<Response>>,
114
+ ): Promise<Response> {
115
+ const result = await barrier.step(generation, (mode: OpsMode) => {
116
+ if (serving) {
117
+ return Promise.resolve(answered(dataRefusal('serving_object')))
118
+ }
119
+ return mode === 'inspect' ? work() : Promise.resolve(answered(dataRefusal('wrong_mode')))
120
+ })
121
+ if (result.kind === 'ran') {
122
+ return result.value
123
+ }
124
+ const refusal: OpsStaleRefusal = { refused: 'stale_generation', published: result.published }
125
+ return jsonResponse(refusal, OPS_STALE_STATUS)
126
+ }
127
+
128
+ function answered(response: Response): TurnStep<Response> {
129
+ return { value: response, holdUntil: null }
130
+ }
131
+
132
+ function dataRefusal(reason: OpsDataRefusalReason): Response {
133
+ const refusal: OpsDataRefusal = { refused: reason, table: null }
134
+ return jsonResponse(refusal, OPS_DATA_REFUSED_STATUS)
135
+ }
136
+
137
+ /** An answer the wire's own parser accepted; a bookmark the storage handed out that it refuses fails the step. */
138
+ function answer(parsed: object | null): Response {
139
+ if (parsed === null) {
140
+ throw new Error("the object's restore answer does not parse as the wire's acknowledgement")
141
+ }
142
+ return jsonResponse(parsed, 200)
143
+ }
144
+
145
+ /** The operations' sentinels in the object's KV storage, each under its own key. */
146
+ interface SentinelStore {
147
+ /** What the operation's key holds, `undefined` when it holds nothing. */
148
+ read(sentinel: string): Promise<unknown>
149
+ write(sentinel: string, value: unknown): Promise<void>
150
+ }
151
+
152
+ function kvSentinels(storage: OpsStorage): SentinelStore {
153
+ return {
154
+ async read(sentinel) {
155
+ if (storage.get === undefined) {
156
+ throw new Error("the object's storage has no get; a Durable Object's storage has it")
157
+ }
158
+ return await storage.get(`${SENTINEL_PREFIX}${sentinel}`)
159
+ },
160
+ async write(sentinel, value) {
161
+ if (storage.put === undefined) {
162
+ throw new Error("the object's storage has no put; a Durable Object's storage has it")
163
+ }
164
+ await storage.put(`${SENTINEL_PREFIX}${sentinel}`, value)
165
+ },
166
+ }
167
+ }
@@ -0,0 +1,192 @@
1
+ // The pending-work tracker (ADR-0069 §2, ADR-0070 §10): the ops operation `timers`, diagnostics only. While it is on, it
2
+ // wraps the isolate's global timers and records each live timer and each `waitUntil` promise with the stack that
3
+ // registered it, scrubbed and bounded. Its scope is the isolate, not one object: every object the isolate hosts shares
4
+ // the switch and the records, and only the isolate's eviction resets them, so the next isolate's tracker starts off.
5
+
6
+ import {
7
+ OPS_TIMER_STACK_MAX_BYTES,
8
+ OPS_TIMER_STACK_MAX_FRAMES,
9
+ OPS_TIMERS_CAP,
10
+ type OpsTimer,
11
+ type OpsTimersRequest,
12
+ type OpsTimersView,
13
+ } from '@fabrika/auth-core'
14
+ import { jsonResponse } from '../errors.js'
15
+ import { type Scrubber, secretScrubber } from '../jobs/scrub.js'
16
+
17
+ /** The global timers the tracker wraps while it is on; `globalThis` fits it. */
18
+ export interface TimerScope {
19
+ setTimeout(handler: unknown, delay?: unknown, ...args: unknown[]): unknown
20
+ setInterval(handler: unknown, delay?: unknown, ...args: unknown[]): unknown
21
+ clearTimeout(id?: unknown): void
22
+ clearInterval(id?: unknown): void
23
+ }
24
+
25
+ export interface TimerTracker {
26
+ /** Switches tracking on (wrapping the scope's timers) or off (restoring them and forgetting every record). */
27
+ track(on: boolean): void
28
+ /**
29
+ * While tracking is on, records `promise` until it settles and answers a chain that settles as it does, rejection
30
+ * included, so the caller hands the chain to `waitUntil` and an unhandled rejection stays unhandled. Off, `promise`.
31
+ */
32
+ pending<T>(promise: Promise<T>): Promise<T>
33
+ /** The live records, oldest first, at most `OPS_TIMERS_CAP` of them. */
34
+ view(): OpsTimersView
35
+ }
36
+
37
+ type Caller = (...args: never[]) => unknown
38
+
39
+ /** A tracker over `scope`'s timers, off until `track(true)`; each stack is scrubbed by `scrubber`. */
40
+ export function timerTracker(scope: TimerScope, now: () => number, scrubber: Scrubber): TimerTracker {
41
+ // Keyed by the timer's handle (`handleKey`), or by a fresh object for a promise; a Map keeps creation order.
42
+ const live = new Map<unknown, OpsTimer>()
43
+ /** The scope's own timers and the tracker's wrappers over them, while tracking is on. */
44
+ let installed: { readonly own: TimerScope; readonly wrapped: TimerScope } | null = null
45
+
46
+ const record = (kind: OpsTimer['kind'], delay: unknown, caller: Caller): OpsTimer => ({
47
+ kind,
48
+ since: new Date(now()).toISOString(),
49
+ delayMs: kind === 'waitUntil' ? null : delayOf(delay),
50
+ stack: stackBelow(caller, scrubber),
51
+ })
52
+
53
+ // A wrapper that other code kept after `track(false)` still works; it records only while tracking is on.
54
+ const wrap = (own: TimerScope): TimerScope => {
55
+ /** The record of a timer about to start, taken in the wrapper's own frame; `null` when nothing is recorded. */
56
+ const recorded = (kind: 'timeout' | 'interval', handler: unknown, delay: unknown, caller: Caller): OpsTimer | null =>
57
+ installed === null || typeof handler !== 'function' ? null : record(kind, delay, caller)
58
+ const schedule = (kind: 'timeout' | 'interval', entry: OpsTimer | null, handler: unknown, delay: unknown, args: readonly unknown[]): unknown => {
59
+ const start = kind === 'timeout' ? own.setTimeout : own.setInterval
60
+ if (entry === null || typeof handler !== 'function') {
61
+ return Reflect.apply(start, scope, [handler, delay, ...args])
62
+ }
63
+ let id: unknown
64
+ // An ordinary function, so the runtime's receiver (Bun passes the handle) reaches the handler.
65
+ const run = function(this: unknown, ...callArgs: unknown[]): unknown {
66
+ if (kind === 'timeout') {
67
+ live.delete(handleKey(id))
68
+ }
69
+ return Reflect.apply(handler, this, callArgs)
70
+ }
71
+ id = Reflect.apply(start, scope, [run, delay, ...args])
72
+ live.set(handleKey(id), entry)
73
+ return id
74
+ }
75
+ const clear = (stop: (id?: unknown) => void, id: unknown): void => {
76
+ // A timeout's and an interval's handles share one pool, so either clear ends either kind.
77
+ live.delete(handleKey(id))
78
+ Reflect.apply(stop, scope, [id])
79
+ }
80
+ // The stack is taken before `schedule`: a proper tail call (JavaScriptCore) would drop the frame `stackBelow` cuts at.
81
+ return {
82
+ setTimeout: function setTimeout(handler, delay, ...args) {
83
+ return schedule('timeout', recorded('timeout', handler, delay, setTimeout), handler, delay, args)
84
+ },
85
+ setInterval: function setInterval(handler, delay, ...args) {
86
+ return schedule('interval', recorded('interval', handler, delay, setInterval), handler, delay, args)
87
+ },
88
+ clearTimeout: (id) => clear(own.clearTimeout, id),
89
+ clearInterval: (id) => clear(own.clearInterval, id),
90
+ }
91
+ }
92
+
93
+ const pending = <T>(promise: Promise<T>): Promise<T> => {
94
+ if (installed === null) {
95
+ return promise
96
+ }
97
+ const key = {}
98
+ live.set(key, record('waitUntil', null, pending))
99
+ return promise.then(
100
+ (value) => {
101
+ live.delete(key)
102
+ return value
103
+ },
104
+ (error: unknown) => {
105
+ live.delete(key)
106
+ throw error
107
+ },
108
+ )
109
+ }
110
+
111
+ return {
112
+ track(on) {
113
+ if (on && installed === null) {
114
+ const own = { setTimeout: scope.setTimeout, setInterval: scope.setInterval, clearTimeout: scope.clearTimeout, clearInterval: scope.clearInterval }
115
+ const wrapped = wrap(own)
116
+ installed = { own, wrapped }
117
+ scope.setTimeout = wrapped.setTimeout
118
+ scope.setInterval = wrapped.setInterval
119
+ scope.clearTimeout = wrapped.clearTimeout
120
+ scope.clearInterval = wrapped.clearInterval
121
+ } else if (!on && installed !== null) {
122
+ const { own, wrapped } = installed
123
+ // Restores only what is still the tracker's own, so a later wrapper by other code stays in place.
124
+ if (scope.setTimeout === wrapped.setTimeout) scope.setTimeout = own.setTimeout
125
+ if (scope.setInterval === wrapped.setInterval) scope.setInterval = own.setInterval
126
+ if (scope.clearTimeout === wrapped.clearTimeout) scope.clearTimeout = own.clearTimeout
127
+ if (scope.clearInterval === wrapped.clearInterval) scope.clearInterval = own.clearInterval
128
+ installed = null
129
+ live.clear()
130
+ }
131
+ },
132
+ pending,
133
+ view() {
134
+ const timers = [...live.values()]
135
+ return { tracking: installed !== null, timers: timers.slice(0, OPS_TIMERS_CAP), omitted: Math.max(0, timers.length - OPS_TIMERS_CAP) }
136
+ },
137
+ }
138
+ }
139
+
140
+ /**
141
+ * One key for every form of a handle a clear accepts: Bun's handle object also clears by its number, workerd's handle is
142
+ * that number already.
143
+ */
144
+ function handleKey(id: unknown): unknown {
145
+ if (typeof id !== 'object' || id === null) {
146
+ return id
147
+ }
148
+ const numeric = Number(id)
149
+ return Number.isSafeInteger(numeric) ? numeric : id
150
+ }
151
+
152
+ /** The delay a timer was asked for, in whole milliseconds: what the runtime coerces to a number, else 0. */
153
+ function delayOf(delay: unknown): number {
154
+ const ms = typeof delay === 'number' || typeof delay === 'string' ? Number(delay) : 0
155
+ return Number.isFinite(ms) && ms > 0 ? Math.min(Math.trunc(ms), Number.MAX_SAFE_INTEGER) : 0
156
+ }
157
+
158
+ /** The frames below `caller`, without the tracker's own: at most `OPS_TIMER_STACK_MAX_FRAMES`, scrubbed, then cut to bytes. */
159
+ function stackBelow(caller: Caller, scrubber: Scrubber): string {
160
+ const holder: { stack?: string } = {}
161
+ Error.captureStackTrace(holder, caller)
162
+ const frames = (holder.stack ?? '').replace(/^Error\n/, '').split('\n').slice(0, OPS_TIMER_STACK_MAX_FRAMES)
163
+ return cutToBytes(scrubber.scrub(frames.join('\n')), OPS_TIMER_STACK_MAX_BYTES)
164
+ }
165
+
166
+ /** `text` cut at a character boundary so its UTF-8 encoding fits `bytes`. */
167
+ function cutToBytes(text: string, bytes: number): string {
168
+ const { read } = new TextEncoder().encodeInto(text, new Uint8Array(bytes))
169
+ return text.slice(0, read)
170
+ }
171
+
172
+ /**
173
+ * The isolate's tracker, shared by every object the isolate hosts. Its scrubber knows the credential shapes only: the
174
+ * tracker is reached before any object captures the environment's secret values.
175
+ */
176
+ const isolateTracker = timerTracker(globalThis, () => Date.now(), secretScrubber([]))
177
+
178
+ /**
179
+ * Records a promise an object hands to `waitUntil` while the tracker is on. Hand the answer, not `promise`, to
180
+ * `waitUntil`: it rejects as `promise` does, so an unhandled rejection still reaches the crash reporting.
181
+ */
182
+ export function trackPending<T>(promise: Promise<T>): Promise<T> {
183
+ return isolateTracker.pending(promise)
184
+ }
185
+
186
+ /** The isolate's live timers and `waitUntil` promises; `track` switches the isolate's tracker on or off first. */
187
+ export function serveTimers(request: OpsTimersRequest): Response {
188
+ if (request.track !== null) {
189
+ isolateTracker.track(request.track)
190
+ }
191
+ return jsonResponse(isolateTracker.view(), 200)
192
+ }
package/src/cloudflare.ts CHANGED
@@ -145,7 +145,8 @@ export interface ContractWorker {
145
145
  /**
146
146
  * The Worker module of a contract application, called only by the generated entrypoint, which also exports
147
147
  * `FabrikaApplication`. The application runs in the environment's Durable Object (ADR-0065): `fetch` forwards every
148
- * request to it with the proxy secret, the proxy's own headers unchanged, and `scheduled` forwards the minute tick.
148
+ * request to it with the proxy secret, the proxy's own headers unchanged, and `scheduled` forwards the repair tick: the
149
+ * object's alarm runs the jobs, and the tick repairs its wakes and arms an alarm nothing armed yet.
149
150
  * An ops request goes to the object its body names instead (ADR-0069). The object captures the environment, imports
150
151
  * the application and boots it.
151
152
  */
@@ -169,7 +170,7 @@ export function createContractWorker(load: ImplementationLoader, options: Contra
169
170
  })
170
171
  const response = await applicationObject(env, applicationObjects(env).object)(tick)
171
172
  if (!response.ok) {
172
- throw new Error(`the application's object answered the minute tick with ${response.status}`)
173
+ throw new Error(`the application's object answered the repair tick with ${response.status}`)
173
174
  }
174
175
  },
175
176
  }