@nimbus-sh/worker 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE.md +5 -0
- package/README.md +10 -0
- package/dist/_shared/session-router.d.ts +7 -0
- package/dist/_shared/session-router.d.ts.map +1 -1
- package/dist/_shared/session-router.js +7 -0
- package/dist/bindings/body.d.ts +6 -0
- package/dist/bindings/body.d.ts.map +1 -0
- package/dist/bindings/body.js +37 -0
- package/dist/bindings/kv.d.ts +0 -2
- package/dist/bindings/kv.d.ts.map +1 -1
- package/dist/bindings/kv.js +3 -42
- package/dist/bindings/r2.d.ts +0 -2
- package/dist/bindings/r2.d.ts.map +1 -1
- package/dist/bindings/r2.js +4 -44
- package/dist/esbuild-cli-artifact.generated.d.ts +18 -0
- package/dist/esbuild-cli-artifact.generated.d.ts.map +1 -0
- package/dist/esbuild-cli-artifact.generated.js +17 -0
- package/dist/esbuild-wasm-bundle.generated.d.ts +24 -24
- package/dist/esbuild-wasm-bundle.generated.d.ts.map +1 -1
- package/dist/esbuild-wasm-bundle.generated.js +24 -24
- package/dist/facets/cirrus-real.d.ts.map +1 -1
- package/dist/facets/cirrus-real.js +11 -6
- package/dist/facets/compose.d.ts +11 -7
- package/dist/facets/compose.d.ts.map +1 -1
- package/dist/facets/compose.js +8 -31
- package/dist/facets/data-plan.d.ts +79 -0
- package/dist/facets/data-plan.d.ts.map +1 -0
- package/dist/facets/data-plan.js +477 -0
- package/dist/facets/esbuild-bundle-pool.d.ts +1 -1
- package/dist/facets/esbuild-bundle-pool.js +3 -3
- package/dist/facets/esbuild-transform.d.ts +38 -8
- package/dist/facets/esbuild-transform.d.ts.map +1 -1
- package/dist/facets/esbuild-transform.js +219 -19
- package/dist/facets/exec-telemetry.d.ts +7 -3
- package/dist/facets/exec-telemetry.d.ts.map +1 -1
- package/dist/facets/manager.d.ts +198 -69
- package/dist/facets/manager.d.ts.map +1 -1
- package/dist/facets/manager.js +1276 -652
- package/dist/facets/opencode-staging.d.ts +0 -2
- package/dist/facets/opencode-staging.d.ts.map +1 -1
- package/dist/facets/opencode-staging.js +5 -11
- package/dist/facets/process.d.ts +4 -0
- package/dist/facets/process.d.ts.map +1 -1
- package/dist/facets/process.js +3 -0
- package/dist/facets/read-profile.d.ts +149 -0
- package/dist/facets/read-profile.d.ts.map +1 -0
- package/dist/facets/read-profile.js +483 -0
- package/dist/facets/real-vite-hmr.d.ts +2 -0
- package/dist/facets/real-vite-hmr.d.ts.map +1 -1
- package/dist/facets/real-vite-hmr.js +3 -2
- package/dist/facets/resident-identity.d.ts +0 -1
- package/dist/facets/resident-identity.d.ts.map +1 -1
- package/dist/facets/resident-identity.js +0 -3
- package/dist/facets/wasm-swap-registry.d.ts +34 -25
- package/dist/facets/wasm-swap-registry.d.ts.map +1 -1
- package/dist/facets/wasm-swap-registry.js +62 -56
- package/dist/facets/ws-terminal.d.ts +4 -4
- package/dist/facets/ws-terminal.d.ts.map +1 -1
- package/dist/facets/ws-terminal.js +4 -5
- package/dist/git/commands.d.ts +33 -10
- package/dist/git/commands.d.ts.map +1 -1
- package/dist/git/commands.js +1891 -197
- package/dist/git/network-facet.d.ts +12 -2
- package/dist/git/network-facet.d.ts.map +1 -1
- package/dist/git/network-facet.js +69 -18
- package/dist/git/unified-diff.d.ts +74 -0
- package/dist/git/unified-diff.d.ts.map +1 -0
- package/dist/git/unified-diff.js +676 -0
- package/dist/git-bundle.generated.d.ts +7 -11
- package/dist/git-bundle.generated.d.ts.map +1 -1
- package/dist/git-bundle.generated.js +7 -11
- package/dist/hosted/commands.d.ts.map +1 -1
- package/dist/hosted/commands.js +64 -245
- package/dist/hosted/runtime.d.ts +22 -15
- package/dist/hosted/runtime.d.ts.map +1 -1
- package/dist/hosted/runtime.js +72 -14
- package/dist/hosted/services.d.ts +8 -1
- package/dist/hosted/services.d.ts.map +1 -1
- package/dist/hosted/services.js +12 -13
- package/dist/hosted/session.d.ts +165 -0
- package/dist/hosted/session.d.ts.map +1 -0
- package/dist/hosted/session.js +132 -0
- package/dist/loaders/generated-workers.d.ts +6 -3
- package/dist/loaders/generated-workers.d.ts.map +1 -1
- package/dist/loaders/generated-workers.js +8 -5
- package/dist/loaders/npm-resolve-preamble.d.ts.map +1 -1
- package/dist/loaders/npm-resolve-preamble.js +0 -27
- package/dist/loaders/pre-bundle-preamble.d.ts +6 -3
- package/dist/loaders/pre-bundle-preamble.d.ts.map +1 -1
- package/dist/loaders/pre-bundle-preamble.js +13 -8
- package/dist/node-shims-artifact.generated.d.ts +13 -4
- package/dist/node-shims-artifact.generated.d.ts.map +1 -1
- package/dist/node-shims-artifact.generated.js +16 -7
- package/dist/npm/bin-links.d.ts +3 -1
- package/dist/npm/bin-links.d.ts.map +1 -1
- package/dist/npm/bin-links.js +32 -8
- package/dist/npm/cache.d.ts +11 -1
- package/dist/npm/cache.d.ts.map +1 -1
- package/dist/npm/cache.js +21 -7
- package/dist/npm/install-batch-facet.d.ts +4 -2
- package/dist/npm/install-batch-facet.d.ts.map +1 -1
- package/dist/npm/install-batch-facet.js +32 -18
- package/dist/npm/installer.d.ts +23 -3
- package/dist/npm/installer.d.ts.map +1 -1
- package/dist/npm/installer.js +330 -131
- package/dist/npm/npx-install.d.ts +3 -1
- package/dist/npm/npx-install.d.ts.map +1 -1
- package/dist/npm/npx-install.js +4 -1
- package/dist/npm/package-lock.d.ts +32 -0
- package/dist/npm/package-lock.d.ts.map +1 -0
- package/dist/npm/package-lock.js +65 -0
- package/dist/npm/placement.d.ts +16 -0
- package/dist/npm/placement.d.ts.map +1 -0
- package/dist/npm/placement.js +33 -0
- package/dist/npm/pre-bundle-facet.d.ts +1 -30
- package/dist/npm/pre-bundle-facet.d.ts.map +1 -1
- package/dist/npm/pre-bundle-facet.js +3 -5
- package/dist/npm/r2-cache.d.ts +12 -6
- package/dist/npm/r2-cache.d.ts.map +1 -1
- package/dist/npm/r2-cache.js +34 -16
- package/dist/npm/resolve-one-facet.d.ts +3 -0
- package/dist/npm/resolve-one-facet.d.ts.map +1 -1
- package/dist/npm/resolve-one-facet.js +15 -3
- package/dist/npm/resolver.d.ts +11 -13
- package/dist/npm/resolver.d.ts.map +1 -1
- package/dist/npm/resolver.js +12 -25
- package/dist/opencode-artifact.generated.js +2 -2
- package/dist/replica/routing.d.ts +2 -38
- package/dist/replica/routing.d.ts.map +1 -1
- package/dist/replica/routing.js +3 -55
- package/dist/router/index.d.ts.map +1 -1
- package/dist/router/index.js +15 -2
- package/dist/router/public-directory.d.ts +18 -0
- package/dist/router/public-directory.d.ts.map +1 -1
- package/dist/router/public-directory.js +1 -1
- package/dist/router/remote-api.d.ts +3 -1
- package/dist/router/remote-api.d.ts.map +1 -1
- package/dist/router/remote-api.js +42 -79
- package/dist/runtime/bash-repl.js +2 -2
- package/dist/runtime/bun-repl.d.ts.map +1 -1
- package/dist/runtime/bun-repl.js +2 -13
- package/dist/runtime/cpython-resident.d.ts.map +1 -1
- package/dist/runtime/cpython-resident.js +2 -10
- package/dist/runtime/esbuild-wasm-bytes.d.ts +37 -18
- package/dist/runtime/esbuild-wasm-bytes.d.ts.map +1 -1
- package/dist/runtime/esbuild-wasm-bytes.js +66 -92
- package/dist/runtime/facet-loader-host.d.ts.map +1 -1
- package/dist/runtime/facet-loader-host.js +0 -7
- package/dist/runtime/git-bundle-artifact.d.ts +17 -0
- package/dist/runtime/git-bundle-artifact.d.ts.map +1 -0
- package/dist/runtime/git-bundle-artifact.js +25 -0
- package/dist/runtime/node-repl.d.ts.map +1 -1
- package/dist/runtime/node-repl.js +2 -13
- package/dist/runtime/node-shims-artifact.d.ts +22 -23
- package/dist/runtime/node-shims-artifact.d.ts.map +1 -1
- package/dist/runtime/node-shims-artifact.js +48 -78
- package/dist/runtime/node-shims.d.ts.map +1 -1
- package/dist/runtime/node-shims.js +2358 -996
- package/dist/runtime/opencode-artifact.d.ts.map +1 -1
- package/dist/runtime/opencode-artifact.js +17 -53
- package/dist/runtime/opencode-facet-runner.d.ts +3 -6
- package/dist/runtime/opencode-facet-runner.d.ts.map +1 -1
- package/dist/runtime/opencode-facet-runner.js +45 -10
- package/dist/runtime/opentui-wasm-bytes.d.ts.map +1 -1
- package/dist/runtime/opentui-wasm-bytes.js +16 -49
- package/dist/runtime/package-manager.d.ts +0 -2
- package/dist/runtime/package-manager.d.ts.map +1 -1
- package/dist/runtime/package-manager.js +0 -3
- package/dist/runtime/python-repl.d.ts.map +1 -1
- package/dist/runtime/python-repl.js +4 -35
- package/dist/runtime/repl-session.d.ts +10 -0
- package/dist/runtime/repl-session.d.ts.map +1 -1
- package/dist/runtime/repl-session.js +7 -0
- package/dist/runtime/ruby-repl.d.ts.map +1 -1
- package/dist/runtime/ruby-repl.js +2 -2
- package/dist/runtime/ruby-resident.d.ts.map +1 -1
- package/dist/runtime/ruby-resident.js +41 -51
- package/dist/runtime/runtime-catalog.d.ts +10 -3
- package/dist/runtime/runtime-catalog.d.ts.map +1 -1
- package/dist/runtime/runtime-catalog.js +125 -15
- package/dist/runtime/sqlite-wasm-bytes.d.ts +4 -4
- package/dist/runtime/sqlite-wasm-bytes.d.ts.map +1 -1
- package/dist/runtime/sqlite-wasm-bytes.js +21 -57
- package/dist/runtime/staged-source.d.ts +67 -0
- package/dist/runtime/staged-source.d.ts.map +1 -0
- package/dist/runtime/staged-source.js +113 -0
- package/dist/runtime/wasi/preamble.d.ts +0 -22
- package/dist/runtime/wasi/preamble.d.ts.map +1 -1
- package/dist/runtime/wasi/preamble.js +98 -1425
- package/dist/runtime-catalog.generated.js +1 -1
- package/dist/session/agent.d.ts +14 -2
- package/dist/session/agent.d.ts.map +1 -1
- package/dist/session/agent.js +57 -19
- package/dist/session/ai.d.ts +0 -11
- package/dist/session/ai.d.ts.map +1 -1
- package/dist/session/ai.js +0 -4
- package/dist/session/fs-watch.d.ts +0 -11
- package/dist/session/fs-watch.d.ts.map +1 -1
- package/dist/session/fs-watch.js +0 -21
- package/dist/session/hibernation.d.ts +68 -2
- package/dist/session/hibernation.d.ts.map +1 -1
- package/dist/session/hibernation.js +95 -1
- package/dist/session/init-phases.d.ts +1 -2
- package/dist/session/init-phases.d.ts.map +1 -1
- package/dist/session/init-phases.js +1 -2
- package/dist/session/init.d.ts.map +1 -1
- package/dist/session/init.js +7 -20
- package/dist/session/keys.d.ts +0 -3
- package/dist/session/keys.d.ts.map +1 -1
- package/dist/session/keys.js +0 -3
- package/dist/session/legacy-reset.d.ts +11 -0
- package/dist/session/legacy-reset.d.ts.map +1 -0
- package/dist/session/legacy-reset.js +16 -0
- package/dist/session/nimbus-session.d.ts +50 -16
- package/dist/session/nimbus-session.d.ts.map +1 -1
- package/dist/session/nimbus-session.js +77 -71
- package/dist/session/npm-install-port.d.ts.map +1 -1
- package/dist/session/npm-install-port.js +2 -0
- package/dist/session/programmatic.d.ts +13 -18
- package/dist/session/programmatic.d.ts.map +1 -1
- package/dist/session/programmatic.js +53 -39
- package/dist/session/replica-routes.d.ts +2 -3
- package/dist/session/replica-routes.d.ts.map +1 -1
- package/dist/session/replica-routes.js +3 -10
- package/dist/session/routes.d.ts.map +1 -1
- package/dist/session/routes.js +11 -12
- package/dist/session/rpc.d.ts +108 -21
- package/dist/session/rpc.d.ts.map +1 -1
- package/dist/session/rpc.js +194 -39
- package/dist/session/start-real-vite.js +2 -2
- package/dist/session/state-store.d.ts +1 -17
- package/dist/session/state-store.d.ts.map +1 -1
- package/dist/session/state-store.js +3 -53
- package/dist/session/supervisor-op.d.ts +7 -0
- package/dist/session/supervisor-op.d.ts.map +1 -1
- package/dist/session/supervisor-op.js +6 -1
- package/dist/session/supervisor-rpc.d.ts +116 -19
- package/dist/session/supervisor-rpc.d.ts.map +1 -1
- package/dist/session/supervisor-rpc.js +194 -84
- package/dist/session/vite-command.d.ts.map +1 -1
- package/dist/session/vite-command.js +17 -14
- package/dist/session/ws.d.ts.map +1 -1
- package/dist/session/ws.js +20 -8
- package/dist/shell/npm-bin-entrypoints.d.ts.map +1 -1
- package/dist/shell/npm-bin-entrypoints.js +30 -5
- package/dist/vfs/facet-resident-limits.d.ts +33 -0
- package/dist/vfs/facet-resident-limits.d.ts.map +1 -0
- package/dist/vfs/facet-resident-limits.js +33 -0
- package/dist/vfs/facet-resident-store.d.ts +2 -10
- package/dist/vfs/facet-resident-store.d.ts.map +1 -1
- package/dist/vfs/facet-resident-store.js +1594 -219
- package/dist/workspace-host.d.ts +3 -1
- package/dist/workspace-host.d.ts.map +1 -1
- package/dist/workspace-host.js +2 -0
- package/dist/wrangler/nimbus-wrangler.d.ts +0 -2
- package/dist/wrangler/nimbus-wrangler.d.ts.map +1 -1
- package/dist/wrangler/nimbus-wrangler.js +8 -22
- package/package.json +34 -7
- package/public/_assets/esbuild-0.24.2.js +2380 -0
- package/public/_assets/opencode/1.16.2/index-attach.js +2 -2
- package/public/_assets/runtime/esbuild-cli-c4fda380d52befc1.js +6892 -0
- package/public/_assets/runtime/git-de3e60c9f3ff006a.js +103 -0
- package/public/_assets/runtime/{node-shims-acaa037e6d26ec0d.js → node-shims-06565b121555c194.js} +2879 -992
- package/public/_assets/runtime/resident-store-6ffb1939c53d982f.js +2264 -0
- package/public/_assets/runtime/vfs-write-ledger-e501e8fc793bc2c1.js +889 -0
- package/scripts/build-opencode-attach-entry.mjs +51 -1
- package/scripts/bundle-esbuild-wasm.mjs +55 -39
- package/scripts/bundle-facet-workers.mjs +159 -10
- package/scripts/bundle-git.mjs +88 -27
- package/scripts/bundle-node-shims.mjs +72 -46
- package/scripts/bundle-runtime.mjs +18 -421
- package/scripts/cf-git-patch.mjs +137 -0
- package/scripts/patch-install-deps.mjs +49 -96
- package/scripts/runtime-specs.mjs +435 -0
- package/vendor/git-http.generated.d.ts +82 -0
- package/vendor/git-upstream.generated.d.ts +4378 -0
- package/vendor/git.LICENSE +7 -0
- package/vendor/git.generated.d.mts +21 -0
- package/vendor/git.generated.mjs +103 -0
- package/dist/replica/suspension.d.ts +0 -36
- package/dist/replica/suspension.d.ts.map +0 -1
- package/dist/replica/suspension.js +0 -49
- package/dist/runtime/static-server.d.ts +0 -3
- package/dist/runtime/static-server.d.ts.map +0 -1
- package/dist/runtime/static-server.js +0 -133
- package/scripts/clean-dist.mjs +0 -7
|
@@ -181,16 +181,8 @@
|
|
|
181
181
|
* \`__residentSynchronizeFromSupervisor\`.
|
|
182
182
|
*/
|
|
183
183
|
import { FS_LIST_PAGE_LIMIT, FS_READ_BATCH_PATH_LIMIT, FS_READ_BATCH_REQUEST_BYTES, } from '@nimbus-sh/core/constants.js';
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
*
|
|
187
|
-
* The measured ceiling is 2,199,981 bytes in a single value (bisected, with a
|
|
188
|
-
* 12-char key; the limit is on the row, so a longer path buys less). 1 MiB
|
|
189
|
-
* leaves better than 2x of headroom against a limit whose exact form is not
|
|
190
|
-
* contractual, and keeps the per-read allocation of a large file bounded.
|
|
191
|
-
* Files at or under it — effectively all of them — are one row and one query.
|
|
192
|
-
*/
|
|
193
|
-
export const RESIDENT_CHUNK_BYTES = 1_048_576;
|
|
184
|
+
import { FACET_OWN_WRITE_MEMORY_BYTES } from '@nimbus-sh/platform/limits.js';
|
|
185
|
+
import { RESIDENT_CHUNK_BYTES } from './facet-resident-limits.js';
|
|
194
186
|
/**
|
|
195
187
|
* Rows written before the materialiser yields.
|
|
196
188
|
*
|
|
@@ -217,6 +209,8 @@ export const RESIDENT_FILL_BATCH_BYTES = FS_READ_BATCH_REQUEST_BYTES;
|
|
|
217
209
|
* The in-facet store. Spliced into the generated module ahead of the shims, in
|
|
218
210
|
* the same way `VFS_WRITE_LEDGER_SOURCE` is (`facets/manager.ts`), because it
|
|
219
211
|
* has to close over the same module scope the shims read `__vfsBundle` from.
|
|
212
|
+
* Like the ledger and the shims it reaches the facet as a staged asset
|
|
213
|
+
* (scripts/bundle-node-shims.mjs), not through the Worker bundle.
|
|
220
214
|
*
|
|
221
215
|
* It presents a Proxy rather than a new API on purpose. `_bundleLookup`,
|
|
222
216
|
* `_writtenCell`, `__readFileOr`, `__fileExists` and the directory scans all
|
|
@@ -227,10 +221,11 @@ export const RESIDENT_FILL_BATCH_BYTES = FS_READ_BATCH_REQUEST_BYTES;
|
|
|
227
221
|
* rather than a new one.
|
|
228
222
|
*/
|
|
229
223
|
export const FACET_RESIDENT_STORE_SOURCE = `
|
|
230
|
-
// The
|
|
231
|
-
//
|
|
232
|
-
// that reads the
|
|
233
|
-
|
|
224
|
+
// The store's tables (__residentTablesOnSql or __residentTablesInMemory),
|
|
225
|
+
// bound once the process exists. Module scope has no ctx, so every trap below
|
|
226
|
+
// runs in handler context, which is also the only context that reads the
|
|
227
|
+
// resident set.
|
|
228
|
+
let __residentT = null;
|
|
234
229
|
let __residentReady = false;
|
|
235
230
|
|
|
236
231
|
/**
|
|
@@ -283,49 +278,246 @@ const __RESIDENT_MAX_LIST_PAGES = 4096;
|
|
|
283
278
|
* and the kind is stored rather than inferred so a text file whose bytes happen
|
|
284
279
|
* to be valid UTF-8 does not change shape between a write and a read.
|
|
285
280
|
*/
|
|
281
|
+
const __RESIDENT_SCHEMA = "2";
|
|
282
|
+
|
|
283
|
+
// N18: the session's storage ledger admits this facet's database growth
|
|
284
|
+
// before it happens. The supervisor grants an allowance (the ledger records
|
|
285
|
+
// it under this facet's name); the store never grows its database past
|
|
286
|
+
// \`__residentCap\`, and asks for more, asynchronously, when it would. What
|
|
287
|
+
// does not fit is not held: a fill degrades to reads through the session,
|
|
288
|
+
// and an own write goes to the session only. Null: no ledger (no cap).
|
|
289
|
+
let __residentCap = null;
|
|
290
|
+
let __residentFacet = null;
|
|
291
|
+
let __residentSupervisor = null;
|
|
292
|
+
let __residentGrantAsk = null;
|
|
293
|
+
const __RESIDENT_ROW_BYTES = 256;
|
|
294
|
+
function __namespaceRowBytes(parent, name, target) {
|
|
295
|
+
// JavaScript keeps path/target strings in the heap. Charge UTF-16 storage
|
|
296
|
+
// plus the existing row allowance, even when the engine can compress text.
|
|
297
|
+
return __RESIDENT_ROW_BYTES + 2 * (String(parent).length + String(name).length + (target == null ? 0 : String(target).length));
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function __residentDbBytes() {
|
|
301
|
+
// A one-shot's heap is one budget: the store's tables and the own writes
|
|
302
|
+
// held beside them (__residentHold) count against the same cap.
|
|
303
|
+
return __residentT.bytes() + (__residentInHeap ? __residentHeldBytes + __residentNamespaceOverlayBytes + __residentNamespaceReserveBytes : 0);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/** Whether the store is a one-shot's heap (__residentBindInMemory) rather than a facet's SQLite. */
|
|
307
|
+
let __residentInHeap = false;
|
|
308
|
+
let __residentNamespaceOverlayBytes = 0;
|
|
309
|
+
let __residentNamespaceReserveBytes = 0;
|
|
310
|
+
function __residentNamespaceOverlayDelta(bytes) {
|
|
311
|
+
if (!__residentInHeap) return;
|
|
312
|
+
if (bytes > 0 && __residentCap !== null && __residentDbBytes() + bytes > __residentCap) {
|
|
313
|
+
const cause = "namespace storage budget exceeded (" + __residentCap + " bytes)";
|
|
314
|
+
__nsMarkReady(__residentT, false, cause);
|
|
315
|
+
throw Object.assign(new Error(cause), { code: "ENOSPC" });
|
|
316
|
+
}
|
|
317
|
+
__residentNamespaceOverlayBytes += bytes;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Whether the store may grow by \`bytes\` now; when not, and \`ask\`, more room
|
|
322
|
+
* is asked for. A copy of an own write never asks: the session's copy is the
|
|
323
|
+
* one that must have the room, and the cache must not take it first.
|
|
324
|
+
*/
|
|
325
|
+
function __residentFits(bytes, ask = true) {
|
|
326
|
+
if (__residentCap === null) return true;
|
|
327
|
+
if (__residentDbBytes() + bytes <= __residentCap) return true;
|
|
328
|
+
if (ask) void __residentAskGrant(bytes);
|
|
329
|
+
return false;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Ask the ledger for \`bytes\` more, reporting what the database measures.
|
|
334
|
+
* Exactly what is needed: room granted and unused is room the session's own
|
|
335
|
+
* writes cannot have. Resolves to whether \`bytes\` now fit.
|
|
336
|
+
*/
|
|
337
|
+
function __residentAskGrant(bytes) {
|
|
338
|
+
const supervisor = __residentSupervisor;
|
|
339
|
+
if (__residentCap === null) return Promise.resolve(true);
|
|
340
|
+
// A store with no ledger row (a one-shot's heap) has a fixed budget: no one to ask.
|
|
341
|
+
if (__residentFacet === null || !supervisor || typeof supervisor.fsStorageGrant !== "function") return Promise.resolve(false);
|
|
342
|
+
if (__residentGrantAsk) return __residentGrantAsk.then(() => __residentDbBytes() + bytes <= __residentCap);
|
|
343
|
+
const want = Math.max(0, __residentDbBytes() + bytes - __residentCap);
|
|
344
|
+
__residentGrantAsk = (async () => {
|
|
345
|
+
try {
|
|
346
|
+
const answer = await supervisor.fsStorageGrant(__residentFacet, want, __residentDbBytes());
|
|
347
|
+
const granted = Number(answer && answer.granted) || 0;
|
|
348
|
+
__residentCap += granted;
|
|
349
|
+
if (answer && typeof answer.cap === "number") __residentCap = Math.max(__residentCap, answer.cap);
|
|
350
|
+
} catch {}
|
|
351
|
+
})().finally(() => { __residentGrantAsk = null; });
|
|
352
|
+
return __residentGrantAsk.then(() => __residentDbBytes() + bytes <= __residentCap);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/** Room for \`bytes\` before an asynchronous fill writes them: true when they fit. */
|
|
356
|
+
async function __residentEnsureRoom(bytes) {
|
|
357
|
+
if (__residentCap === null || __residentDbBytes() + bytes <= __residentCap) return true;
|
|
358
|
+
return __residentAskGrant(bytes);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* What holding a cell costs the database: its bytes (a text cell's UTF-8
|
|
363
|
+
* length) on overflow pages (1% over, for page headers and the b-tree pages
|
|
364
|
+
* above them), a row per chunk and head, and a page of slack. The manager
|
|
365
|
+
* charges a launch's module map the same way (manager.ts residentCellCost).
|
|
366
|
+
*/
|
|
367
|
+
function __residentCellCost(cell) {
|
|
368
|
+
const isText = typeof cell === "string";
|
|
369
|
+
const size = isText ? cell.length : __residentBytes(cell).byteLength;
|
|
370
|
+
const limit = isText ? Math.floor(__RESIDENT_CHUNK_BYTES / 3) : __RESIDENT_CHUNK_BYTES;
|
|
371
|
+
const chunks = Math.max(1, Math.ceil(size / limit));
|
|
372
|
+
return Math.ceil((isText ? __residentUtf8Length(cell) : size) * 1.01) + (chunks + 1) * __RESIDENT_ROW_BYTES + 4096;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
function __residentUtf8Length(text) {
|
|
376
|
+
let bytes = 0;
|
|
377
|
+
for (let i = 0; i < text.length; i++) {
|
|
378
|
+
const c = text.charCodeAt(i);
|
|
379
|
+
if (c < 0x80) bytes += 1;
|
|
380
|
+
else if (c < 0x800) bytes += 2;
|
|
381
|
+
// A surrogate pair is one 4-byte character; a lone surrogate encodes as 3.
|
|
382
|
+
else if (c >= 0xd800 && c < 0xdc00 && i + 1 < text.length && (text.charCodeAt(i + 1) & 0xfc00) === 0xdc00) { bytes += 4; i++; }
|
|
383
|
+
else bytes += 3;
|
|
384
|
+
}
|
|
385
|
+
return bytes;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/** Room for what an ACQUIRE answer pushes, asked for before it is applied. */
|
|
389
|
+
async function __residentRoomForPushed(result) {
|
|
390
|
+
if (__residentCap === null || !result || !Array.isArray(result.paths)) return;
|
|
391
|
+
let bytes = 0;
|
|
392
|
+
for (const entry of result.paths) {
|
|
393
|
+
if (entry && entry.bytes != null) bytes += __residentCellCost(__residentBytes(entry.bytes));
|
|
394
|
+
}
|
|
395
|
+
if (bytes > 0) await __residentEnsureRoom(bytes);
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** What the database measures, after a fill batch: the ledger settles to it, or records overshoot. */
|
|
399
|
+
async function __residentReportSize() {
|
|
400
|
+
const supervisor = __residentSupervisor;
|
|
401
|
+
if (__residentCap === null || __residentFacet === null || !supervisor || typeof supervisor.fsStorageGrant !== "function") return;
|
|
402
|
+
try { await supervisor.fsStorageGrant(__residentFacet, 0, __residentDbBytes()); } catch {}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/** The allowance this launch was admitted for (the fabric's startArgs.storage). */
|
|
406
|
+
function __residentSetStorage(storage, supervisor) {
|
|
407
|
+
__residentSupervisor = supervisor || null;
|
|
408
|
+
if (!storage || typeof storage.facet !== "string" || typeof storage.grant !== "number") return;
|
|
409
|
+
__residentFacet = storage.facet;
|
|
410
|
+
__residentCap = __residentDbBytes() + storage.grant;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** The cap this store enforces, for startProcess to report. */
|
|
414
|
+
function __residentStorageCap() {
|
|
415
|
+
return __residentCap;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// An own write the store could not hold (no room in the session's storage)
|
|
419
|
+
// is kept here, so the program still reads what it wrote: a bounded LRU
|
|
420
|
+
// (Map order is recency) of { cell, rev }, dated like a file row (own until
|
|
421
|
+
// its write-back is acknowledged, then the write's revision) and judged by
|
|
422
|
+
// every report like one. A path it could not keep either is remembered by
|
|
423
|
+
// name only, so a miss on it says why.
|
|
424
|
+
const __residentHeld = new Map();
|
|
425
|
+
let __residentHeldBytes = 0;
|
|
426
|
+
const __residentUnheld = new Set();
|
|
427
|
+
const __RESIDENT_HELD_MAX_BYTES = ${FACET_OWN_WRITE_MEMORY_BYTES};
|
|
428
|
+
|
|
429
|
+
function __residentCellSize(held) {
|
|
430
|
+
const cell = held.cell;
|
|
431
|
+
return typeof cell === "string" ? cell.length * 3 : __residentBytes(cell).byteLength;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function __residentForgetHeld(path) {
|
|
435
|
+
const cell = __residentHeld.get(path);
|
|
436
|
+
if (cell !== undefined) {
|
|
437
|
+
__residentHeld.delete(path);
|
|
438
|
+
__residentHeldBytes -= __residentCellSize(cell);
|
|
439
|
+
}
|
|
440
|
+
__residentUnheld.delete(path);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function __residentHold(path, cell) {
|
|
444
|
+
__residentForgetHeld(path);
|
|
445
|
+
const size = __residentCellSize({ cell });
|
|
446
|
+
// In a facet the held cells have their own bound. In a one-shot's heap they
|
|
447
|
+
// share the store's: what the cap leaves beside the tables.
|
|
448
|
+
const limit = __residentInHeap ? Math.max(0, Number(__residentCap) - __residentT.bytes() - __residentNamespaceOverlayBytes - __residentNamespaceReserveBytes) : __RESIDENT_HELD_MAX_BYTES;
|
|
449
|
+
if (size > limit) { __residentUnheld.add(path); return; }
|
|
450
|
+
while (__residentHeldBytes + size > limit) {
|
|
451
|
+
const [oldest, old] = __residentHeld.entries().next().value;
|
|
452
|
+
__residentHeld.delete(oldest);
|
|
453
|
+
__residentHeldBytes -= __residentCellSize(old);
|
|
454
|
+
__residentUnheld.add(oldest);
|
|
455
|
+
}
|
|
456
|
+
__residentHeld.set(path, { cell, rev: __RK_OWN_WRITE });
|
|
457
|
+
__residentHeldBytes += size;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/** Whether a miss on \`path\` is the storage's: its own write was held nowhere. */
|
|
461
|
+
function __residentStorageMiss(path) {
|
|
462
|
+
return __residentUnheld.has(path);
|
|
463
|
+
}
|
|
286
464
|
const __RK_TEXT = 0;
|
|
287
465
|
const __RK_BINARY = 1;
|
|
288
466
|
const __RK_DENIED = 2;
|
|
289
467
|
|
|
290
468
|
/**
|
|
291
|
-
* The revision of a cell this facet
|
|
469
|
+
* The revision of a cell holding this facet's own bytes that the authority has
|
|
470
|
+
* not acknowledged yet: a write that has not flushed, or a cell one of its own
|
|
471
|
+
* mutations holds while that mutation is in flight (\`__residentLease\`).
|
|
292
472
|
*
|
|
293
473
|
* Read-your-writes: strictly newer than anything the authority could report,
|
|
294
474
|
* so it is never evicted by a delta and never mistaken for something the
|
|
295
475
|
* authority vouched for. A distinct sentinel rather than NULL, because NULL
|
|
296
476
|
* would reintroduce the undated row the NOT NULL constraint exists to forbid.
|
|
477
|
+
*
|
|
478
|
+
* It is the ONE revision a delta cannot evict, so it must mean exactly that
|
|
479
|
+
* and nothing else. The acknowledgement dates the row (\`__residentStamp\`),
|
|
480
|
+
* and bytes the authority served are dated as they are installed
|
|
481
|
+
* (\`__residentFill\`). A row left here after either would hold its bytes
|
|
482
|
+
* against every later write by anyone, for the life of the store.
|
|
297
483
|
*/
|
|
298
484
|
const __RK_OWN_WRITE = -1;
|
|
299
485
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
486
|
+
/**
|
|
487
|
+
* The store's own operations, over the four tables it keeps: 'file' (one row
|
|
488
|
+
* per held path: kind, size, chunk count, revision, content key), 'chunk'
|
|
489
|
+
* (the bytes), 'meta' (schema, cursor, namespace readiness) and 'ns' (the
|
|
490
|
+
* namespace: a stat per visible name, keyed (parent, name)). Everything above
|
|
491
|
+
* this seam decides; the backing only stores. There are exactly two backings,
|
|
492
|
+
* with one contract:
|
|
493
|
+
*
|
|
494
|
+
* - __residentTablesOnSql: the process facet's own SQLite (ctx.storage.sql),
|
|
495
|
+
* durable across the facet's incarnations. Every statement the store has
|
|
496
|
+
* ever issued lives here, unchanged.
|
|
497
|
+
* - __residentTablesInMemory: a one-shot process's store, in its heap, for
|
|
498
|
+
* the life of the run (runOnce hosts no SQLite). Same rows, same answers,
|
|
499
|
+
* and a byte count that plays the database's size for the store's budget.
|
|
500
|
+
*
|
|
501
|
+
* Rows come back as plain objects with the column names as keys: file heads
|
|
502
|
+
* { kind, size, chunks }, chunks { txt, bin } (bin as the backing holds it;
|
|
503
|
+
* callers normalise with __residentBytes), ns rows with every stat column.
|
|
504
|
+
*/
|
|
505
|
+
function __residentTablesOnSql(sql) {
|
|
506
|
+
let schema = null;
|
|
507
|
+
for (const row of sql.exec("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'meta'")) schema = "";
|
|
508
|
+
if (schema !== null) for (const row of sql.exec("SELECT v FROM meta WHERE k = 'schema'")) schema = String(row.v);
|
|
509
|
+
if (schema !== __RESIDENT_SCHEMA) {
|
|
510
|
+
for (const table of ["chunk", "file", "ns", "meta"]) sql.exec("DROP TABLE IF EXISTS " + table);
|
|
307
511
|
}
|
|
308
|
-
// '
|
|
309
|
-
//
|
|
310
|
-
// what makes _statLadder and __fileExists cheap: the common case reads a
|
|
311
|
-
// short row and never pages a megabyte of blob in to answer "is it there".
|
|
312
|
-
// 'rev' is NOT NULL, and that constraint IS the coherence guarantee rather
|
|
313
|
-
// than a note about it. An undated row can never be invalidated, so it would
|
|
314
|
-
// be served stale forever; making the column nullable would leave the rule
|
|
315
|
-
// to be remembered at every insert, and the one that forgot would be
|
|
316
|
-
// indistinguishable from the ones that did not. Here the row simply cannot
|
|
317
|
-
// be written. This is the r2-cache posture — an unverifiable key cannot be
|
|
318
|
-
// constructed — moved to the only storage layer this store has.
|
|
319
|
-
//
|
|
320
|
-
// __RK_OWN_WRITE is the one negative value: this facet's own unflushed
|
|
321
|
-
// bytes, which are strictly newer than anything the authority can report and
|
|
322
|
-
// are always readable. It is a real provenance, not an absence of one.
|
|
512
|
+
// 'ckey' is the authority's content identity for the bytes (equal keys,
|
|
513
|
+
// equal bytes), when it said so; null for bytes it did not describe.
|
|
323
514
|
sql.exec(
|
|
324
515
|
"CREATE TABLE IF NOT EXISTS file (" +
|
|
325
516
|
"path TEXT PRIMARY KEY, kind INTEGER NOT NULL, size INTEGER NOT NULL, " +
|
|
326
|
-
"chunks INTEGER NOT NULL, rev INTEGER NOT NULL CHECK (rev >= -1)" +
|
|
517
|
+
"chunks INTEGER NOT NULL, rev INTEGER NOT NULL CHECK (rev >= -1), ckey TEXT" +
|
|
327
518
|
")"
|
|
328
519
|
);
|
|
520
|
+
sql.exec("CREATE INDEX IF NOT EXISTS file_ckey ON file (ckey) WHERE ckey IS NOT NULL");
|
|
329
521
|
sql.exec(
|
|
330
522
|
"CREATE TABLE IF NOT EXISTS chunk (" +
|
|
331
523
|
"path TEXT NOT NULL, part INTEGER NOT NULL, txt TEXT, bin BLOB, " +
|
|
@@ -335,9 +527,390 @@ function __residentBind(ctx) {
|
|
|
335
527
|
// The store's own cursor, in the same storage as the rows it describes, so
|
|
336
528
|
// the two cannot be separated by an isolate restart.
|
|
337
529
|
sql.exec("CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL)");
|
|
338
|
-
|
|
530
|
+
sql.exec("INSERT OR REPLACE INTO meta (k, v) VALUES ('schema', ?)", __RESIDENT_SCHEMA);
|
|
531
|
+
// The namespace: every name the process's credential can see, with its stat
|
|
532
|
+
// and no content. Keyed (parent, name) so a point lookup and a directory
|
|
533
|
+
// listing are both one index seek, and no path is stored twice.
|
|
534
|
+
sql.exec(
|
|
535
|
+
"CREATE TABLE IF NOT EXISTS ns (" +
|
|
536
|
+
"parent TEXT NOT NULL, name TEXT NOT NULL, kind INTEGER NOT NULL, size INTEGER NOT NULL, " +
|
|
537
|
+
"mode INTEGER NOT NULL, uid INTEGER NOT NULL, gid INTEGER NOT NULL, atime REAL NOT NULL, " +
|
|
538
|
+
"mtime REAL NOT NULL, ctime REAL NOT NULL, ino INTEGER NOT NULL, rev INTEGER NOT NULL, target TEXT, " +
|
|
539
|
+
"PRIMARY KEY (parent, name)" +
|
|
540
|
+
") WITHOUT ROWID"
|
|
541
|
+
);
|
|
542
|
+
const first = (rows) => { for (const row of rows) return row; return undefined; };
|
|
543
|
+
const ranged = (select, from, to, tail = "") => to === null
|
|
544
|
+
? sql.exec(select + " WHERE path >= ?" + tail, from)
|
|
545
|
+
: sql.exec(select + " WHERE path >= ? AND path < ?" + tail, from, to);
|
|
546
|
+
return {
|
|
547
|
+
bytes() {
|
|
548
|
+
const size = sql.databaseSize;
|
|
549
|
+
if (typeof size === "number" && size > 0) return size;
|
|
550
|
+
for (const row of sql.exec("SELECT page_count * page_size AS n FROM pragma_page_count(), pragma_page_size()")) return Number(row.n);
|
|
551
|
+
return 0;
|
|
552
|
+
},
|
|
553
|
+
databaseSize() { return Number(sql.databaseSize ?? 0); },
|
|
554
|
+
metaGet(k) { const row = first(sql.exec("SELECT v FROM meta WHERE k = ?", k)); return row === undefined ? undefined : String(row.v); },
|
|
555
|
+
metaSet(k, v) { sql.exec("INSERT OR REPLACE INTO meta (k, v) VALUES (?, ?)", k, v); },
|
|
556
|
+
fileHead(path) { return first(sql.exec("SELECT kind, size, chunks FROM file WHERE path = ?", path)); },
|
|
557
|
+
fileRev(path) { const row = first(sql.exec("SELECT rev FROM file WHERE path = ?", path)); return row === undefined ? undefined : Number(row.rev); },
|
|
558
|
+
fileRevKey(path) { return first(sql.exec("SELECT rev, ckey FROM file WHERE path = ?", path)); },
|
|
559
|
+
fileOwnSize(path) {
|
|
560
|
+
const row = first(sql.exec("SELECT size FROM file WHERE path = ? AND rev = ?", path, __RK_OWN_WRITE));
|
|
561
|
+
return row === undefined ? undefined : Number(row.size);
|
|
562
|
+
},
|
|
563
|
+
filePut(path, kind, size, chunks, rev, ckey) {
|
|
564
|
+
sql.exec("INSERT OR REPLACE INTO file (path, kind, size, chunks, rev, ckey) VALUES (?, ?, ?, ?, ?, ?)", path, kind, size, chunks, rev, ckey);
|
|
565
|
+
},
|
|
566
|
+
fileSetRev(path, rev) { sql.exec("UPDATE file SET rev = ? WHERE path = ?", rev, path); },
|
|
567
|
+
fileStampOwn(path, rev) { sql.exec("UPDATE file SET rev = ? WHERE path = ? AND rev = ?", rev, path, __RK_OWN_WRITE); },
|
|
568
|
+
fileDelete(path) { sql.exec("DELETE FROM file WHERE path = ?", path); },
|
|
569
|
+
fileByKey(ckey, notPath) {
|
|
570
|
+
return first(sql.exec("SELECT path, kind, size, chunks FROM file WHERE ckey = ? AND rev >= 0 AND path <> ? LIMIT 1", ckey, notPath));
|
|
571
|
+
},
|
|
572
|
+
filePaths(from, to) { return [...ranged("SELECT path FROM file", from, to)].map((row) => String(row.path)); },
|
|
573
|
+
fileAny(from, to) { return first(ranged("SELECT path FROM file", from, to, " LIMIT 1")) !== undefined; },
|
|
574
|
+
fileRevsIn(from, to) { return [...ranged("SELECT path, rev FROM file", from, to)]; },
|
|
575
|
+
// In key order, as every other scan here is: the primary key's index
|
|
576
|
+
// serves it, and both backings then answer in one order.
|
|
577
|
+
fileAllPaths() { return [...sql.exec("SELECT path FROM file ORDER BY path")].map((row) => String(row.path)); },
|
|
578
|
+
fileRows() { return [...sql.exec("SELECT path, rev, ckey FROM file ORDER BY path")]; },
|
|
579
|
+
fileOwnPaths() { return [...sql.exec("SELECT path FROM file WHERE rev = ? ORDER BY path", __RK_OWN_WRITE)].map((row) => String(row.path)); },
|
|
580
|
+
fileStats() { return first(sql.exec("SELECT count(*) AS n, coalesce(sum(size), 0) AS b FROM file")); },
|
|
581
|
+
chunkFirst(path) { return first(sql.exec("SELECT txt, bin FROM chunk WHERE path = ? AND part = 0", path)); },
|
|
582
|
+
chunkParts(path) { return [...sql.exec("SELECT txt, bin FROM chunk WHERE path = ? ORDER BY part", path)]; },
|
|
583
|
+
chunkAddText(path, part, txt) { sql.exec("INSERT INTO chunk (path, part, txt) VALUES (?, ?, ?)", path, part, txt); },
|
|
584
|
+
chunkAddBin(path, part, bin) { sql.exec("INSERT INTO chunk (path, part, bin) VALUES (?, ?, ?)", path, part, bin); },
|
|
585
|
+
chunkSetBin(path, part, bin) { sql.exec("INSERT OR REPLACE INTO chunk (path, part, bin) VALUES (?, ?, ?)", path, part, bin); },
|
|
586
|
+
chunkCopy(to, from) { sql.exec("INSERT INTO chunk (path, part, txt, bin) SELECT ?, part, txt, bin FROM chunk WHERE path = ?", to, from); },
|
|
587
|
+
chunkDelete(path) { sql.exec("DELETE FROM chunk WHERE path = ?", path); },
|
|
588
|
+
// Own rows are the one provenance a new incarnation cannot date.
|
|
589
|
+
dropOwn() {
|
|
590
|
+
sql.exec("DELETE FROM chunk WHERE path IN (SELECT path FROM file WHERE rev = ?)", __RK_OWN_WRITE);
|
|
591
|
+
sql.exec("DELETE FROM file WHERE rev = ?", __RK_OWN_WRITE);
|
|
592
|
+
},
|
|
593
|
+
// Every dated row, and any chunk no own row claims.
|
|
594
|
+
dropDated() {
|
|
595
|
+
sql.exec("DELETE FROM chunk WHERE path NOT IN (SELECT path FROM file WHERE rev = ?)", __RK_OWN_WRITE);
|
|
596
|
+
sql.exec("DELETE FROM file WHERE rev <> ?", __RK_OWN_WRITE);
|
|
597
|
+
},
|
|
598
|
+
// Every dated row and its chunks.
|
|
599
|
+
dropDatedFiles() {
|
|
600
|
+
sql.exec("DELETE FROM chunk WHERE path IN (SELECT path FROM file WHERE rev <> ?)", __RK_OWN_WRITE);
|
|
601
|
+
sql.exec("DELETE FROM file WHERE rev <> ?", __RK_OWN_WRITE);
|
|
602
|
+
},
|
|
603
|
+
clear() {
|
|
604
|
+
sql.exec("DELETE FROM chunk");
|
|
605
|
+
sql.exec("DELETE FROM file");
|
|
606
|
+
sql.exec("DELETE FROM ns");
|
|
607
|
+
sql.exec("DELETE FROM meta");
|
|
608
|
+
},
|
|
609
|
+
nsPut(parent, name, kind, size, mode, uid, gid, atime, mtime, ctime, ino, rev, target) {
|
|
610
|
+
sql.exec(
|
|
611
|
+
"INSERT OR REPLACE INTO ns (parent, name, kind, size, mode, uid, gid, atime, mtime, ctime, ino, rev, target) " +
|
|
612
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
613
|
+
parent, name, kind, size, mode, uid, gid, atime, mtime, ctime, ino, rev, target,
|
|
614
|
+
);
|
|
615
|
+
},
|
|
616
|
+
nsGet(parent, name) {
|
|
617
|
+
return first(sql.exec(
|
|
618
|
+
"SELECT kind, size, mode, uid, gid, atime, mtime, ctime, ino, rev, target FROM ns WHERE parent = ? AND name = ?",
|
|
619
|
+
parent, name,
|
|
620
|
+
));
|
|
621
|
+
},
|
|
622
|
+
nsDelete(parent, name) { sql.exec("DELETE FROM ns WHERE parent = ? AND name = ?", parent, name); },
|
|
623
|
+
nsDeleteChildren(parent) { sql.exec("DELETE FROM ns WHERE parent = ?", parent); },
|
|
624
|
+
nsDeleteParents(from, to) {
|
|
625
|
+
if (to === null) sql.exec("DELETE FROM ns WHERE parent >= ?", from);
|
|
626
|
+
else sql.exec("DELETE FROM ns WHERE parent >= ? AND parent < ?", from, to);
|
|
627
|
+
},
|
|
628
|
+
nsClear() { sql.exec("DELETE FROM ns"); },
|
|
629
|
+
nsKeys() { return [...sql.exec("SELECT parent, name FROM ns")]; },
|
|
630
|
+
nsOfKind(kind) { return [...sql.exec("SELECT parent, name FROM ns WHERE kind = ?", kind)]; },
|
|
631
|
+
nsChildren(parent) { return [...sql.exec("SELECT name, kind FROM ns WHERE parent = ?", parent)]; },
|
|
632
|
+
nsCount() { const row = first(sql.exec("SELECT count(*) AS n FROM ns")); return row === undefined ? 0 : Number(row.n); },
|
|
633
|
+
};
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function __residentBind(ctx) {
|
|
637
|
+
const sql = ctx && ctx.storage && ctx.storage.sql;
|
|
638
|
+
if (!sql || typeof sql.exec !== "function") {
|
|
639
|
+
throw new Error(
|
|
640
|
+
"Nimbus: this facet has no synchronous SQLite (ctx.storage.sql); " +
|
|
641
|
+
"the resident set cannot be served and every sync read would raise EAGAIN"
|
|
642
|
+
);
|
|
643
|
+
}
|
|
644
|
+
return __residentBindTables(__residentTablesOnSql(sql));
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* The one-shot backing: the same tables, in the process's heap, for the life
|
|
649
|
+
* of one run. A one-shot is hosted by runOnce, which gives it no SQLite, and
|
|
650
|
+
* it keeps nothing after it exits. Its answers are the SQLite backing's:
|
|
651
|
+
* scans come back in key order (a b-tree's), reads hand back copies (a
|
|
652
|
+
* program that writes into a Buffer it read must not change the store), and
|
|
653
|
+
* \`bytes()\` counts what it holds in the units __residentCellCost charges (a
|
|
654
|
+
* row, and the payload's bytes), so the store's one budget rule serves both.
|
|
655
|
+
*/
|
|
656
|
+
function __residentTablesInMemory() {
|
|
657
|
+
const files = new Map();
|
|
658
|
+
const chunks = new Map();
|
|
659
|
+
const meta = new Map();
|
|
660
|
+
const ns = new Map();
|
|
661
|
+
const byKey = new Map();
|
|
662
|
+
let sorted = null;
|
|
663
|
+
let used = 0;
|
|
664
|
+
let names = 0;
|
|
665
|
+
let namespaceBytes = 0;
|
|
666
|
+
const utf8 = (text) => __residentUtf8Length(text);
|
|
667
|
+
// SQLite's binary collation orders TEXT by its UTF-8 bytes: code-point
|
|
668
|
+
// order, which UTF-16 comparison is not past the BMP.
|
|
669
|
+
const cmp = (a, b) => {
|
|
670
|
+
const n = Math.min(a.length, b.length);
|
|
671
|
+
for (let i = 0; i < n; i++) {
|
|
672
|
+
const x = a.codePointAt(i), y = b.codePointAt(i);
|
|
673
|
+
if (x !== y) return x < y ? -1 : 1;
|
|
674
|
+
if (x > 0xffff) i++;
|
|
675
|
+
}
|
|
676
|
+
return a.length === b.length ? 0 : a.length < b.length ? -1 : 1;
|
|
677
|
+
};
|
|
678
|
+
const payload = (cell) => (cell.txt != null ? utf8(cell.txt) : cell.bin ? cell.bin.byteLength : 0);
|
|
679
|
+
const unkey = (path, row) => {
|
|
680
|
+
if (row && row.ckey != null) {
|
|
681
|
+
const set = byKey.get(row.ckey);
|
|
682
|
+
if (set) { set.delete(path); if (set.size === 0) byKey.delete(row.ckey); }
|
|
683
|
+
}
|
|
684
|
+
};
|
|
685
|
+
const paths = () => {
|
|
686
|
+
if (sorted === null) sorted = [...files.keys()].sort(cmp);
|
|
687
|
+
return sorted;
|
|
688
|
+
};
|
|
689
|
+
const lowerBound = (list, from) => {
|
|
690
|
+
let lo = 0, hi = list.length;
|
|
691
|
+
while (lo < hi) { const mid = (lo + hi) >>> 1; if (cmp(list[mid], from) < 0) lo = mid + 1; else hi = mid; }
|
|
692
|
+
return lo;
|
|
693
|
+
};
|
|
694
|
+
const inRange = (from, to) => {
|
|
695
|
+
const list = paths();
|
|
696
|
+
const out = [];
|
|
697
|
+
for (let i = lowerBound(list, from); i < list.length && (to === null || cmp(list[i], to) < 0); i++) out.push(list[i]);
|
|
698
|
+
return out;
|
|
699
|
+
};
|
|
700
|
+
const copy = (cell) => ({ txt: cell.txt ?? null, bin: cell.bin ? cell.bin.slice() : null });
|
|
701
|
+
const dropChunks = (path) => {
|
|
702
|
+
const parts = chunks.get(path);
|
|
703
|
+
if (!parts) return;
|
|
704
|
+
for (const cell of parts.values()) used -= __RESIDENT_ROW_BYTES + payload(cell);
|
|
705
|
+
chunks.delete(path);
|
|
706
|
+
};
|
|
707
|
+
const dropFile = (path) => {
|
|
708
|
+
const row = files.get(path);
|
|
709
|
+
if (row === undefined) return;
|
|
710
|
+
unkey(path, row);
|
|
711
|
+
files.delete(path);
|
|
712
|
+
sorted = null;
|
|
713
|
+
used -= __RESIDENT_ROW_BYTES;
|
|
714
|
+
};
|
|
715
|
+
const addChunk = (path, part, cell, replace) => {
|
|
716
|
+
let parts = chunks.get(path);
|
|
717
|
+
if (!parts) { parts = new Map(); chunks.set(path, parts); }
|
|
718
|
+
const had = parts.get(part);
|
|
719
|
+
if (had !== undefined) {
|
|
720
|
+
if (!replace) throw new Error("UNIQUE constraint failed: chunk.path, chunk.part");
|
|
721
|
+
used -= __RESIDENT_ROW_BYTES + payload(had);
|
|
722
|
+
}
|
|
723
|
+
parts.set(part, cell);
|
|
724
|
+
used += __RESIDENT_ROW_BYTES + payload(cell);
|
|
725
|
+
};
|
|
726
|
+
const nsRow = (parent, name) => { const dir = ns.get(parent); return dir ? dir.get(name) : undefined; };
|
|
727
|
+
const nsDrop = (parent, name) => {
|
|
728
|
+
const dir = ns.get(parent);
|
|
729
|
+
const row = dir?.get(name);
|
|
730
|
+
if (row === undefined) return;
|
|
731
|
+
dir.delete(name);
|
|
732
|
+
names--;
|
|
733
|
+
const bytes = __namespaceRowBytes(parent, name, row.target);
|
|
734
|
+
used -= bytes; namespaceBytes -= bytes;
|
|
735
|
+
if (dir.size === 0) ns.delete(parent);
|
|
736
|
+
};
|
|
737
|
+
const nsDropParent = (parent) => {
|
|
738
|
+
const dir = ns.get(parent);
|
|
739
|
+
if (!dir) return;
|
|
740
|
+
names -= dir.size;
|
|
741
|
+
for (const [name, row] of dir) {
|
|
742
|
+
const bytes = __namespaceRowBytes(parent, name, row.target);
|
|
743
|
+
used -= bytes; namespaceBytes -= bytes;
|
|
744
|
+
}
|
|
745
|
+
ns.delete(parent);
|
|
746
|
+
};
|
|
747
|
+
const byName = cmp;
|
|
748
|
+
const dropWhere = (keepFile, keepChunkPath) => {
|
|
749
|
+
for (const path of [...chunks.keys()]) if (!keepChunkPath(path)) dropChunks(path);
|
|
750
|
+
for (const [path, row] of [...files]) if (!keepFile(row)) dropFile(path);
|
|
751
|
+
};
|
|
752
|
+
return {
|
|
753
|
+
bytes() { return used; },
|
|
754
|
+
databaseSize() { return used; },
|
|
755
|
+
metaGet(k) { return meta.get(k); },
|
|
756
|
+
metaSet(k, v) { meta.set(k, String(v)); },
|
|
757
|
+
fileHead(path) { const row = files.get(path); return row === undefined ? undefined : { kind: row.kind, size: row.size, chunks: row.chunks }; },
|
|
758
|
+
fileRev(path) { const row = files.get(path); return row === undefined ? undefined : row.rev; },
|
|
759
|
+
fileRevKey(path) { const row = files.get(path); return row === undefined ? undefined : { rev: row.rev, ckey: row.ckey }; },
|
|
760
|
+
fileOwnSize(path) { const row = files.get(path); return row !== undefined && row.rev === __RK_OWN_WRITE ? row.size : undefined; },
|
|
761
|
+
filePut(path, kind, size, chunkCount, rev, ckey) {
|
|
762
|
+
if (!(rev >= -1)) throw new Error("CHECK constraint failed: rev >= -1");
|
|
763
|
+
const had = files.get(path);
|
|
764
|
+
if (had === undefined) { sorted = null; used += __RESIDENT_ROW_BYTES; } else unkey(path, had);
|
|
765
|
+
const row = { kind: Number(kind), size: Number(size), chunks: Number(chunkCount), rev: Number(rev), ckey: ckey == null ? null : String(ckey) };
|
|
766
|
+
files.set(path, row);
|
|
767
|
+
if (row.ckey !== null) { let set = byKey.get(row.ckey); if (!set) { set = new Set(); byKey.set(row.ckey, set); } set.add(path); }
|
|
768
|
+
},
|
|
769
|
+
fileSetRev(path, rev) { const row = files.get(path); if (row !== undefined) row.rev = Number(rev); },
|
|
770
|
+
fileStampOwn(path, rev) { const row = files.get(path); if (row !== undefined && row.rev === __RK_OWN_WRITE) row.rev = Number(rev); },
|
|
771
|
+
fileDelete(path) { dropFile(path); },
|
|
772
|
+
fileByKey(ckey, notPath) {
|
|
773
|
+
const set = byKey.get(String(ckey));
|
|
774
|
+
if (!set) return undefined;
|
|
775
|
+
for (const path of set) {
|
|
776
|
+
const row = files.get(path);
|
|
777
|
+
if (path !== notPath && row && row.rev >= 0) return { path, kind: row.kind, size: row.size, chunks: row.chunks };
|
|
778
|
+
}
|
|
779
|
+
return undefined;
|
|
780
|
+
},
|
|
781
|
+
filePaths(from, to) { return inRange(from, to); },
|
|
782
|
+
fileAny(from, to) { const list = paths(); const i = lowerBound(list, from); return i < list.length && (to === null || cmp(list[i], to) < 0); },
|
|
783
|
+
fileRevsIn(from, to) { return inRange(from, to).map((path) => ({ path, rev: files.get(path).rev })); },
|
|
784
|
+
fileAllPaths() { return [...paths()]; },
|
|
785
|
+
fileRows() { return paths().map((path) => { const row = files.get(path); return { path, rev: row.rev, ckey: row.ckey }; }); },
|
|
786
|
+
fileOwnPaths() { return paths().filter((path) => files.get(path).rev === __RK_OWN_WRITE); },
|
|
787
|
+
fileStats() { let b = 0; for (const row of files.values()) b += row.size; return { n: files.size, b }; },
|
|
788
|
+
chunkFirst(path) { const parts = chunks.get(path); const cell = parts && parts.get(0); return cell === undefined ? undefined : copy(cell); },
|
|
789
|
+
chunkParts(path) {
|
|
790
|
+
const parts = chunks.get(path);
|
|
791
|
+
if (!parts) return [];
|
|
792
|
+
return [...parts.keys()].sort((a, b) => a - b).map((part) => copy(parts.get(part)));
|
|
793
|
+
},
|
|
794
|
+
chunkAddText(path, part, txt) { addChunk(path, Number(part), { txt: String(txt), bin: null }, false); },
|
|
795
|
+
chunkAddBin(path, part, bin) { addChunk(path, Number(part), { txt: null, bin: __residentBytes(bin).slice() }, false); },
|
|
796
|
+
chunkSetBin(path, part, bin) { addChunk(path, Number(part), { txt: null, bin: __residentBytes(bin).slice() }, true); },
|
|
797
|
+
chunkCopy(to, from) {
|
|
798
|
+
const parts = chunks.get(from);
|
|
799
|
+
if (!parts) return;
|
|
800
|
+
// One statement in SQLite: a conflicting part fails the whole copy, and nothing lands.
|
|
801
|
+
const held = chunks.get(to);
|
|
802
|
+
if (held) for (const part of parts.keys()) if (held.has(part)) throw new Error("UNIQUE constraint failed: chunk.path, chunk.part");
|
|
803
|
+
for (const [part, cell] of [...parts]) addChunk(to, part, copy(cell), false);
|
|
804
|
+
},
|
|
805
|
+
chunkDelete(path) { dropChunks(path); },
|
|
806
|
+
dropOwn() {
|
|
807
|
+
const own = new Set(paths().filter((path) => files.get(path).rev === __RK_OWN_WRITE));
|
|
808
|
+
dropWhere((row) => row.rev !== __RK_OWN_WRITE, (path) => !own.has(path));
|
|
809
|
+
},
|
|
810
|
+
dropDated() {
|
|
811
|
+
const own = new Set(paths().filter((path) => files.get(path).rev === __RK_OWN_WRITE));
|
|
812
|
+
dropWhere((row) => row.rev === __RK_OWN_WRITE, (path) => own.has(path));
|
|
813
|
+
},
|
|
814
|
+
dropDatedFiles() {
|
|
815
|
+
const dated = new Set(paths().filter((path) => files.get(path).rev !== __RK_OWN_WRITE));
|
|
816
|
+
dropWhere((row) => row.rev === __RK_OWN_WRITE, (path) => !dated.has(path));
|
|
817
|
+
},
|
|
818
|
+
clear() {
|
|
819
|
+
files.clear(); chunks.clear(); meta.clear(); ns.clear(); byKey.clear();
|
|
820
|
+
sorted = null; used = 0; names = 0; namespaceBytes = 0;
|
|
821
|
+
},
|
|
822
|
+
nsPut(parent, name, kind, size, mode, uid, gid, atime, mtime, ctime, ino, rev, target) {
|
|
823
|
+
let dir = ns.get(parent);
|
|
824
|
+
const previous = dir?.get(name);
|
|
825
|
+
const bytes = __namespaceRowBytes(parent, name, target);
|
|
826
|
+
const growth = bytes - (previous === undefined ? 0 : __namespaceRowBytes(parent, name, previous.target));
|
|
827
|
+
if (__residentInHeap && __residentCap !== null && used + __residentHeldBytes + __residentNamespaceOverlayBytes + __residentNamespaceReserveBytes + growth > __residentCap) {
|
|
828
|
+
const cause = "namespace storage budget exceeded (" + __residentCap + " bytes)";
|
|
829
|
+
__nsMarkReady(__residentT, false, cause);
|
|
830
|
+
throw Object.assign(new Error(cause), { code: "ENOSPC" });
|
|
831
|
+
}
|
|
832
|
+
if (!dir) { dir = new Map(); ns.set(parent, dir); }
|
|
833
|
+
if (previous === undefined) names++;
|
|
834
|
+
used += growth; namespaceBytes += growth;
|
|
835
|
+
dir.set(name, {
|
|
836
|
+
kind: Number(kind), size: Number(size), mode: Number(mode), uid: Number(uid), gid: Number(gid),
|
|
837
|
+
atime: Number(atime), mtime: Number(mtime), ctime: Number(ctime), ino: Number(ino), rev: Number(rev),
|
|
838
|
+
target: target == null ? null : String(target),
|
|
839
|
+
});
|
|
840
|
+
},
|
|
841
|
+
nsGet(parent, name) { const row = nsRow(parent, name); return row === undefined ? undefined : { ...row }; },
|
|
842
|
+
nsDelete(parent, name) { nsDrop(parent, name); },
|
|
843
|
+
nsDeleteChildren(parent) { nsDropParent(parent); },
|
|
844
|
+
nsDeleteParents(from, to) {
|
|
845
|
+
for (const parent of [...ns.keys()]) if (cmp(parent, from) >= 0 && (to === null || cmp(parent, to) < 0)) nsDropParent(parent);
|
|
846
|
+
},
|
|
847
|
+
nsClear() { used -= namespaceBytes; names = 0; namespaceBytes = 0; ns.clear(); },
|
|
848
|
+
nsBytes() { return namespaceBytes; },
|
|
849
|
+
nsKeys() {
|
|
850
|
+
const out = [];
|
|
851
|
+
for (const parent of [...ns.keys()].sort(byName)) for (const name of [...ns.get(parent).keys()].sort(byName)) out.push({ parent, name });
|
|
852
|
+
return out;
|
|
853
|
+
},
|
|
854
|
+
nsOfKind(kind) {
|
|
855
|
+
const out = [];
|
|
856
|
+
for (const parent of [...ns.keys()].sort(byName)) {
|
|
857
|
+
const dir = ns.get(parent);
|
|
858
|
+
for (const name of [...dir.keys()].sort(byName)) if (dir.get(name).kind === Number(kind)) out.push({ parent, name });
|
|
859
|
+
}
|
|
860
|
+
return out;
|
|
861
|
+
},
|
|
862
|
+
nsChildren(parent) {
|
|
863
|
+
const dir = ns.get(parent);
|
|
864
|
+
if (!dir) return [];
|
|
865
|
+
return [...dir.keys()].sort(byName).map((name) => ({ name, kind: dir.get(name).kind }));
|
|
866
|
+
},
|
|
867
|
+
nsCount() { return names; },
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
/**
|
|
872
|
+
* A one-shot's store: the heap backing, capped at \`budget\` bytes and never
|
|
873
|
+
* growing past it (there is no ledger row to grow).
|
|
874
|
+
*/
|
|
875
|
+
function __residentBindInMemory(budget) {
|
|
876
|
+
const t = __residentBindTables(__residentTablesInMemory());
|
|
877
|
+
__residentInHeap = true;
|
|
878
|
+
__residentFacet = null;
|
|
879
|
+
__residentCap = Number(budget);
|
|
880
|
+
__residentNamespaceOverlayBytes = 0;
|
|
881
|
+
__residentNamespaceReserveBytes = 0;
|
|
882
|
+
return t;
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
/**
|
|
886
|
+
* Bind the store to its tables: a facet's SQLite (__residentBind), or a
|
|
887
|
+
* one-shot's heap (__residentTablesInMemory).
|
|
888
|
+
*
|
|
889
|
+
* 'file' carries one row per path, so existence, size and kind are one query
|
|
890
|
+
* that never touches content. 'chunk' carries the bytes. Splitting them is
|
|
891
|
+
* what makes _statLadder and __fileExists cheap: the common case reads a
|
|
892
|
+
* short row and never pages a megabyte of blob in to answer "is it there".
|
|
893
|
+
* 'rev' is never undated, and that IS the coherence guarantee rather than a
|
|
894
|
+
* note about it. An undated row can never be invalidated, so it would be
|
|
895
|
+
* served stale forever. The SQLite backing makes the column NOT NULL, so such
|
|
896
|
+
* a row cannot be written; this is the r2-cache posture (an unverifiable key
|
|
897
|
+
* cannot be constructed) moved to the only storage layer this store has.
|
|
898
|
+
*
|
|
899
|
+
* __RK_OWN_WRITE is the one negative value: this facet's own bytes that the
|
|
900
|
+
* authority has not acknowledged, which are strictly newer than anything it
|
|
901
|
+
* can report and are always readable. It is a real provenance, not an absence
|
|
902
|
+
* of one.
|
|
903
|
+
*/
|
|
904
|
+
function __residentBindTables(t) {
|
|
905
|
+
// A store can outlive its process. Own rows left by an earlier one are
|
|
906
|
+
// writes that never reached the authority or cells whose acknowledgement it
|
|
907
|
+
// never saw: neither can be dated, so neither is a file.
|
|
908
|
+
t.dropOwn();
|
|
909
|
+
__residentT = t;
|
|
339
910
|
__residentReady = true;
|
|
340
|
-
|
|
911
|
+
__nsLoadLinks(t);
|
|
912
|
+
__nsOk = t.metaGet("ns") === "1";
|
|
913
|
+
return t;
|
|
341
914
|
}
|
|
342
915
|
|
|
343
916
|
function __residentRequire() {
|
|
@@ -354,24 +927,21 @@ function __residentRequire() {
|
|
|
354
927
|
"the program runs; serving a row now would serve bytes of unknown age."
|
|
355
928
|
);
|
|
356
929
|
}
|
|
357
|
-
return
|
|
930
|
+
return __residentT;
|
|
358
931
|
}
|
|
359
932
|
|
|
360
933
|
/** The persisted cursor, or null when this store has never been populated. */
|
|
361
934
|
function __residentCursor() {
|
|
362
935
|
if (!__residentReady) return null;
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
else rev = Number(row.v);
|
|
367
|
-
}
|
|
368
|
-
return epoch === null || rev === null ? null : { epoch, rev };
|
|
936
|
+
const epoch = __residentT.metaGet("epoch");
|
|
937
|
+
const rev = __residentT.metaGet("rev");
|
|
938
|
+
return epoch === undefined || rev === undefined ? null : { epoch, rev: Number(rev) };
|
|
369
939
|
}
|
|
370
940
|
|
|
371
941
|
/** Stamp the store with the authority state its rows are known-good at. */
|
|
372
|
-
function __residentWriteCursor(
|
|
373
|
-
|
|
374
|
-
|
|
942
|
+
function __residentWriteCursor(t, cursor) {
|
|
943
|
+
t.metaSet("epoch", String(cursor.epoch));
|
|
944
|
+
t.metaSet("rev", String(Number(cursor.rev)));
|
|
375
945
|
}
|
|
376
946
|
|
|
377
947
|
/**
|
|
@@ -384,55 +954,148 @@ function __residentWriteCursor(sql, cursor) {
|
|
|
384
954
|
*
|
|
385
955
|
* Returns the paths that WERE held and are now gone, which is what
|
|
386
956
|
* \`_acquireAndRefetch\` re-reads live — the same contract \`_acquireBarrier\`
|
|
387
|
-
* already has
|
|
957
|
+
* already has — and how many named rows were kept because their revision
|
|
958
|
+
* already covers the report (this facet's own writes coming back).
|
|
388
959
|
*/
|
|
389
960
|
function __residentAdmit(result) {
|
|
390
961
|
if (!__residentReady) throw new Error("Nimbus: __residentAdmit before __residentBind");
|
|
391
|
-
const
|
|
962
|
+
const t = __residentT;
|
|
392
963
|
const dropped = [];
|
|
964
|
+
const relist = [];
|
|
965
|
+
let kept = 0, pushed = 0;
|
|
966
|
+
const epoch = result && result.epoch != null ? String(result.epoch) : null;
|
|
967
|
+
const rev = result && result.rev != null ? Number(result.rev) : null;
|
|
968
|
+
if (epoch === null || rev === null) {
|
|
969
|
+
throw new Error(
|
|
970
|
+
"Nimbus: fsAcquire returned no cursor; the resident store stays sealed " +
|
|
971
|
+
"rather than serving rows it cannot date"
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
// Answers can be admitted out of the order they were served in (two
|
|
975
|
+
// barriers in flight). One older than the store's cursor describes a state
|
|
976
|
+
// the store has already moved past: admitting it would move the cursor
|
|
977
|
+
// back and install its bytes over newer ones. Within an epoch the cursor
|
|
978
|
+
// only moves forward, and only entries above it are applied.
|
|
979
|
+
const at = __residentCursor();
|
|
980
|
+
const floor = at !== null && !__residentSealed && at.epoch === epoch && !(result && result.poison) ? at.rev : null;
|
|
981
|
+
if (floor !== null && rev <= floor) {
|
|
982
|
+
return { dropped, kept, pushed: 0, relist, cursor: at, stale: rev < floor };
|
|
983
|
+
}
|
|
984
|
+
// The namespace moves with the cursor or not at all: a delta without stats,
|
|
985
|
+
// or a poison, leaves it describing a revision the cursor has left, so it
|
|
986
|
+
// stops answering until a listing restores it.
|
|
987
|
+
// An answer naming no path changes no name, with or without stats.
|
|
988
|
+
const namespaced = __nsOk && !!result && !result.poison
|
|
989
|
+
&& (result.namespace === true || (Array.isArray(result.paths) && result.paths.length === 0))
|
|
990
|
+
&& (!Array.isArray(result.paths) || result.paths.every((entry) => entry.stat === null || __nsDescribes(entry.stat)));
|
|
991
|
+
if (!namespaced) {
|
|
992
|
+
__nsMarkReady(t, false, !result || result.poison ? "the session's change log was truncated past this process's cursor"
|
|
993
|
+
: "a change reported without the stat every name needs");
|
|
994
|
+
}
|
|
393
995
|
if (!result || result.poison) {
|
|
394
996
|
// A delta admission has no absolute listing to vouch for a row, so a
|
|
395
|
-
// poison here means nothing
|
|
396
|
-
// delete per path: a poison at pi scale is 7,141 rows, and eviction is on
|
|
397
|
-
// the hot path (measured 45 ms whole-store, vs a per-row walk).
|
|
997
|
+
// poison here means nothing the authority dated can be kept.
|
|
398
998
|
//
|
|
399
999
|
// This is the LAST resort, not the poison policy.
|
|
400
1000
|
// \`__residentSynchronizeFromSupervisor\` repairs the same poison against
|
|
401
1001
|
// absolute per-path revisions and keeps every row it can prove current;
|
|
402
1002
|
// dropping the store is what made a poison cost a whole filesystem.
|
|
403
|
-
|
|
404
|
-
sql.exec("DELETE FROM file");
|
|
1003
|
+
__residentDropDated();
|
|
405
1004
|
} else if (Array.isArray(result.paths)) {
|
|
406
1005
|
for (const entry of result.paths) {
|
|
1006
|
+
if (floor !== null && Number(entry.rev) <= floor) continue;
|
|
407
1007
|
const path = entry.path;
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
//
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
1008
|
+
if (namespaced && __nsReady()) {
|
|
1009
|
+
const again = __nsApplyEntry(t, entry);
|
|
1010
|
+
if (again !== null) relist.push(again);
|
|
1011
|
+
}
|
|
1012
|
+
// A row at or above the reported revision already holds that mutation —
|
|
1013
|
+
// this facet's own write coming back, or bytes fetched after it; anything
|
|
1014
|
+
// else is someone else's and is dropped. A row of own bytes
|
|
1015
|
+
// (__RK_OWN_WRITE) is newer than any authority revision by construction
|
|
1016
|
+
// and is kept — read-your-writes survives a peer's mutation of the same
|
|
1017
|
+
// path, and the report is adjudicated when the write's own revision
|
|
1018
|
+
// arrives (the shims note it for the ledger before admitting).
|
|
1019
|
+
// A subtree-scoped or structural entry covers every row at or under its
|
|
1020
|
+
// path, each judged the same way: something beneath it this facet may
|
|
1021
|
+
// not see changed, or the directory went, or who may enter it did. A
|
|
1022
|
+
// row there may be stale, or no longer this facet's to be served.
|
|
1023
|
+
if (entry.subtree || entry.structural) {
|
|
1024
|
+
const end = __residentPrefixEnd(path + "/");
|
|
1025
|
+
for (const row of t.fileRevsIn(path + "/", end)) {
|
|
1026
|
+
const stamped = Number(row.rev);
|
|
1027
|
+
if (stamped === __RK_OWN_WRITE || stamped >= Number(entry.rev)) continue;
|
|
1028
|
+
t.chunkDelete(row.path);
|
|
1029
|
+
t.fileDelete(row.path);
|
|
1030
|
+
dropped.push(String(row.path));
|
|
1031
|
+
}
|
|
1032
|
+
for (const [heldPath, heldCell] of [...__residentHeld]) {
|
|
1033
|
+
if (!heldPath.startsWith(path + "/") || heldCell.rev === __RK_OWN_WRITE || heldCell.rev >= Number(entry.rev)) continue;
|
|
1034
|
+
__residentForgetHeld(heldPath);
|
|
1035
|
+
}
|
|
1036
|
+
// Names under it may have become hidden or visible without being
|
|
1037
|
+
// named: the namespace relists that subtree.
|
|
1038
|
+
if (namespaced && entry.subtree && entry.stat && entry.stat.type === "directory") relist.push(path);
|
|
1039
|
+
}
|
|
1040
|
+
let held = false, stamped = 0, heldKey = null;
|
|
1041
|
+
const row = t.fileRevKey(path);
|
|
1042
|
+
if (row !== undefined) {
|
|
1043
|
+
held = true; stamped = Number(row.rev); heldKey = row.ckey == null ? null : String(row.ckey);
|
|
1044
|
+
}
|
|
1045
|
+
if (held && (stamped === __RK_OWN_WRITE || stamped >= Number(entry.rev))) { kept++; continue; }
|
|
1046
|
+
// A cell held in memory (an own write the store had no room for) is
|
|
1047
|
+
// judged the same way; superseded, it goes, and the report's bytes, if
|
|
1048
|
+
// any, are held as the store can.
|
|
1049
|
+
const heldCell = __residentHeld.get(path);
|
|
1050
|
+
if (heldCell !== undefined) {
|
|
1051
|
+
if (heldCell.rev === __RK_OWN_WRITE || heldCell.rev >= Number(entry.rev)) { kept++; continue; }
|
|
1052
|
+
__residentForgetHeld(path);
|
|
1053
|
+
}
|
|
1054
|
+
// The same bytes under a newer revision (a chmod, a touch, a rewrite
|
|
1055
|
+
// with identical content): kept, and dated at the report.
|
|
1056
|
+
if (held && heldKey !== null && entry.contentKey != null && heldKey === String(entry.contentKey)) {
|
|
1057
|
+
t.fileSetRev(path, Number(entry.rev));
|
|
1058
|
+
kept++;
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1061
|
+
// Pushed content: the file's bytes at the answer's revision, so dating
|
|
1062
|
+
// them at the path's own revision cannot be newer than they are.
|
|
1063
|
+
if (entry.bytes != null && entry.stat && entry.stat.type === "file") {
|
|
1064
|
+
if (__residentPut(t, path, __residentBytes(entry.bytes), Number(entry.rev), entry.contentKey)) pushed++;
|
|
1065
|
+
else if (held) dropped.push(path);
|
|
1066
|
+
continue;
|
|
416
1067
|
}
|
|
417
1068
|
if (!held) continue;
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
sql.exec("DELETE FROM file WHERE path = ?", path);
|
|
1069
|
+
t.chunkDelete(path);
|
|
1070
|
+
t.fileDelete(path);
|
|
421
1071
|
dropped.push(path);
|
|
422
1072
|
}
|
|
423
1073
|
}
|
|
424
|
-
|
|
425
|
-
const rev = result && result.rev != null ? Number(result.rev) : null;
|
|
426
|
-
if (epoch === null || rev === null) {
|
|
427
|
-
throw new Error(
|
|
428
|
-
"Nimbus: fsAcquire returned no cursor; the resident store stays sealed " +
|
|
429
|
-
"rather than serving rows it cannot date"
|
|
430
|
-
);
|
|
431
|
-
}
|
|
432
|
-
__residentWriteCursor(sql, { epoch, rev });
|
|
1074
|
+
__residentWriteCursor(t, { epoch, rev });
|
|
433
1075
|
__residentSealed = false;
|
|
434
1076
|
__residentSealReason = "";
|
|
435
|
-
return { dropped, cursor: { epoch, rev } };
|
|
1077
|
+
return { dropped, kept, pushed, relist, cursor: { epoch, rev } };
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
/**
|
|
1081
|
+
* Drop every row the authority dated, and nothing else: what a poison does to
|
|
1082
|
+
* the rows when nothing can vouch for them. The cursor is left alone, because
|
|
1083
|
+
* a store that holds no dated row cannot hold a stale one whatever its cursor
|
|
1084
|
+
* says.
|
|
1085
|
+
*
|
|
1086
|
+
* This facet's own bytes are not the authority's to vouch for, and stay:
|
|
1087
|
+
* dropping a row a write or mutation still in flight owns would hand the next
|
|
1088
|
+
* fill the pre-write bytes. Its acknowledgement dates it, or evicts it, when
|
|
1089
|
+
* it lands.
|
|
1090
|
+
*
|
|
1091
|
+
* Two statements rather than a delete per path: at pi scale that is 7,141
|
|
1092
|
+
* rows, and this is on the hot path (measured 45 ms whole-store, vs a per-row
|
|
1093
|
+
* walk).
|
|
1094
|
+
*/
|
|
1095
|
+
function __residentDropDated() {
|
|
1096
|
+
if (!__residentReady) throw new Error("Nimbus: __residentDropDated before __residentBind");
|
|
1097
|
+
__residentT.dropDated();
|
|
1098
|
+
for (const [path, held] of [...__residentHeld]) if (held.rev !== __RK_OWN_WRITE) __residentForgetHeld(path);
|
|
436
1099
|
}
|
|
437
1100
|
|
|
438
1101
|
/** Re-seal — for a store whose backing is being replaced (slot handover). */
|
|
@@ -443,13 +1106,7 @@ function __residentSeal(reason) {
|
|
|
443
1106
|
|
|
444
1107
|
/** One row of 'file', or undefined. The existence/shape query. */
|
|
445
1108
|
function __residentHead(path) {
|
|
446
|
-
|
|
447
|
-
return __residentHeadOn(sql, path);
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
function __residentHeadOn(sql, path) {
|
|
451
|
-
for (const row of sql.exec("SELECT kind, size, chunks FROM file WHERE path = ?", path)) return row;
|
|
452
|
-
return undefined;
|
|
1109
|
+
return __residentRequire().fileHead(path);
|
|
453
1110
|
}
|
|
454
1111
|
|
|
455
1112
|
/**
|
|
@@ -461,22 +1118,28 @@ function __residentHeadOn(sql, path) {
|
|
|
461
1118
|
* is not held.
|
|
462
1119
|
*/
|
|
463
1120
|
function __residentGet(path) {
|
|
1121
|
+
if (globalThis.__nimbusProfileStaged) globalThis.__nimbusProfileStaged.delete(String(path).replace(/^\\/+/, ""));
|
|
464
1122
|
const head = __residentHead(path);
|
|
465
|
-
if (head === undefined)
|
|
1123
|
+
if (head === undefined) {
|
|
1124
|
+
const held = __residentHeld.get(path);
|
|
1125
|
+
if (held === undefined) return undefined;
|
|
1126
|
+
__residentHeld.delete(path);
|
|
1127
|
+
__residentHeld.set(path, held);
|
|
1128
|
+
return held.cell;
|
|
1129
|
+
}
|
|
466
1130
|
const kind = Number(head.kind);
|
|
467
1131
|
if (kind === __RK_DENIED) return { error: "EACCES" };
|
|
468
|
-
const
|
|
1132
|
+
const t = __residentRequire();
|
|
469
1133
|
const chunks = Number(head.chunks);
|
|
470
1134
|
if (chunks === 1) {
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
return undefined;
|
|
1135
|
+
const row = t.chunkFirst(path);
|
|
1136
|
+
if (row === undefined) return undefined;
|
|
1137
|
+
return kind === __RK_TEXT ? row.txt : __residentBytes(row.bin);
|
|
475
1138
|
}
|
|
476
1139
|
// Chunked: a file past the single-value ceiling. Reassembled here so callers
|
|
477
1140
|
// never learn the file was split.
|
|
478
1141
|
const parts = [];
|
|
479
|
-
for (const row of
|
|
1142
|
+
for (const row of t.chunkParts(path)) {
|
|
480
1143
|
parts.push(kind === __RK_TEXT ? row.txt : __residentBytes(row.bin));
|
|
481
1144
|
}
|
|
482
1145
|
if (parts.length === 0) return undefined;
|
|
@@ -503,15 +1166,14 @@ function __residentBytes(value) {
|
|
|
503
1166
|
* (read-your-writes) and never mistaken for something the authority vouched
|
|
504
1167
|
* for. There is no third case: the column refuses one.
|
|
505
1168
|
*/
|
|
506
|
-
function __residentPut(
|
|
507
|
-
|
|
1169
|
+
function __residentPut(t, path, cell, rev, ckey) {
|
|
1170
|
+
// Whatever this puts supersedes a held cell, held or not.
|
|
1171
|
+
__residentForgetHeld(path);
|
|
1172
|
+
t.chunkDelete(path);
|
|
508
1173
|
const stamp = typeof rev === "number" ? rev : __RK_OWN_WRITE;
|
|
509
1174
|
if (cell && typeof cell === "object" && cell.error) {
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
path, __RK_DENIED, stamp
|
|
513
|
-
);
|
|
514
|
-
return;
|
|
1175
|
+
t.filePut(path, __RK_DENIED, 0, 0, stamp, null);
|
|
1176
|
+
return true;
|
|
515
1177
|
}
|
|
516
1178
|
const isText = typeof cell === "string";
|
|
517
1179
|
const kind = isText ? __RK_TEXT : __RK_BINARY;
|
|
@@ -521,20 +1183,26 @@ function __residentPut(sql, path, cell, rev) {
|
|
|
521
1183
|
// multi-byte string would otherwise cross it while measuring short.
|
|
522
1184
|
const limit = isText ? Math.floor(__RESIDENT_CHUNK_BYTES / 3) : __RESIDENT_CHUNK_BYTES;
|
|
523
1185
|
const chunks = Math.max(1, Math.ceil(size / limit));
|
|
1186
|
+
// Not admitted: nothing is held for the path (its old bytes are gone above).
|
|
1187
|
+
if (!__residentFits(__residentCellCost(cell), stamp !== __RK_OWN_WRITE)) {
|
|
1188
|
+
t.fileDelete(path);
|
|
1189
|
+
if (stamp === __RK_OWN_WRITE) __residentHold(path, cell);
|
|
1190
|
+
return false;
|
|
1191
|
+
}
|
|
524
1192
|
for (let part = 0; part < chunks; part++) {
|
|
525
1193
|
const slice = body.slice(part * limit, (part + 1) * limit);
|
|
526
|
-
if (isText)
|
|
527
|
-
else
|
|
1194
|
+
if (isText) t.chunkAddText(path, part, slice);
|
|
1195
|
+
else t.chunkAddBin(path, part, slice);
|
|
528
1196
|
}
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
path, kind, size, chunks, stamp
|
|
532
|
-
);
|
|
1197
|
+
t.filePut(path, kind, size, chunks, stamp, stamp === __RK_OWN_WRITE || ckey == null ? null : String(ckey));
|
|
1198
|
+
return true;
|
|
533
1199
|
}
|
|
534
1200
|
|
|
535
1201
|
/** One chunk row, written as it arrives. The streaming filler's primitive. */
|
|
536
|
-
function __residentPutChunk(
|
|
537
|
-
|
|
1202
|
+
function __residentPutChunk(t, path, part, bytes) {
|
|
1203
|
+
if (!__residentFits(Math.ceil(bytes.byteLength * 1.01) + __RESIDENT_ROW_BYTES + 4096)) return false;
|
|
1204
|
+
t.chunkSetBin(path, part, bytes);
|
|
1205
|
+
return true;
|
|
538
1206
|
}
|
|
539
1207
|
|
|
540
1208
|
/**
|
|
@@ -546,63 +1214,125 @@ function __residentPutChunk(sql, path, part, bytes) {
|
|
|
546
1214
|
* a large file leaves chunks nothing can read, never a short file something
|
|
547
1215
|
* can. Writing the head first would invert that into a truncated read.
|
|
548
1216
|
*/
|
|
549
|
-
function __residentPutHead(
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
1217
|
+
function __residentPutHead(t, path, kind, size, chunks, rev, ckey) {
|
|
1218
|
+
t.filePut(
|
|
1219
|
+
path, kind, size, chunks, typeof rev === "number" ? rev : __RK_OWN_WRITE,
|
|
1220
|
+
typeof rev === "number" && ckey != null ? String(ckey) : null,
|
|
553
1221
|
);
|
|
554
1222
|
}
|
|
555
1223
|
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
1224
|
+
/**
|
|
1225
|
+
* Hold \`path\` with the bytes of a dated row whose content identity is
|
|
1226
|
+
* \`ckey\`, by copying rows inside this store: equal keys are equal bytes, so
|
|
1227
|
+
* no fetch is needed. Returns whether a source existed.
|
|
1228
|
+
*/
|
|
1229
|
+
function __residentCopyByKey(t, path, ckey, rev) {
|
|
1230
|
+
const source = t.fileByKey(String(ckey), path);
|
|
1231
|
+
if (source === undefined) return false;
|
|
1232
|
+
if (!__residentFits(Number(source.size) + (Number(source.chunks) + 1) * __RESIDENT_ROW_BYTES)) return false;
|
|
1233
|
+
t.chunkDelete(path);
|
|
1234
|
+
t.chunkCopy(path, String(source.path));
|
|
1235
|
+
t.filePut(path, Number(source.kind), Number(source.size), Number(source.chunks), rev, String(ckey));
|
|
1236
|
+
return true;
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
function __residentDelete(t, path) {
|
|
1240
|
+
const had = __residentHead(path) !== undefined || __residentHeld.has(path);
|
|
1241
|
+
__residentForgetHeld(path);
|
|
1242
|
+
t.chunkDelete(path);
|
|
1243
|
+
t.fileDelete(path);
|
|
560
1244
|
return had;
|
|
561
1245
|
}
|
|
562
1246
|
|
|
563
1247
|
/**
|
|
564
|
-
*
|
|
565
|
-
*
|
|
566
|
-
* store is for — the seal stops READS of undated rows, not the writes that date
|
|
567
|
-
* them.
|
|
1248
|
+
* What a row is dated at: its revision, __RK_OWN_WRITE for this facet's own
|
|
1249
|
+
* unacknowledged bytes, or undefined when the path is not held.
|
|
568
1250
|
*/
|
|
569
|
-
function
|
|
570
|
-
|
|
571
|
-
if (typeof rev !== "number") {
|
|
572
|
-
throw new Error(
|
|
573
|
-
"Nimbus: refusing to populate '" + path + "' with no authority revision. " +
|
|
574
|
-
"An undated row cannot be invalidated, so it would be served stale forever."
|
|
575
|
-
);
|
|
576
|
-
}
|
|
577
|
-
__residentPut(__residentSql, path, cell, rev);
|
|
1251
|
+
function __residentProvenance(path) {
|
|
1252
|
+
return __residentRequire().fileRev(path);
|
|
578
1253
|
}
|
|
579
1254
|
|
|
580
1255
|
/**
|
|
581
|
-
*
|
|
582
|
-
*
|
|
583
|
-
*
|
|
584
|
-
*
|
|
585
|
-
*
|
|
586
|
-
*
|
|
1256
|
+
* Date this facet's own bytes with the revision the authority acknowledged
|
|
1257
|
+
* them at: the revision a flushed write produced, or the one an own mutation's
|
|
1258
|
+
* receipt settles on. The store's half of the stamp \`__vfsBundleRevisions\`
|
|
1259
|
+
* carries in heap (_shared/vfs-write-ledger.ts), kept in the row it describes
|
|
1260
|
+
* so the two cannot drift.
|
|
1261
|
+
*
|
|
1262
|
+
* Only a row still holding own bytes is dated. One that was dropped or
|
|
1263
|
+
* replaced meanwhile holds something this acknowledgement says nothing about.
|
|
1264
|
+
*
|
|
1265
|
+
* Without it nothing dates an own row, and __RK_OWN_WRITE is the revision a
|
|
1266
|
+
* delta never evicts: a file this process wrote would serve its own bytes over
|
|
1267
|
+
* every later write to it by anyone.
|
|
587
1268
|
*/
|
|
588
1269
|
function __residentStamp(path, rev) {
|
|
589
|
-
|
|
1270
|
+
const heldCell = __residentHeld.get(path);
|
|
1271
|
+
if (heldCell !== undefined && heldCell.rev === __RK_OWN_WRITE) heldCell.rev = rev;
|
|
1272
|
+
const t = __residentRequire();
|
|
1273
|
+
const size = t.fileOwnSize(path);
|
|
1274
|
+
if (size === undefined) return;
|
|
1275
|
+
t.fileStampOwn(path, rev);
|
|
1276
|
+
__nsNoteOwnFile(path, size, rev);
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
/**
|
|
1280
|
+
* Hold a dated row as this facet's own for the window of one of its own
|
|
1281
|
+
* mutations (a ranged write, a truncate, a chmod), and return the revision it
|
|
1282
|
+
* was dated at: the mutation's receipt is judged against it when the window
|
|
1283
|
+
* closes (\`__nimbusEndOwnMutation\`). Undefined when there is no dated row —
|
|
1284
|
+
* an own row is held already, and an absent one has nothing to protect.
|
|
1285
|
+
*
|
|
1286
|
+
* The hold is what keeps a barrier that reports the mutation before its own
|
|
1287
|
+
* response arrives from evicting the cell the mutation is about to overlay.
|
|
1288
|
+
* The supervisor answers fsAcquire at once, while a write's response waits on
|
|
1289
|
+
* the Durable Object's output gate, so that ordering is ordinary.
|
|
1290
|
+
*/
|
|
1291
|
+
function __residentLease(path) {
|
|
1292
|
+
const t = __residentRequire();
|
|
1293
|
+
const rev = t.fileRev(path);
|
|
1294
|
+
if (rev === undefined || rev === __RK_OWN_WRITE) return undefined;
|
|
1295
|
+
t.fileSetRev(path, __RK_OWN_WRITE);
|
|
1296
|
+
return rev;
|
|
590
1297
|
}
|
|
591
1298
|
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
1299
|
+
/**
|
|
1300
|
+
* Install bytes a live read returned, dated \`rev\`: the cursor the read was
|
|
1301
|
+
* issued under, so the bytes hold every mutation at or below it. An async read
|
|
1302
|
+
* must write through, or the next synchronous read goes back in time relative
|
|
1303
|
+
* to what the program was just handed (protocol §5.9).
|
|
1304
|
+
*
|
|
1305
|
+
* Refused over a row of own bytes, which are newer than anything a read can
|
|
1306
|
+
* return, and over a row dated after \`rev\`, which is newer than this read can
|
|
1307
|
+
* vouch for. Returns whether the bytes were installed.
|
|
1308
|
+
*/
|
|
1309
|
+
function __residentFill(path, cell, rev) {
|
|
1310
|
+
if (typeof rev !== "number" || rev < 0) {
|
|
1311
|
+
throw new Error(
|
|
1312
|
+
"Nimbus: refusing to fill '" + path + "' with no authority revision. " +
|
|
1313
|
+
"An undated row cannot be invalidated, so it would be served stale forever."
|
|
1314
|
+
);
|
|
597
1315
|
}
|
|
598
|
-
|
|
1316
|
+
const t = __residentRequire();
|
|
1317
|
+
const held = t.fileRev(path);
|
|
1318
|
+
if (held !== undefined && (held === __RK_OWN_WRITE || held > rev)) return false;
|
|
1319
|
+
const heldCell = __residentHeld.get(path);
|
|
1320
|
+
if (heldCell !== undefined && (heldCell.rev === __RK_OWN_WRITE || heldCell.rev > rev)) return false;
|
|
1321
|
+
return __residentPut(t, path, cell, rev);
|
|
599
1322
|
}
|
|
600
1323
|
|
|
601
1324
|
/** Every held path. Backs the Object.keys / for-in scans in the shims. */
|
|
602
1325
|
function __residentKeys() {
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
1326
|
+
return __residentRequire().fileAllPaths();
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
/**
|
|
1330
|
+
* Every path held as this facet's own unacknowledged bytes: the rows a
|
|
1331
|
+
* poison's repair keeps without a revision to vouch for them, whose reports
|
|
1332
|
+
* the shims deliver to their owners (\`_settleSkippedReports\`).
|
|
1333
|
+
*/
|
|
1334
|
+
function __residentOwnPaths() {
|
|
1335
|
+
return __residentRequire().fileOwnPaths();
|
|
606
1336
|
}
|
|
607
1337
|
|
|
608
1338
|
/**
|
|
@@ -640,12 +1370,7 @@ function __residentPrefixEnd(prefix) {
|
|
|
640
1370
|
function __residentKeysUnder(prefix) {
|
|
641
1371
|
const from = String(prefix);
|
|
642
1372
|
const to = __residentPrefixEnd(from);
|
|
643
|
-
|
|
644
|
-
const rows = to === null
|
|
645
|
-
? __residentRequire().exec("SELECT path FROM file WHERE path >= ?", from)
|
|
646
|
-
: __residentRequire().exec("SELECT path FROM file WHERE path >= ? AND path < ?", from, to);
|
|
647
|
-
for (const row of rows) out.push(String(row.path));
|
|
648
|
-
return out;
|
|
1373
|
+
return __residentRequire().filePaths(from, to);
|
|
649
1374
|
}
|
|
650
1375
|
|
|
651
1376
|
/**
|
|
@@ -657,11 +1382,310 @@ function __residentKeysUnder(prefix) {
|
|
|
657
1382
|
function __residentHasUnder(prefix) {
|
|
658
1383
|
const from = String(prefix);
|
|
659
1384
|
const to = __residentPrefixEnd(from);
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
1385
|
+
return __residentRequire().fileAny(from, to);
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
// ── The namespace ───────────────────────────────────────────────────────────
|
|
1389
|
+
//
|
|
1390
|
+
// Every name the credential can see, exact at the store's cursor: filled from
|
|
1391
|
+
// a complete listing, then moved forward by each ACQUIRE delta, whose entries
|
|
1392
|
+
// carry the path's stat at the answer's revision (null when it is gone). A
|
|
1393
|
+
// change of presence or stat is always a logged mutation of that path, except
|
|
1394
|
+
// what vanishes beneath a removed or replaced directory, which the delta
|
|
1395
|
+
// application drops by subtree. So a synchronous stat, exists or readdir is
|
|
1396
|
+
// answered exactly, and an absent name is known to be absent.
|
|
1397
|
+
//
|
|
1398
|
+
// This table holds the AUTHORITY's view only. The process's own pending
|
|
1399
|
+
// structural effects are an overlay kept by the shims, retired once a barrier
|
|
1400
|
+
// begun after they settled has applied.
|
|
1401
|
+
|
|
1402
|
+
const __NS_FILE = 0;
|
|
1403
|
+
const __NS_DIR = 1;
|
|
1404
|
+
const __NS_LINK = 2;
|
|
1405
|
+
/** Symlink hops before a resolution is ELOOP, as Linux's MAXSYMLINKS. */
|
|
1406
|
+
const __NS_MAX_HOPS = 40;
|
|
1407
|
+
/** Paths of every symlink in the table: resolution walks only past these. */
|
|
1408
|
+
let __nsLinks = new Set();
|
|
1409
|
+
/** The credential names are judged traversable for. */
|
|
1410
|
+
let __nsCred = null;
|
|
1411
|
+
|
|
1412
|
+
function __nsSetCred(cred) { __nsCred = cred || null; }
|
|
1413
|
+
|
|
1414
|
+
function __nsSplit(k) {
|
|
1415
|
+
const i = k.lastIndexOf("/");
|
|
1416
|
+
return i < 0 ? ["", k] : [k.slice(0, i), k.slice(i + 1)];
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
function __nsKindCode(type) {
|
|
1420
|
+
return type === "directory" ? __NS_DIR : type === "symlink" ? __NS_LINK : __NS_FILE;
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
function __nsLoadLinks(t) {
|
|
1424
|
+
__nsLinks = new Set();
|
|
1425
|
+
for (const row of t.nsOfKind(__NS_LINK)) {
|
|
1426
|
+
const parent = String(row.parent);
|
|
1427
|
+
__nsLinks.add(parent ? parent + "/" + String(row.name) : String(row.name));
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
/** Whether the table describes the store's cursor. */
|
|
1432
|
+
function __nsReady() {
|
|
1433
|
+
return __residentReady && !__residentSealed && __nsOk;
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
/** The persisted 'ns' flag, mirrored in heap because every stat asks it. */
|
|
1437
|
+
let __nsOk = false;
|
|
1438
|
+
/** Why the namespace stopped answering, while it does not: what a refused sync call names. */
|
|
1439
|
+
let __nsCause = "the process has not listed it yet";
|
|
1440
|
+
|
|
1441
|
+
function __nsMarkReady(t, ready, cause) {
|
|
1442
|
+
t.metaSet("ns", ready ? "1" : "0");
|
|
1443
|
+
__nsOk = !!ready;
|
|
1444
|
+
if (!ready) __nsCause = String(cause || "a change it could not follow");
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1447
|
+
/** Why the namespace is not answering (meaningful only while __nsReady() is false). */
|
|
1448
|
+
function __nsNotReadyCause() {
|
|
1449
|
+
if (!__residentReady) return "the process's store is not bound";
|
|
1450
|
+
if (__residentSealed) return __residentSealReason || "the store has not reconciled with the session";
|
|
1451
|
+
return __nsCause;
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
/**
|
|
1455
|
+
* Whether a stat from the authority describes its name: a type, and the
|
|
1456
|
+
* mode, owner and group every permission check reads. One without them is
|
|
1457
|
+
* no record. Stored, its missing fields would become zeros, a root-owned
|
|
1458
|
+
* mode-0 file the authority never reported. The namespace then stops
|
|
1459
|
+
* answering (as for a delta without stats) until a listing that does
|
|
1460
|
+
* describe every name restores it.
|
|
1461
|
+
*/
|
|
1462
|
+
function __nsDescribes(stat) {
|
|
1463
|
+
if (!stat || typeof stat.type !== "string") return false;
|
|
1464
|
+
for (const field of ["mode", "uid", "gid"]) {
|
|
1465
|
+
if (stat[field] === null || stat[field] === undefined || !Number.isInteger(Number(stat[field]))) return false;
|
|
1466
|
+
}
|
|
1467
|
+
return true;
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
function __nsPut(t, k, stat, rev, target) {
|
|
1471
|
+
const [parent, name] = __nsSplit(k);
|
|
1472
|
+
const kind = __nsKindCode(stat.type);
|
|
1473
|
+
t.nsPut(
|
|
1474
|
+
parent, name, kind, Number(stat.size) || 0, Number(stat.mode) || 0, Number(stat.uid) || 0,
|
|
1475
|
+
Number(stat.gid) || 0, Number(stat.atime) || 0, Number(stat.mtime) || 0, Number(stat.ctime) || 0,
|
|
1476
|
+
Number(stat.ino) || 0, Number(rev) || 0, kind === __NS_LINK ? String(target ?? "") : null,
|
|
1477
|
+
);
|
|
1478
|
+
if (kind === __NS_LINK) __nsLinks.add(k); else __nsLinks.delete(k);
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
// The authority operation may succeed while metadata cannot be cached.
|
|
1482
|
+
// nsPut has sealed the sync view on heap ENOSPC; preserve the async result.
|
|
1483
|
+
function __nsTryPut(t, k, stat, rev, target) {
|
|
1484
|
+
try { __nsPut(t, k, stat, rev, target); return true; }
|
|
1485
|
+
catch (error) {
|
|
1486
|
+
if (__residentInHeap && error && error.code === "ENOSPC") return false;
|
|
1487
|
+
throw error;
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
/** Delete everything beneath \`k\`, and \`k\` itself when \`self\`. */
|
|
1492
|
+
function __nsDeleteTree(t, k, self) {
|
|
1493
|
+
if (k === "") {
|
|
1494
|
+
t.nsClear();
|
|
1495
|
+
__nsLinks = new Set();
|
|
1496
|
+
return;
|
|
1497
|
+
}
|
|
1498
|
+
const prefix = k + "/";
|
|
1499
|
+
const end = __residentPrefixEnd(prefix);
|
|
1500
|
+
t.nsDeleteChildren(k);
|
|
1501
|
+
t.nsDeleteParents(prefix, end);
|
|
1502
|
+
if (self) {
|
|
1503
|
+
const [parent, name] = __nsSplit(k);
|
|
1504
|
+
t.nsDelete(parent, name);
|
|
1505
|
+
__nsLinks.delete(k);
|
|
1506
|
+
}
|
|
1507
|
+
for (const link of __nsLinks) if (link.startsWith(prefix)) __nsLinks.delete(link);
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
/** The row at exactly \`k\` (no symlink resolution), or undefined. */
|
|
1511
|
+
function __nsRowAt(t, k) {
|
|
1512
|
+
if (k === "") {
|
|
1513
|
+
return { kind: __NS_DIR, size: 0, mode: 0o40755, uid: 0, gid: 0, atime: 0, mtime: 0, ctime: 0, ino: 1, rev: 0, target: null };
|
|
1514
|
+
}
|
|
1515
|
+
const [parent, name] = __nsSplit(k);
|
|
1516
|
+
return t.nsGet(parent, name);
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
function __nsJoinTarget(linkPath, target) {
|
|
1520
|
+
const t = String(target);
|
|
1521
|
+
const parts = (t.startsWith("/") ? [] : __nsSplit(linkPath)[0].split("/").filter(Boolean));
|
|
1522
|
+
for (const seg of t.split("/")) {
|
|
1523
|
+
if (seg === "" || seg === ".") continue;
|
|
1524
|
+
if (seg === "..") parts.pop(); else parts.push(seg);
|
|
1525
|
+
}
|
|
1526
|
+
return parts.join("/");
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
/**
|
|
1530
|
+
* Resolve \`k\` through the symlinks the namespace holds: { path, row } for the
|
|
1531
|
+
* name it denotes, null when nothing is there, or "ELOOP". The leaf is
|
|
1532
|
+
* followed only when \`followLeaf\`. Walks component by component only when a
|
|
1533
|
+
* symlink sits on the path; otherwise it is one lookup.
|
|
1534
|
+
*/
|
|
1535
|
+
function __nsResolve(k, followLeaf) {
|
|
1536
|
+
const t = __residentRequire();
|
|
1537
|
+
let path = k;
|
|
1538
|
+
for (let hops = 0; hops <= __NS_MAX_HOPS; hops++) {
|
|
1539
|
+
let redirected = false;
|
|
1540
|
+
if (__nsLinks.size > 0) {
|
|
1541
|
+
const segs = path.split("/");
|
|
1542
|
+
for (let i = 1; i <= segs.length; i++) {
|
|
1543
|
+
const prefix = segs.slice(0, i).join("/");
|
|
1544
|
+
if (!__nsLinks.has(prefix)) continue;
|
|
1545
|
+
if (i === segs.length && !followLeaf) break;
|
|
1546
|
+
const row = __nsRowAt(t, prefix);
|
|
1547
|
+
if (!row || Number(row.kind) !== __NS_LINK) break;
|
|
1548
|
+
const rest = segs.slice(i).join("/");
|
|
1549
|
+
const target = __nsJoinTarget(prefix, row.target);
|
|
1550
|
+
path = rest ? (target ? target + "/" + rest : rest) : target;
|
|
1551
|
+
redirected = true;
|
|
1552
|
+
break;
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
if (redirected) continue;
|
|
1556
|
+
const row = __nsRowAt(t, path);
|
|
1557
|
+
return row === undefined ? null : { path, row };
|
|
1558
|
+
}
|
|
1559
|
+
return "ELOOP";
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
/** The entries directly under directory \`k\` (already resolved): [{ name, kind }]. */
|
|
1563
|
+
function __nsChildren(k) {
|
|
1564
|
+
return __residentRequire().nsChildren(k).map((row) => ({ name: String(row.name), kind: Number(row.kind) }));
|
|
1565
|
+
}
|
|
1566
|
+
|
|
1567
|
+
/** Whether the credential may search directory \`row\` (POSIX x). */
|
|
1568
|
+
function __nsTraversable(row) {
|
|
1569
|
+
const cred = __nsCred;
|
|
1570
|
+
if (!row || Number(row.kind) !== __NS_DIR) return false;
|
|
1571
|
+
const mode = Number(row.mode) & 0o777;
|
|
1572
|
+
if (!cred) return true;
|
|
1573
|
+
if (Number(cred.uid) === 0) return (mode & 0o111) !== 0;
|
|
1574
|
+
const groups = Array.isArray(cred.groups) ? cred.groups.map(Number) : [];
|
|
1575
|
+
const shift = Number(cred.uid) === Number(row.uid) ? 6
|
|
1576
|
+
: (Number(cred.gid) === Number(row.gid) || groups.includes(Number(row.gid))) ? 3 : 0;
|
|
1577
|
+
return ((mode >> shift) & 1) === 1;
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
/**
|
|
1581
|
+
* Apply one delta entry that carries a stat. Returns a directory whose
|
|
1582
|
+
* subtree must be relisted — one that just became searchable, whose
|
|
1583
|
+
* descendants no delta names — or null.
|
|
1584
|
+
*/
|
|
1585
|
+
function __nsApplyEntry(t, entry) {
|
|
1586
|
+
const k = String(entry.path).replace(/^\\/+/, "");
|
|
1587
|
+
if (k === "") return null;
|
|
1588
|
+
if (entry.stat === null) { __nsDeleteTree(t, k, true); return null; }
|
|
1589
|
+
const before = __nsRowAt(t, k);
|
|
1590
|
+
if (!__nsTryPut(t, k, entry.stat, entry.rev, entry.linkTarget)) return null;
|
|
1591
|
+
const after = __nsRowAt(t, k);
|
|
1592
|
+
if (Number(after.kind) !== __NS_DIR) { __nsDeleteTree(t, k, false); return null; }
|
|
1593
|
+
const was = before !== undefined && __nsTraversable(before);
|
|
1594
|
+
const now = __nsTraversable(after);
|
|
1595
|
+
if (was && !now) __nsDeleteTree(t, k, false);
|
|
1596
|
+
if (!was && now && before !== undefined) return k;
|
|
1597
|
+
return null;
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/**
|
|
1601
|
+
* Bring the table to a complete listing: upsert what differs, delete what the
|
|
1602
|
+
* listing no longer names. \`entries\` are VfsListEntry (path, stat, rev,
|
|
1603
|
+
* linkTarget). Called once per enumeration, after its last page.
|
|
1604
|
+
*/
|
|
1605
|
+
function __nsReplace(t, entries) {
|
|
1606
|
+
const listed = new Set();
|
|
1607
|
+
for (const entry of entries) listed.add(String(entry.path).replace(/^\\/+/, ""));
|
|
1608
|
+
// Remove stale rows before admitting replacements; do not transiently
|
|
1609
|
+
// charge both versions of a complete namespace against the fixed cap.
|
|
1610
|
+
for (const row of t.nsKeys()) {
|
|
1611
|
+
const parent = String(row.parent);
|
|
1612
|
+
const k = parent ? parent + "/" + String(row.name) : String(row.name);
|
|
1613
|
+
if (!listed.has(k)) { t.nsDelete(parent, String(row.name)); __nsLinks.delete(k); }
|
|
1614
|
+
}
|
|
1615
|
+
// Free shrinking text before growing another surviving row; final-state
|
|
1616
|
+
// admission must not depend on alphabetical replacement order.
|
|
1617
|
+
if (__residentInHeap) {
|
|
1618
|
+
for (const entry of entries) {
|
|
1619
|
+
const k = String(entry.path).replace(/^\\/+/, "");
|
|
1620
|
+
if (!k) continue;
|
|
1621
|
+
const [parent, name] = __nsSplit(k);
|
|
1622
|
+
const held = t.nsGet(parent, name);
|
|
1623
|
+
if (held && __namespaceRowBytes(parent, name, held.target) > __namespaceRowBytes(parent, name, entry.linkTarget)) {
|
|
1624
|
+
__nsPut(t, k, entry.stat, entry.rev, entry.linkTarget);
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
for (const entry of entries) {
|
|
1629
|
+
const k = String(entry.path).replace(/^\\/+/, "");
|
|
1630
|
+
if (k === "") continue;
|
|
1631
|
+
listed.add(k);
|
|
1632
|
+
const held = __nsRowAt(t, k);
|
|
1633
|
+
const s = entry.stat;
|
|
1634
|
+
if (held !== undefined
|
|
1635
|
+
&& Number(held.kind) === __nsKindCode(s.type) && Number(held.size) === Number(s.size)
|
|
1636
|
+
&& Number(held.mode) === Number(s.mode) && Number(held.uid) === Number(s.uid)
|
|
1637
|
+
&& Number(held.gid) === Number(s.gid) && Number(held.mtime) === Number(s.mtime)
|
|
1638
|
+
&& Number(held.ctime) === Number(s.ctime) && Number(held.ino) === Number(s.ino)
|
|
1639
|
+
&& Number(held.rev) === Number(entry.rev)
|
|
1640
|
+
&& (held.target ?? null) === (entry.linkTarget ?? null)) continue;
|
|
1641
|
+
__nsPut(t, k, s, entry.rev, entry.linkTarget);
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
/**
|
|
1646
|
+
* Record this facet's own acknowledged write in the namespace before the
|
|
1647
|
+
* delta that reports it: a file it created is then listed at once. Only
|
|
1648
|
+
* when the table has nothing as new, so an authority stat already applied at
|
|
1649
|
+
* or past \`rev\` is never overwritten by this approximation; the delta that
|
|
1650
|
+
* reports the write replaces it with the authority's stat.
|
|
1651
|
+
*/
|
|
1652
|
+
function __nsNoteOwnFile(k, size, rev) {
|
|
1653
|
+
if (!__nsReady()) return;
|
|
1654
|
+
const t = __residentT;
|
|
1655
|
+
const held = __nsRowAt(t, k);
|
|
1656
|
+
if (held !== undefined && Number(held.rev) >= rev) return;
|
|
1657
|
+
const now = Date.now();
|
|
1658
|
+
__nsTryPut(t, k, {
|
|
1659
|
+
// A file the process created: its umask's mode, until the delta reports
|
|
1660
|
+
// the authority's own stat for it.
|
|
1661
|
+
type: "file", size, mode: held ? Number(held.mode) : 0o100666 & ~Number(__nsCred?.umask ?? 0o022),
|
|
1662
|
+
uid: held ? Number(held.uid) : Number(__nsCred?.uid ?? 0),
|
|
1663
|
+
gid: held ? Number(held.gid) : Number(__nsCred?.gid ?? 0),
|
|
1664
|
+
atime: now, mtime: now, ctime: now, ino: held ? Number(held.ino) : 0,
|
|
1665
|
+
}, rev, null);
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
/**
|
|
1669
|
+
* Record what the authority just said of a name this process asked about (an
|
|
1670
|
+
* async stat, read or accepted write): the sync view answers it at once,
|
|
1671
|
+
* as it would after the delta that reports it. Dated at the store's cursor,
|
|
1672
|
+
* so any row as new is kept, and the next delta naming the path replaces it
|
|
1673
|
+
* with the authority's own dating. \`stat\` null: the authority said it is not
|
|
1674
|
+
* there, and a row older than the cursor goes.
|
|
1675
|
+
*/
|
|
1676
|
+
function __nsNoteLiveStat(k, stat) {
|
|
1677
|
+
if (!__nsReady()) return;
|
|
1678
|
+
const cursor = __residentCursor();
|
|
1679
|
+
if (cursor === null) return;
|
|
1680
|
+
const t = __residentT;
|
|
1681
|
+
const held = __nsRowAt(t, k);
|
|
1682
|
+
if (held !== undefined && Number(held.rev) >= cursor.rev) return;
|
|
1683
|
+
if (stat === null) {
|
|
1684
|
+
if (held !== undefined) { const [parent, name] = __nsSplit(k); t.nsDelete(parent, name); __nsLinks.delete(k); }
|
|
1685
|
+
return;
|
|
1686
|
+
}
|
|
1687
|
+
if (!__nsDescribes(stat)) return;
|
|
1688
|
+
__nsTryPut(t, k, stat, cursor.rev, stat.linkTarget ?? stat.target ?? null);
|
|
665
1689
|
}
|
|
666
1690
|
|
|
667
1691
|
/**
|
|
@@ -671,27 +1695,30 @@ function __residentHasUnder(prefix) {
|
|
|
671
1695
|
* next tenant a filesystem, and dropping the rows while leaving a cursor behind
|
|
672
1696
|
* would leave the store claiming to be current at a revision it holds nothing
|
|
673
1697
|
* from. Clearing the cursor forces the next incarnation through
|
|
674
|
-
* \`__residentAdmit\` from scratch.
|
|
1698
|
+
* \`__residentAdmit\` from scratch. \`__residentBoot\` clears the same way when a
|
|
1699
|
+
* kept store cannot be reconciled.
|
|
675
1700
|
*/
|
|
676
1701
|
function __residentClear() {
|
|
677
1702
|
if (!__residentReady) throw new Error("Nimbus: __residentClear before __residentBind");
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
1703
|
+
__residentT.clear();
|
|
1704
|
+
// Held cells go with the rows, but for own writes still in flight.
|
|
1705
|
+
for (const [path, held] of [...__residentHeld]) if (held.rev !== __RK_OWN_WRITE) __residentForgetHeld(path);
|
|
1706
|
+
__nsLinks = new Set();
|
|
1707
|
+
__nsOk = false;
|
|
1708
|
+
__residentSeal("the store was cleared");
|
|
683
1709
|
}
|
|
684
1710
|
|
|
685
1711
|
function __residentStats() {
|
|
686
1712
|
if (!__residentReady) throw new Error("Nimbus: __residentStats before __residentBind");
|
|
687
|
-
const
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
1713
|
+
const t = __residentT;
|
|
1714
|
+
const counted = t.fileStats();
|
|
1715
|
+
const files = counted === undefined ? 0 : Number(counted.n);
|
|
1716
|
+
const bytes = counted === undefined ? 0 : Number(counted.b);
|
|
1717
|
+
const names = t.nsCount();
|
|
692
1718
|
return {
|
|
693
|
-
files, bytes,
|
|
694
|
-
|
|
1719
|
+
files, bytes, names,
|
|
1720
|
+
namespace: __nsReady(),
|
|
1721
|
+
databaseSize: t.databaseSize(),
|
|
695
1722
|
sealed: __residentSealed,
|
|
696
1723
|
undatedSnapshot: __residentUndated,
|
|
697
1724
|
cursor: __residentCursor(),
|
|
@@ -699,26 +1726,219 @@ function __residentStats() {
|
|
|
699
1726
|
}
|
|
700
1727
|
|
|
701
1728
|
/**
|
|
702
|
-
*
|
|
703
|
-
*
|
|
1729
|
+
* Make this incarnation's store servable before the program's first
|
|
1730
|
+
* instruction. The one boot path the resident body takes.
|
|
704
1731
|
*
|
|
705
|
-
*
|
|
706
|
-
*
|
|
707
|
-
*
|
|
708
|
-
* the
|
|
709
|
-
*
|
|
1732
|
+
* A COLD store — no persisted cursor — adopts this spawn's own snapshot, the
|
|
1733
|
+
* module bundle dated at the cursor it was read at, and then fills the rest of
|
|
1734
|
+
* the filesystem from the authority. Its rows are current as of the spawn
|
|
1735
|
+
* whether or not the fill completes, which is all the protocol asks of a
|
|
1736
|
+
* process's first block (§5.3); the first ACQUIRE's delta brings them the rest
|
|
1737
|
+
* of the way.
|
|
1738
|
+
*
|
|
1739
|
+
* A KEPT store holds a PREVIOUS process's rows, dated against a cursor this
|
|
1740
|
+
* incarnation has never applied. A durable application's facet is one: it
|
|
1741
|
+
* keeps its \`app-slot-\` name across launches and its release never deletes
|
|
1742
|
+
* storage, so every relaunch or re-drive of the application opens what its
|
|
1743
|
+
* last process left. The rows are the asset — inside one supervisor
|
|
1744
|
+
* incarnation a relaunch fetches what changed, not the filesystem; across two,
|
|
1745
|
+
* whose revisions are unrelated, the reconcile rebuilds every dated row — and
|
|
1746
|
+
* they are also arbitrarily old: only the reconcile says which of them still
|
|
1747
|
+
* describe the filesystem, and nothing else runs before the program's first
|
|
1748
|
+
* synchronous reads. So the store stays SEALED through the reconcile, and a
|
|
1749
|
+
* reconcile that cannot vouch for its rows — the listing failed, came back
|
|
1750
|
+
* short, or there is no supervisor to ask — does not open it. The kept store
|
|
1751
|
+
* is emptied instead and this launch boots exactly as a cold one does: the
|
|
1752
|
+
* rare failure costs a relaunch its speedup, never a stale byte.
|
|
1753
|
+
*
|
|
1754
|
+
* The previous process's own unacknowledged rows are the one thing the
|
|
1755
|
+
* reconcile cannot judge, since it keeps own rows whatever the listing says.
|
|
1756
|
+
* \`__residentBind\` drops them before any of this runs.
|
|
1757
|
+
*
|
|
1758
|
+
* \`takeBundle\` hands over the module bundle and drops the module's own
|
|
1759
|
+
* reference to it. The parsed bundle is the largest allocation in the facet
|
|
1760
|
+
* before the program starts, so a cold boot releases it the moment it is
|
|
1761
|
+
* adopted, and a kept store that reconciles never adopts it at all.
|
|
1762
|
+
*
|
|
1763
|
+
* Returns the cursor to publish and, when the boot fell short of the whole
|
|
1764
|
+
* filesystem, why.
|
|
1765
|
+
*/
|
|
1766
|
+
async function __residentBoot(takeBundle, moduleCursor, supervisor) {
|
|
1767
|
+
if (!__residentReady) throw new Error("Nimbus: __residentBoot before __residentBind");
|
|
1768
|
+
let failure = null;
|
|
1769
|
+
if (__residentCursor() !== null) {
|
|
1770
|
+
let pass = null;
|
|
1771
|
+
try { pass = await __residentSynchronizeFromSupervisor(supervisor); }
|
|
1772
|
+
catch (e) { failure = (e && e.message) || String(e); }
|
|
1773
|
+
if (pass && pass.cursor) return { cursor: await __residentCatchUp(supervisor, pass.cursor), failure: null };
|
|
1774
|
+
failure = "a kept store could not be reconciled ("
|
|
1775
|
+
+ (failure || (pass && (pass.skipped || pass.incomplete)) || "the listing vouched for nothing")
|
|
1776
|
+
+ "); it was emptied and this launch booted from its own snapshot";
|
|
1777
|
+
}
|
|
1778
|
+
// A store with no cursor holds nothing any incarnation can date.
|
|
1779
|
+
__residentClear();
|
|
1780
|
+
const bundle = takeBundle();
|
|
1781
|
+
// The module map is the program's code: it is held whole or the launch
|
|
1782
|
+
// fails. Spawn admitted it (N18); this is room for it now, asked for when
|
|
1783
|
+
// the store's cap does not already cover it.
|
|
1784
|
+
let moduleBytes = 0;
|
|
1785
|
+
for (const path of Object.keys(bundle || {})) moduleBytes += __residentCellCost(bundle[path]);
|
|
1786
|
+
if (!(await __residentEnsureRoom(moduleBytes))) {
|
|
1787
|
+
throw Object.assign(new Error(
|
|
1788
|
+
"ENOSPC: workspace storage is full: this process's modules (" + Math.ceil(moduleBytes / 1048576)
|
|
1789
|
+
+ " MiB) do not fit in what is left of the session's storage"
|
|
1790
|
+
), { code: "ENOSPC" });
|
|
1791
|
+
}
|
|
1792
|
+
const adopted = __residentAdoptModuleBundle(bundle, moduleCursor);
|
|
1793
|
+
// The pass that just failed is not retried. The store now holds what a
|
|
1794
|
+
// cold launch whose fill failed holds, and its first ACQUIRE's delta
|
|
1795
|
+
// brings it current the same way.
|
|
1796
|
+
if (failure !== null) return { cursor: adopted, failure };
|
|
1797
|
+
let pass = null;
|
|
1798
|
+
try { pass = await __residentSynchronizeFromSupervisor(supervisor); }
|
|
1799
|
+
catch (e) { failure = (e && e.message) || String(e); }
|
|
1800
|
+
if (pass && pass.cursor) return { cursor: await __residentCatchUp(supervisor, pass.cursor), failure };
|
|
1801
|
+
return { cursor: adopted, failure };
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1804
|
+
/**
|
|
1805
|
+
* The acquire options this store asks for: stats with every delta entry, and
|
|
1806
|
+
* the content of files changed under the push roots — the process's working
|
|
1807
|
+
* tree and /tmp, where what a peer writes after launch is what it goes on to
|
|
1808
|
+
* read (build output, manifests, temp files), less dependencies and VCS state.
|
|
1809
|
+
*/
|
|
1810
|
+
let __residentPushRoots = [];
|
|
1811
|
+
/** Dependencies and VCS state: written in bulk, read through the module map or not at all. */
|
|
1812
|
+
const __RESIDENT_PUSH_EXCLUDE = ["node_modules", ".git"];
|
|
1813
|
+
function __residentSetPushRoots(roots) {
|
|
1814
|
+
__residentPushRoots = (Array.isArray(roots) ? roots : []).map((r) => String(r).replace(/^\\/+|\\/+$/g, ""));
|
|
1815
|
+
}
|
|
1816
|
+
/** Whether a changed file at \`path\` is one the push roots would carry. */
|
|
1817
|
+
function __residentPushable(path) {
|
|
1818
|
+
for (const root of __residentPushRoots) {
|
|
1819
|
+
if (root !== "" && path !== root && !path.startsWith(root + "/")) continue;
|
|
1820
|
+
const rest = root === "" ? path : path.slice(root.length + 1);
|
|
1821
|
+
if (!rest.split("/").some((seg) => __RESIDENT_PUSH_EXCLUDE.includes(seg))) return true;
|
|
1822
|
+
}
|
|
1823
|
+
return false;
|
|
1824
|
+
}
|
|
1825
|
+
function __residentAcquireOptions() {
|
|
1826
|
+
return __residentPushRoots.length > 0
|
|
1827
|
+
? { namespace: true, push: { roots: __residentPushRoots, exclude: __RESIDENT_PUSH_EXCLUDE } }
|
|
1828
|
+
: { namespace: true };
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
/**
|
|
1832
|
+
* One delta from a listing's cursor, applied before the program runs. A
|
|
1833
|
+
* listing's pages are walked at successive revisions, so the namespace it
|
|
1834
|
+
* builds can mix states no single revision had (a rename seen on both sides
|
|
1835
|
+
* or neither); the delta from its cursor names every path that moved during
|
|
1836
|
+
* the walk, and leaves the table exact at the delta's revision. At boot there
|
|
1837
|
+
* are no own writes, so nothing here needs the ledger's report notes.
|
|
1838
|
+
*/
|
|
1839
|
+
async function __residentCatchUp(supervisor, cursor) {
|
|
1840
|
+
if (!supervisor || typeof supervisor.fsAcquire !== "function") return cursor;
|
|
1841
|
+
let result;
|
|
1842
|
+
try { result = await supervisor.fsAcquire(cursor.epoch, cursor.rev, __residentAcquireOptions()); }
|
|
1843
|
+
catch { result = null; }
|
|
1844
|
+
if (!result || result.poison || typeof result.rev !== "number" || typeof result.epoch !== "string") {
|
|
1845
|
+
// Unanswered: the namespace keeps the listing's revision, which the
|
|
1846
|
+
// program's first barrier brings forward like any other.
|
|
1847
|
+
return cursor;
|
|
1848
|
+
}
|
|
1849
|
+
await __residentRoomForPushed(result);
|
|
1850
|
+
const applied = __residentAdmit(result);
|
|
1851
|
+
for (const dir of applied.relist) await __nsRelist(supervisor, dir);
|
|
1852
|
+
// A dropped row was held, so it is wanted again at its new revision.
|
|
1853
|
+
const refetch = [];
|
|
1854
|
+
for (const path of applied.dropped) {
|
|
1855
|
+
const found = __nsResolve(path, false);
|
|
1856
|
+
if (!found || found === "ELOOP" || Number(found.row.kind) !== __NS_FILE) continue;
|
|
1857
|
+
refetch.push({ path, size: Number(found.row.size), rev: Number(found.row.rev), epoch: applied.cursor.epoch });
|
|
1858
|
+
}
|
|
1859
|
+
if (refetch.length > 0) await __residentFetchFiles(supervisor, refetch);
|
|
1860
|
+
return applied.cursor;
|
|
1861
|
+
}
|
|
1862
|
+
|
|
1863
|
+
/**
|
|
1864
|
+
* Re-read the names beneath \`dir\` from the authority: a directory that just
|
|
1865
|
+
* became searchable, whose descendants no delta will ever name.
|
|
1866
|
+
*/
|
|
1867
|
+
async function __nsRelist(supervisor, dir) {
|
|
1868
|
+
if (!supervisor || typeof supervisor.fsList !== "function") { __nsMarkReady(__residentT, false, "the session cannot list its filesystem"); return; }
|
|
1869
|
+
const t = __residentT;
|
|
1870
|
+
const prefix = dir + "/";
|
|
1871
|
+
const seen = [];
|
|
1872
|
+
let replaceBytes = 0;
|
|
1873
|
+
let incomingBytes = 0;
|
|
1874
|
+
if (__residentInHeap) {
|
|
1875
|
+
for (const row of t.nsKeys()) {
|
|
1876
|
+
const key = row.parent ? row.parent + "/" + row.name : row.name;
|
|
1877
|
+
if (!key.startsWith(prefix)) continue;
|
|
1878
|
+
replaceBytes += __namespaceRowBytes(row.parent, row.name, t.nsGet(row.parent, row.name)?.target);
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
let after = prefix;
|
|
1882
|
+
let done = false;
|
|
1883
|
+
for (let page = 0; page < __RESIDENT_MAX_LIST_PAGES && !done; page++) {
|
|
1884
|
+
let listed;
|
|
1885
|
+
try { listed = await supervisor.fsList(after, __RESIDENT_LIST_PAGE); }
|
|
1886
|
+
catch (e) { __nsMarkReady(t, false, "a listing of /" + dir + " failed (" + ((e && e.message) || String(e)) + ")"); return; }
|
|
1887
|
+
if (!listed || !Array.isArray(listed.entries)) { __nsMarkReady(t, false, "a listing of /" + dir + " came back without entries"); return; }
|
|
1888
|
+
for (const entry of listed.entries) {
|
|
1889
|
+
const k = String(entry.path).replace(/^\\/+/, "");
|
|
1890
|
+
if (!k.startsWith(prefix)) {
|
|
1891
|
+
if (k > prefix) { done = true; break; }
|
|
1892
|
+
continue;
|
|
1893
|
+
}
|
|
1894
|
+
if (entry.stat) {
|
|
1895
|
+
if (__residentInHeap && __residentCap !== null) {
|
|
1896
|
+
const [parent, name] = __nsSplit(k);
|
|
1897
|
+
incomingBytes += __namespaceRowBytes(parent, name, entry.linkTarget);
|
|
1898
|
+
if (__residentDbBytes() - replaceBytes + incomingBytes > __residentCap) {
|
|
1899
|
+
__nsMarkReady(t, false, "namespace storage budget exceeded (" + __residentCap + " bytes)");
|
|
1900
|
+
return;
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
seen.push(entry);
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1906
|
+
if (listed.next === null || listed.next === undefined) done = true;
|
|
1907
|
+
else after = listed.next;
|
|
1908
|
+
}
|
|
1909
|
+
if (!done) { __nsMarkReady(t, false, "a listing of /" + dir + " did not finish"); return; }
|
|
1910
|
+
if (!seen.every((entry) => __nsDescribes(entry.stat))) { __nsMarkReady(t, false, "a listing of /" + dir + " named an entry without its stat"); return; }
|
|
1911
|
+
// Exactly what the listing has under the directory: a name it no longer
|
|
1912
|
+
// shows (hidden now, or gone) leaves, one it shows is written.
|
|
1913
|
+
__nsDeleteTree(t, dir, false);
|
|
1914
|
+
for (const entry of seen) __nsPut(t, String(entry.path).replace(/^\\/+/, ""), entry.stat, entry.rev, entry.linkTarget);
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1917
|
+
/**
|
|
1918
|
+
* The launch's data plan (facets/data-plan.ts): the paths whose content a
|
|
1919
|
+
* synchronous read may want, held from boot beside the module map. A launch
|
|
1920
|
+
* with none (a re-entry by an inbound request) holds what it already had.
|
|
1921
|
+
*/
|
|
1922
|
+
let __residentPlan = new Set();
|
|
1923
|
+
function __residentSetPlan(paths) {
|
|
1924
|
+
__residentPlan = new Set(Array.isArray(paths) ? paths.map((p) => String(p).replace(/^\\/+/, "")) : []);
|
|
1925
|
+
}
|
|
1926
|
+
|
|
1927
|
+
/**
|
|
1928
|
+
* Adopt the module map's bundle into an EMPTY store — the first fill, and the
|
|
1929
|
+
* one that costs nothing extra, because those bytes are already in the facet.
|
|
1930
|
+
* The cursor the bundle was read at becomes the store's.
|
|
710
1931
|
*
|
|
711
1932
|
* Returns the cursor the caller should publish, so there is one answer to
|
|
712
1933
|
* "what state does this facet cache" rather than two that can disagree.
|
|
713
1934
|
*/
|
|
714
1935
|
function __residentAdoptModuleBundle(bundle, moduleCursor) {
|
|
715
1936
|
if (!__residentReady) throw new Error("Nimbus: __residentAdoptModuleBundle before __residentBind");
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
//
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
return persisted;
|
|
1937
|
+
if (__residentCursor() !== null) {
|
|
1938
|
+
// Adopting over a kept store would re-date its rows at this spawn's
|
|
1939
|
+
// cursor without checking one of them. __residentBoot reconciles a kept
|
|
1940
|
+
// store, or empties it first.
|
|
1941
|
+
throw new Error("Nimbus: a module bundle is adopted only into an empty store");
|
|
722
1942
|
}
|
|
723
1943
|
if (!moduleCursor || moduleCursor.epoch == null || moduleCursor.rev == null) {
|
|
724
1944
|
// An undated snapshot. Adopt NOTHING and serve an empty store.
|
|
@@ -741,7 +1961,7 @@ function __residentAdoptModuleBundle(bundle, moduleCursor) {
|
|
|
741
1961
|
}
|
|
742
1962
|
const rev = Number(moduleCursor.rev);
|
|
743
1963
|
for (const path of Object.keys(bundle || {})) {
|
|
744
|
-
__residentPut(
|
|
1964
|
+
__residentPut(__residentT, path, bundle[path], rev);
|
|
745
1965
|
}
|
|
746
1966
|
return __residentAdmit({ poison: false, paths: [], epoch: String(moduleCursor.epoch), rev }).cursor;
|
|
747
1967
|
}
|
|
@@ -750,13 +1970,12 @@ function __residentAdoptModuleBundle(bundle, moduleCursor) {
|
|
|
750
1970
|
* Enumerate the whole filesystem, one page at a time.
|
|
751
1971
|
*
|
|
752
1972
|
* WHAT EXISTS has to come from the authority, not from anything the facet was
|
|
753
|
-
* shipped
|
|
754
|
-
*
|
|
755
|
-
* FOUR entries
|
|
756
|
-
*
|
|
757
|
-
*
|
|
758
|
-
*
|
|
759
|
-
* the admission problem it exists to delete.
|
|
1973
|
+
* shipped. That was measured rather than assumed: for a working tree holding a
|
|
1974
|
+
* 25 MiB file outside the cwd, the spawn-time stat table this replaced arrived
|
|
1975
|
+
* with FOUR entries ("home", "opt", "var", "home/user"), every one a
|
|
1976
|
+
* directory. A launch ships only what it staged plus its ancestors; a store
|
|
1977
|
+
* that enumerated from that would hold only what the bundle already had, which
|
|
1978
|
+
* is the admission problem it exists to delete.
|
|
760
1979
|
*
|
|
761
1980
|
* Every page is dated, and a page whose epoch differs from the first ABORTS
|
|
762
1981
|
* the listing. The supervisor was replaced mid-walk, so the pages already
|
|
@@ -767,6 +1986,9 @@ function __residentAdoptModuleBundle(bundle, moduleCursor) {
|
|
|
767
1986
|
*/
|
|
768
1987
|
async function __residentEnumerate(supervisor) {
|
|
769
1988
|
const entries = [];
|
|
1989
|
+
// Every name, whatever its kind: the namespace is built from these.
|
|
1990
|
+
const names = [];
|
|
1991
|
+
let namespaceBytes = 0;
|
|
770
1992
|
let after = null;
|
|
771
1993
|
let cursor = null;
|
|
772
1994
|
for (let page = 0; page < __RESIDENT_MAX_LIST_PAGES; page++) {
|
|
@@ -777,13 +1999,26 @@ async function __residentEnumerate(supervisor) {
|
|
|
777
1999
|
if (cursor === null) {
|
|
778
2000
|
cursor = { epoch: String(listed.epoch), rev: Number(listed.rev) };
|
|
779
2001
|
} else if (String(listed.epoch) !== cursor.epoch) {
|
|
780
|
-
return { entries, cursor, complete: false, reason: "the supervisor changed incarnation mid-enumeration" };
|
|
2002
|
+
return { entries, names, cursor, complete: false, reason: "the supervisor changed incarnation mid-enumeration" };
|
|
781
2003
|
}
|
|
782
2004
|
for (const entry of listed.entries) {
|
|
2005
|
+
if (entry.stat) {
|
|
2006
|
+
if (__residentInHeap && __residentCap !== null) {
|
|
2007
|
+
const key = String(entry.path).replace(/^\\/+/, "");
|
|
2008
|
+
const [parent, name] = __nsSplit(key);
|
|
2009
|
+
if (key) namespaceBytes += __namespaceRowBytes(parent, name, entry.linkTarget);
|
|
2010
|
+
if (__residentDbBytes() - __residentT.nsBytes() + namespaceBytes > __residentCap) {
|
|
2011
|
+
const cause = "namespace storage budget exceeded (" + __residentCap + " bytes)";
|
|
2012
|
+
__nsMarkReady(__residentT, false, cause);
|
|
2013
|
+
throw Object.assign(new Error(cause), { code: "ENOSPC" });
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
names.push(entry);
|
|
2017
|
+
}
|
|
783
2018
|
// Directories carry no content, and a symlink's size is its TARGET's
|
|
784
2019
|
// length rather than the resolved file's — reading one by that size
|
|
785
|
-
// would store a truncated file and call it whole.
|
|
786
|
-
//
|
|
2020
|
+
// would store a truncated file and call it whole. Content is fetched
|
|
2021
|
+
// by the path a symlink resolves to.
|
|
787
2022
|
if (entry.kind !== "file") continue;
|
|
788
2023
|
const path = String(entry.path).replace(/^\\/+/, "");
|
|
789
2024
|
if (!path) continue;
|
|
@@ -795,14 +2030,14 @@ async function __residentEnumerate(supervisor) {
|
|
|
795
2030
|
if (!Number.isSafeInteger(size) || size < 0 || !Number.isSafeInteger(rev) || rev < 0) {
|
|
796
2031
|
throw new Error("Nimbus: fsList returned unusable metadata for '" + path + "'");
|
|
797
2032
|
}
|
|
798
|
-
entries.push({ path, size, rev, epoch: cursor.epoch });
|
|
2033
|
+
entries.push({ path, size, rev, epoch: cursor.epoch, ckey: entry.contentKey == null ? null : String(entry.contentKey) });
|
|
799
2034
|
}
|
|
800
2035
|
if (listed.next === null || listed.next === undefined) {
|
|
801
|
-
return { entries, cursor, complete: true, reason: null };
|
|
2036
|
+
return { entries, names, namespaceBytes, cursor, complete: true, reason: null };
|
|
802
2037
|
}
|
|
803
2038
|
after = listed.next;
|
|
804
2039
|
}
|
|
805
|
-
return { entries, cursor, complete: false, reason: "the filesystem exceeded the enumeration page bound" };
|
|
2040
|
+
return { entries, names, cursor, complete: false, reason: "the filesystem exceeded the enumeration page bound" };
|
|
806
2041
|
}
|
|
807
2042
|
|
|
808
2043
|
/**
|
|
@@ -834,6 +2069,23 @@ async function __residentEnumerate(supervisor) {
|
|
|
834
2069
|
* whose path moved at that same revision.
|
|
835
2070
|
*/
|
|
836
2071
|
async function __residentFetchFiles(supervisor, files) {
|
|
2072
|
+
// Content already held under another path, or wanted twice in this batch,
|
|
2073
|
+
// is fetched once: equal keys are equal bytes (the second copy is local).
|
|
2074
|
+
const t = __residentT;
|
|
2075
|
+
const requested = files.length;
|
|
2076
|
+
let copied = 0;
|
|
2077
|
+
const wanted = [];
|
|
2078
|
+
const later = [];
|
|
2079
|
+
const fetching = new Set();
|
|
2080
|
+
for (const file of files) {
|
|
2081
|
+
if (file.ckey != null) {
|
|
2082
|
+
if (__residentCopyByKey(t, file.path, file.ckey, file.rev)) { copied++; continue; }
|
|
2083
|
+
if (fetching.has(file.ckey)) { later.push(file); continue; }
|
|
2084
|
+
fetching.add(file.ckey);
|
|
2085
|
+
}
|
|
2086
|
+
wanted.push(file);
|
|
2087
|
+
}
|
|
2088
|
+
files = wanted;
|
|
837
2089
|
// One flat list of ranges across all files, so packing is a single pass and
|
|
838
2090
|
// a 25 MiB file and a 40-byte one are the same shape.
|
|
839
2091
|
const ranges = [];
|
|
@@ -850,6 +2102,7 @@ async function __residentFetchFiles(supervisor, files) {
|
|
|
850
2102
|
offset,
|
|
851
2103
|
length: Math.min(__RESIDENT_CHUNK_BYTES, Math.max(0, file.size - offset)),
|
|
852
2104
|
size: file.size,
|
|
2105
|
+
ckey: file.ckey ?? null,
|
|
853
2106
|
});
|
|
854
2107
|
}
|
|
855
2108
|
}
|
|
@@ -872,6 +2125,12 @@ async function __residentFetchFiles(supervisor, files) {
|
|
|
872
2125
|
batch.push(ranges[at]);
|
|
873
2126
|
at++;
|
|
874
2127
|
}
|
|
2128
|
+
// N18: room for the batch first; what the ledger will not admit is read
|
|
2129
|
+
// through the session instead of held.
|
|
2130
|
+
if (!(await __residentEnsureRoom(bytes + batch.length * __RESIDENT_ROW_BYTES))) {
|
|
2131
|
+
for (const r of batch) failedPaths.add(r.path);
|
|
2132
|
+
continue;
|
|
2133
|
+
}
|
|
875
2134
|
let results;
|
|
876
2135
|
try {
|
|
877
2136
|
results = await supervisor.fsReadBatch(
|
|
@@ -890,7 +2149,7 @@ async function __residentFetchFiles(supervisor, files) {
|
|
|
890
2149
|
// path this store will not hold, and a read of it falls through to
|
|
891
2150
|
// exactly the miss it would have had before.
|
|
892
2151
|
if (entry && entry.error && entry.error.code === "EACCES" && range.part === 0) {
|
|
893
|
-
__residentPut(
|
|
2152
|
+
__residentPut(t, range.path, { error: "EACCES" }, range.rev);
|
|
894
2153
|
filled++;
|
|
895
2154
|
} else failedPaths.add(range.path);
|
|
896
2155
|
continue;
|
|
@@ -899,11 +2158,11 @@ async function __residentFetchFiles(supervisor, files) {
|
|
|
899
2158
|
const chunk = __residentBytes(entry.bytes);
|
|
900
2159
|
if (chunk.byteLength !== range.length) { failedPaths.add(range.path); continue; }
|
|
901
2160
|
fetchedBytes += chunk.byteLength;
|
|
902
|
-
__residentPutChunk(
|
|
2161
|
+
if (!__residentPutChunk(t, range.path, range.part, chunk)) { failedPaths.add(range.path); continue; }
|
|
903
2162
|
const seen = (landed.get(range.path) || 0) + 1;
|
|
904
2163
|
landed.set(range.path, seen);
|
|
905
2164
|
if (seen === range.parts && !failedPaths.has(range.path)) {
|
|
906
|
-
__residentPutHead(
|
|
2165
|
+
__residentPutHead(t, range.path, __RK_BINARY, range.size, range.parts, range.rev, range.ckey);
|
|
907
2166
|
filled++;
|
|
908
2167
|
}
|
|
909
2168
|
}
|
|
@@ -914,11 +2173,19 @@ async function __residentFetchFiles(supervisor, files) {
|
|
|
914
2173
|
// nothing will ever serve.
|
|
915
2174
|
for (const path of failedPaths) {
|
|
916
2175
|
failed++;
|
|
917
|
-
|
|
918
|
-
|
|
2176
|
+
t.chunkDelete(path);
|
|
2177
|
+
t.fileDelete(path);
|
|
919
2178
|
}
|
|
920
2179
|
|
|
921
|
-
|
|
2180
|
+
for (const file of later) {
|
|
2181
|
+
if (__residentCopyByKey(t, file.path, file.ckey, file.rev)) copied++;
|
|
2182
|
+
else failed++;
|
|
2183
|
+
}
|
|
2184
|
+
if (ranges.length > 0) await __residentReportSize();
|
|
2185
|
+
return {
|
|
2186
|
+
requested, filled: filled + copied, failed,
|
|
2187
|
+
bytes: fetchedBytes, ranges: ranges.length, copied,
|
|
2188
|
+
};
|
|
922
2189
|
}
|
|
923
2190
|
|
|
924
2191
|
/**
|
|
@@ -963,9 +2230,16 @@ async function __residentFetchFiles(supervisor, files) {
|
|
|
963
2230
|
* had, and a poisoned caller has already taken its cold cache through
|
|
964
2231
|
* \`__residentAdmit\`.
|
|
965
2232
|
*
|
|
966
|
-
* A row stamped __RK_OWN_WRITE is this facet's own
|
|
967
|
-
* than anything the authority can report — and is kept whether listed
|
|
968
|
-
* the same read-your-writes rule the delta path applies.
|
|
2233
|
+
* A row stamped __RK_OWN_WRITE is this facet's own unacknowledged bytes —
|
|
2234
|
+
* newer than anything the authority can report — and is kept whether listed
|
|
2235
|
+
* or not, the same read-your-writes rule the delta path applies. But the
|
|
2236
|
+
* pass moves the cursor without a delta, so the report a delta would have
|
|
2237
|
+
* carried for such a path is returned instead (\`own\`): the listed revision,
|
|
2238
|
+
* or the listing's own cursor for a path it no longer lists, which bounds
|
|
2239
|
+
* whatever removed it. The write or mutation that owns the row adjudicates it
|
|
2240
|
+
* when its revision arrives, exactly as it does a delta's report. Across
|
|
2241
|
+
* incarnations, or against a short listing, no report can be dated and each
|
|
2242
|
+
* comes back null.
|
|
969
2243
|
*
|
|
970
2244
|
* Writes landing DURING the pass are covered the way they always were: the
|
|
971
2245
|
* published cursor is the one read BEFORE the walk, so they report revisions
|
|
@@ -994,12 +2268,13 @@ async function __residentSynchronizeFromSupervisor(supervisor) {
|
|
|
994
2268
|
return { requested: 0, filled: 0, failed: 0, skipped: "fsList reported no authority cursor" };
|
|
995
2269
|
}
|
|
996
2270
|
|
|
997
|
-
const
|
|
2271
|
+
const t = __residentT;
|
|
998
2272
|
const held = __residentCursor();
|
|
999
2273
|
// May a row this pass cannot prove current be DROPPED? Only against a
|
|
1000
2274
|
// complete enumeration: a truncated one cannot tell a path that was removed
|
|
1001
2275
|
// from one that was never walked.
|
|
1002
2276
|
const judgeable = listing.complete;
|
|
2277
|
+
const describable = judgeable && listing.names.every((entry) => __nsDescribes(entry.stat));
|
|
1003
2278
|
// May a held revision be COMPARED with a listed one? Only inside one
|
|
1004
2279
|
// supervisor incarnation: across a restart the clocks are unrelated, and an
|
|
1005
2280
|
// untouched path lists at rev 0, which would vouch for anything.
|
|
@@ -1009,44 +2284,103 @@ async function __residentSynchronizeFromSupervisor(supervisor) {
|
|
|
1009
2284
|
for (const file of listing.entries) listed.set(file.path, file);
|
|
1010
2285
|
|
|
1011
2286
|
const rows = [];
|
|
1012
|
-
for (const row of
|
|
1013
|
-
rows.push({ path: String(row.path), rev: Number(row.rev) });
|
|
2287
|
+
for (const row of t.fileRows()) {
|
|
2288
|
+
rows.push({ path: String(row.path), rev: Number(row.rev), ckey: row.ckey == null ? null : String(row.ckey) });
|
|
1014
2289
|
}
|
|
1015
2290
|
const current = new Set();
|
|
1016
2291
|
const dropped = [];
|
|
2292
|
+
const own = [];
|
|
2293
|
+
let rekeyed = 0;
|
|
1017
2294
|
for (const row of rows) {
|
|
1018
2295
|
const entry = listed.get(row.path);
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
2296
|
+
if (row.rev === __RK_OWN_WRITE) {
|
|
2297
|
+
current.add(row.path);
|
|
2298
|
+
own.push({
|
|
2299
|
+
path: row.path,
|
|
2300
|
+
rev: !comparable ? null : entry !== undefined ? entry.rev : listing.cursor.rev,
|
|
2301
|
+
});
|
|
2302
|
+
continue;
|
|
2303
|
+
}
|
|
2304
|
+
const keep = (comparable && entry !== undefined && row.rev >= entry.rev) || !judgeable;
|
|
1022
2305
|
if (keep) { current.add(row.path); continue; }
|
|
2306
|
+
// The listing names the same bytes (a revision moved, the content did
|
|
2307
|
+
// not, or another epoch): kept and dated at the listing. Equal keys are
|
|
2308
|
+
// equal bytes, so this needs no comparable clock.
|
|
2309
|
+
if (entry !== undefined && row.ckey !== null && entry.ckey != null && row.ckey === entry.ckey) {
|
|
2310
|
+
t.fileSetRev(row.path, entry.rev);
|
|
2311
|
+
current.add(row.path);
|
|
2312
|
+
rekeyed++;
|
|
2313
|
+
continue;
|
|
2314
|
+
}
|
|
1023
2315
|
dropped.push(row.path);
|
|
1024
2316
|
}
|
|
2317
|
+
// Cells held in the heap (own writes the store had no room for) are judged
|
|
2318
|
+
// as file rows are, on every path a file row is (N18). Undated, a write-back
|
|
2319
|
+
// still in flight: newer than anything listed, it stays and is reported as
|
|
2320
|
+
// own. Dated: kept only where the listing proves it current; otherwise it
|
|
2321
|
+
// goes, and is fetched again like any dropped row, so a sync read of it is
|
|
2322
|
+
// a miss rather than bytes the authority has replaced.
|
|
2323
|
+
for (const [path, cell] of [...__residentHeld]) {
|
|
2324
|
+
if (current.has(path)) continue;
|
|
2325
|
+
const entry = listed.get(path);
|
|
2326
|
+
if (cell.rev === __RK_OWN_WRITE) {
|
|
2327
|
+
current.add(path);
|
|
2328
|
+
own.push({ path, rev: !comparable ? null : entry !== undefined ? entry.rev : listing.cursor.rev });
|
|
2329
|
+
continue;
|
|
2330
|
+
}
|
|
2331
|
+
if ((comparable && entry !== undefined && cell.rev >= entry.rev) || !judgeable) { current.add(path); continue; }
|
|
2332
|
+
__residentForgetHeld(path);
|
|
2333
|
+
dropped.push(path);
|
|
2334
|
+
}
|
|
1025
2335
|
// Sweep what could not be vouched for. When NOTHING could — a new supervisor
|
|
1026
2336
|
// incarnation, whose revisions say nothing about ours — that is every row but
|
|
1027
2337
|
// this facet's own unflushed writes, and at pi scale a per-path delete would
|
|
1028
2338
|
// be ~32,000 statements in one turn against an object that has been observed
|
|
1029
2339
|
// resetting under exactly that load. So it is a predicate there, and per path
|
|
1030
2340
|
// where a reconcile has made the set small by construction.
|
|
1031
|
-
if (dropped.length > 0 && !comparable) {
|
|
1032
|
-
|
|
1033
|
-
sql.exec("DELETE FROM file WHERE rev <> ?", __RK_OWN_WRITE);
|
|
2341
|
+
if (dropped.length > 0 && !comparable && rekeyed === 0) {
|
|
2342
|
+
t.dropDatedFiles();
|
|
1034
2343
|
} else {
|
|
1035
2344
|
for (const path of dropped) {
|
|
1036
|
-
|
|
1037
|
-
|
|
2345
|
+
t.chunkDelete(path);
|
|
2346
|
+
t.fileDelete(path);
|
|
1038
2347
|
}
|
|
1039
2348
|
}
|
|
1040
2349
|
|
|
2350
|
+
// What to hold: the launch's data plan, every row this pass had to drop as
|
|
2351
|
+
// stale — it was held for a reason — and what the pushes a poison lost
|
|
2352
|
+
// would have carried: files under the push roots changed since the cursor
|
|
2353
|
+
// (all of them there, when the revisions are not comparable).
|
|
2354
|
+
const plan = __residentPlan;
|
|
2355
|
+
const wanted = new Set(dropped);
|
|
1041
2356
|
const fetch = [];
|
|
1042
|
-
for (const file of listing.entries)
|
|
1043
|
-
|
|
2357
|
+
for (const file of listing.entries) {
|
|
2358
|
+
if (current.has(file.path)) continue;
|
|
2359
|
+
if (plan.has(file.path) || wanted.has(file.path)
|
|
2360
|
+
|| (__residentPushable(file.path) && (!comparable || file.rev > held.rev))) fetch.push(file);
|
|
2361
|
+
}
|
|
2362
|
+
// Reserve mandatory metadata growth while optional data is fetched, but
|
|
2363
|
+
// retain the existing namespace publication order across the await.
|
|
2364
|
+
const namespaceGrowth = __residentInHeap && describable ? Math.max(0, listing.namespaceBytes - t.nsBytes()) : 0;
|
|
2365
|
+
if (__residentInHeap && __residentCap !== null && __residentDbBytes() + namespaceGrowth > __residentCap) {
|
|
2366
|
+
const cause = "namespace storage budget exceeded (" + __residentCap + " bytes)";
|
|
2367
|
+
__nsMarkReady(t, false, cause);
|
|
2368
|
+
throw Object.assign(new Error(cause), { code: "ENOSPC" });
|
|
2369
|
+
}
|
|
2370
|
+
__residentNamespaceReserveBytes += namespaceGrowth;
|
|
2371
|
+
let filled;
|
|
2372
|
+
try { filled = await __residentFetchFiles(supervisor, fetch); }
|
|
2373
|
+
finally { __residentNamespaceReserveBytes -= namespaceGrowth; }
|
|
1044
2374
|
|
|
1045
2375
|
// The cursor may only advance to a state the rows actually describe, and
|
|
1046
2376
|
// after a truncated listing they do not: a path in an unwalked page could
|
|
1047
2377
|
// have moved since the held cursor and would never be reported again.
|
|
1048
2378
|
if (judgeable) {
|
|
1049
|
-
|
|
2379
|
+
// A listing that cannot describe every name restores the rows it vouches
|
|
2380
|
+
// for and the cursor, but not the namespace (see __nsDescribes).
|
|
2381
|
+
if (describable) __nsReplace(t, listing.names);
|
|
2382
|
+
__residentWriteCursor(t, listing.cursor);
|
|
2383
|
+
__nsMarkReady(t, describable, "the session's listing named an entry without its stat");
|
|
1050
2384
|
__residentUndated = false;
|
|
1051
2385
|
__residentSealed = false;
|
|
1052
2386
|
__residentSealReason = "";
|
|
@@ -1059,7 +2393,9 @@ async function __residentSynchronizeFromSupervisor(supervisor) {
|
|
|
1059
2393
|
bytes: filled.bytes,
|
|
1060
2394
|
ranges: filled.ranges,
|
|
1061
2395
|
kept: current.size,
|
|
2396
|
+
rekeyed,
|
|
1062
2397
|
dropped: dropped.length,
|
|
2398
|
+
own,
|
|
1063
2399
|
reconciled: comparable,
|
|
1064
2400
|
complete: listing.complete,
|
|
1065
2401
|
cursor: judgeable ? listing.cursor : null,
|
|
@@ -1067,6 +2403,45 @@ async function __residentSynchronizeFromSupervisor(supervisor) {
|
|
|
1067
2403
|
};
|
|
1068
2404
|
}
|
|
1069
2405
|
|
|
2406
|
+
/**
|
|
2407
|
+
* Refetch, after a delta the store applied without a listing, what it should
|
|
2408
|
+
* hold and does not: the paths the delta dropped and the launch's data plan.
|
|
2409
|
+
* The namespace (current as of that delta) gives each one's size and
|
|
2410
|
+
* revision, as a listing entry would.
|
|
2411
|
+
*/
|
|
2412
|
+
async function __residentRefillFromNamespace(supervisor, dropped) {
|
|
2413
|
+
const t = __residentT;
|
|
2414
|
+
const cursor = __residentCursor();
|
|
2415
|
+
if (cursor === null) return { requested: 0, filled: 0, failed: 0, bytes: 0 };
|
|
2416
|
+
const fetch = [];
|
|
2417
|
+
const seen = new Set();
|
|
2418
|
+
for (const path of [...dropped, ...__residentPlan]) {
|
|
2419
|
+
if (seen.has(path)) continue;
|
|
2420
|
+
seen.add(path);
|
|
2421
|
+
if (__residentHead(path) !== undefined || __residentHeld.has(path)) continue;
|
|
2422
|
+
const row = __nsRowAt(t, path);
|
|
2423
|
+
if (row === undefined || Number(row.kind) !== __NS_FILE) continue;
|
|
2424
|
+
fetch.push({ path, size: Number(row.size), rev: Number(row.rev), epoch: cursor.epoch, ckey: null });
|
|
2425
|
+
}
|
|
2426
|
+
return __residentFetchFiles(supervisor, fetch);
|
|
2427
|
+
}
|
|
2428
|
+
|
|
2429
|
+
/**
|
|
2430
|
+
* The launch's last step before user code: the namespace must answer, since
|
|
2431
|
+
* every synchronous stat, exists and readdir reads it. A boot that left it
|
|
2432
|
+
* unready (its listing failed, came back short, or could not describe every
|
|
2433
|
+
* name) is repaired once more. Resolves to null when it answers, or to the
|
|
2434
|
+
* launch's failure, naming why; nothing is ever run on a namespace that
|
|
2435
|
+
* does not.
|
|
2436
|
+
*/
|
|
2437
|
+
async function __residentRequireNamespace(supervisor, bootFailure) {
|
|
2438
|
+
if (__nsReady()) return null;
|
|
2439
|
+
try { await __residentSynchronizeFromSupervisor(supervisor); } catch {}
|
|
2440
|
+
if (__nsReady()) return null;
|
|
2441
|
+
return "node: the process was not started: its view of the filesystem could not be listed ("
|
|
2442
|
+
+ __nsNotReadyCause() + (bootFailure ? "; " + bootFailure : "") + ")";
|
|
2443
|
+
}
|
|
2444
|
+
|
|
1070
2445
|
/**
|
|
1071
2446
|
* The resident set, as the shims have always seen it.
|
|
1072
2447
|
*
|
|
@@ -1079,7 +2454,7 @@ const __nimbusResidentBundle = new Proxy(Object.create(null), {
|
|
|
1079
2454
|
},
|
|
1080
2455
|
has(_t, path) {
|
|
1081
2456
|
if (typeof path !== "string") return false;
|
|
1082
|
-
return __residentHead(path) !== undefined;
|
|
2457
|
+
return __residentHead(path) !== undefined || __residentHeld.has(path);
|
|
1083
2458
|
},
|
|
1084
2459
|
set(_t, path, cell) {
|
|
1085
2460
|
// The program's own write, stamped __RK_OWN_WRITE: not the authority's
|