@nimbus-sh/platform 0.8.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/composition.d.ts +3 -2
- package/dist/composition.d.ts.map +1 -1
- package/dist/composition.js +34 -19
- package/dist/crc32.d.ts +5 -5
- package/dist/crc32.d.ts.map +1 -1
- package/dist/crc32.js +8 -12
- package/dist/diag-counters.d.ts +18 -10
- package/dist/diag-counters.d.ts.map +1 -1
- package/dist/diag-counters.js +7 -20
- package/dist/heap-estimate.d.ts +5 -20
- package/dist/heap-estimate.d.ts.map +1 -1
- package/dist/heap-estimate.js +6 -33
- package/dist/heavy-alloc-coord.d.ts +131 -11
- package/dist/heavy-alloc-coord.d.ts.map +1 -1
- package/dist/heavy-alloc-coord.js +241 -22
- package/dist/hex.d.ts +20 -0
- package/dist/hex.d.ts.map +1 -0
- package/dist/hex.js +29 -0
- package/dist/install-phase.d.ts +1 -15
- package/dist/install-phase.d.ts.map +1 -1
- package/dist/lexical-path.d.ts +11 -0
- package/dist/lexical-path.d.ts.map +1 -0
- package/dist/lexical-path.js +19 -0
- package/dist/limits.d.ts +4 -0
- package/dist/limits.d.ts.map +1 -1
- package/dist/limits.js +25 -0
- package/dist/module-source.d.ts.map +1 -1
- package/dist/module-source.js +2 -1
- package/dist/oom-classify.d.ts +5 -1
- package/dist/oom-classify.d.ts.map +1 -1
- package/dist/oom-classify.js +7 -1
- package/dist/realm-builtin.d.ts +2 -0
- package/dist/realm-builtin.d.ts.map +1 -0
- package/dist/realm-builtin.js +11 -0
- package/dist/utf8.d.ts +13 -1
- package/dist/utf8.d.ts.map +1 -1
- package/dist/utf8.js +67 -1
- package/dist/w7-frame.d.ts +47 -13
- package/dist/w7-frame.d.ts.map +1 -1
- package/dist/w7-frame.js +42 -24
- package/dist/wave-writer.d.ts +11 -6
- package/dist/wave-writer.d.ts.map +1 -1
- package/dist/wave-writer.js +28 -29
- package/dist/weighted-credit-pool.d.ts +43 -0
- package/dist/weighted-credit-pool.d.ts.map +1 -1
- package/dist/weighted-credit-pool.js +250 -46
- package/dist/worker-code.d.ts +48 -0
- package/dist/worker-code.d.ts.map +1 -0
- package/dist/worker-code.js +10 -0
- package/package.json +1 -1
- package/src/composition.ts +41 -21
- package/src/crc32.ts +9 -12
- package/src/diag-counters.ts +23 -22
- package/src/heap-estimate.ts +11 -40
- package/src/heavy-alloc-coord.ts +275 -20
- package/src/hex.ts +29 -0
- package/src/install-phase.ts +1 -15
- package/src/lexical-path.ts +17 -0
- package/src/limits.ts +29 -0
- package/src/module-source.ts +3 -1
- package/src/oom-classify.ts +9 -1
- package/src/realm-builtin.ts +22 -0
- package/src/utf8.ts +69 -1
- package/src/w7-frame.ts +74 -33
- package/src/wave-writer.ts +28 -27
- package/src/weighted-credit-pool.ts +256 -51
- package/src/worker-code.ts +38 -0
package/src/limits.ts
CHANGED
|
@@ -197,6 +197,26 @@ export const FACET_OWN_WRITE_MEMORY_BYTES = 32 * 1024 * 1024;
|
|
|
197
197
|
// holds nothing more and the session answers.
|
|
198
198
|
export const WASI_RESIDENT_STORE_BYTES = 32 * 1024 * 1024;
|
|
199
199
|
|
|
200
|
+
// A file too big to hold whole (over WASI_RESIDENT_FILE_CAP_BYTES: a pack, a
|
|
201
|
+
// deep history's .idx) is read through the resident store a page at a time:
|
|
202
|
+
// git's own unit for a paged pack index, and a sixteenth of its 1 MiB pack
|
|
203
|
+
// window, so one window is one batch read.
|
|
204
|
+
export const RESIDENT_PAGE_BYTES = 64 * 1024;
|
|
205
|
+
|
|
206
|
+
// The most a resident store holds of such pages, least recently used out
|
|
207
|
+
// first; in a one-shot's heap, never more than half its budget. Measured on vscode's full history (2.18M objects):
|
|
208
|
+
// a .idx of 57.5 MiB, of which every command touched ~49 MiB, and a .rev of
|
|
209
|
+
// 8.6 MiB, so the bound keeps a deep history's index and its hot pack windows.
|
|
210
|
+
export const RESIDENT_PAGE_CACHE_BYTES = 512 * 1024 * 1024;
|
|
211
|
+
|
|
212
|
+
// The pages are a cache in the session's one storage limit (N18), which the
|
|
213
|
+
// user's files share: they never hold more than this share of what they hold
|
|
214
|
+
// plus what the ledger says is still free at the last grant, so a filling
|
|
215
|
+
// session takes room from the cache, not from the user. Evicted pages are
|
|
216
|
+
// reused by the same facet database; its size, which df counts, stays at its
|
|
217
|
+
// high-water mark until the facet's storage is deleted.
|
|
218
|
+
export const RESIDENT_PAGE_CACHE_FREE_SHARE = 0.25;
|
|
219
|
+
|
|
200
220
|
// Shared allowance for transient allocations in the supervisor DO. With the
|
|
201
221
|
// VFS LRU shrunk to 8 MiB during an active reservation, 40 MiB of admitted
|
|
202
222
|
// payload plus the 9 MiB bundle baseline stays below the 64 MiB soft ceiling
|
|
@@ -204,6 +224,15 @@ export const WASI_RESIDENT_STORE_BYTES = 32 * 1024 * 1024;
|
|
|
204
224
|
// This is 31.25% of the platform's 128 MiB hard isolate ceiling.
|
|
205
225
|
export const SUPERVISOR_IN_FLIGHT_ALLOCATION_BUDGET_BYTES = 40 * 1024 * 1024;
|
|
206
226
|
|
|
227
|
+
// The longest a facet's load waits for that allowance before it is refused
|
|
228
|
+
// (acquireSupervisorLoadAllocation). The wait is FIFO, so a load waits only
|
|
229
|
+
// for the holds ahead of it, each of which ends when its own work does. A
|
|
230
|
+
// load is the one owner that waits while it holds a Dynamic Worker (the
|
|
231
|
+
// loader asks for its code then), so a launch that holds credit while it
|
|
232
|
+
// waits for a worker of the same Durable Object can close a cycle nothing
|
|
233
|
+
// ends: the bound refuses the load, by name, instead of leaving both to hang.
|
|
234
|
+
export const SUPERVISOR_LOAD_WAIT_MS = 60_000;
|
|
235
|
+
|
|
207
236
|
// Reserved slice of that allowance for chunk-sized filesystem reads, so a
|
|
208
237
|
// read is never queued behind a multi-megabyte owner for a wait that has
|
|
209
238
|
// nothing to do with its own cost. A read only draws on this when the shared
|
package/src/module-source.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { utf8Length } from './utf8.js';
|
|
2
|
+
|
|
1
3
|
/** A deployed source asset: its digest, not its pathname, is its identity. */
|
|
2
4
|
export interface ModuleSourceAsset {
|
|
3
5
|
path: string;
|
|
@@ -26,7 +28,7 @@ export class ModuleSource {
|
|
|
26
28
|
private joined: string | undefined;
|
|
27
29
|
readonly byteLength: number;
|
|
28
30
|
constructor(readonly parts: readonly ModuleSourcePart[]) {
|
|
29
|
-
this.byteLength = parts.reduce((bytes, part) => bytes + (typeof part === 'string' ?
|
|
31
|
+
this.byteLength = parts.reduce((bytes, part) => bytes + (typeof part === 'string' ? utf8Length(part) : part.byteLength), 0);
|
|
30
32
|
}
|
|
31
33
|
get text(): string {
|
|
32
34
|
return this.joined ??= this.parts.map((part) => typeof part === 'string' ? part : part.text).join('');
|
package/src/oom-classify.ts
CHANGED
|
@@ -194,7 +194,7 @@ export function classifyMessage(msg: string): OomCause {
|
|
|
194
194
|
* why an object reset mid-write still failed a whole install. Nimbus has
|
|
195
195
|
* measured the live-write wording twice: a 45.7 MB single-turn write reset the
|
|
196
196
|
* object once and the same write then succeeded 12/12 on retry
|
|
197
|
-
* (`vfs/
|
|
197
|
+
* (`vfs/resident`), and it reset a session DO mid-npm-install
|
|
198
198
|
* on 2026-08-10 (probe `agentic-cli/new/pi-official-installer`). Retrying is
|
|
199
199
|
* bounded, so the one case that is NOT transient — crossing the object's
|
|
200
200
|
* storage budget, which the platform also reports this way
|
|
@@ -273,6 +273,10 @@ export type DoCallClass =
|
|
|
273
273
|
| 'storage_reset'
|
|
274
274
|
/** The runtime flagged the error `retryable` itself. */
|
|
275
275
|
| 'retryable_flag'
|
|
276
|
+
/** The platform failed the call without saying why ("internal error;
|
|
277
|
+
* reference = …", isUnexplainedPlatformError): whether it ran is unknown.
|
|
278
|
+
* Not a transient class: only a callee that converges repeats it. */
|
|
279
|
+
| 'platform_internal'
|
|
276
280
|
/** The object shed the call under queue pressure. NEVER retried: retrying
|
|
277
281
|
* an overloaded object is what overloaded it. */
|
|
278
282
|
| 'overloaded'
|
|
@@ -317,6 +321,7 @@ export function classifyDoCall(input: unknown): DoCallClass {
|
|
|
317
321
|
const seen = new Set<Error>();
|
|
318
322
|
const messages: string[] = [];
|
|
319
323
|
let flagged = false;
|
|
324
|
+
let unexplained = false;
|
|
320
325
|
let link: Error | null = input;
|
|
321
326
|
while (link !== null && !seen.has(link)) {
|
|
322
327
|
seen.add(link);
|
|
@@ -324,6 +329,7 @@ export function classifyDoCall(input: unknown): DoCallClass {
|
|
|
324
329
|
|| isDoOverloaded(link.message);
|
|
325
330
|
if (overloaded) return 'overloaded';
|
|
326
331
|
if ('retryable' in link && link.retryable === true) flagged = true;
|
|
332
|
+
if (isUnexplainedPlatformError(link)) unexplained = true;
|
|
327
333
|
messages.push(link.message ?? '');
|
|
328
334
|
const cause: unknown = link.cause;
|
|
329
335
|
link = cause instanceof Error ? cause : null;
|
|
@@ -333,6 +339,8 @@ export function classifyDoCall(input: unknown): DoCallClass {
|
|
|
333
339
|
if (pattern.test(chain)) return transient;
|
|
334
340
|
}
|
|
335
341
|
if (flagged) return 'retryable_flag';
|
|
342
|
+
// Only where nothing in the chain explains it: a wrapper's own words are not the platform's.
|
|
343
|
+
if (unexplained) return 'platform_internal';
|
|
336
344
|
return 'permanent';
|
|
337
345
|
}
|
|
338
346
|
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A Node builtin of the realm's own process (`process.getBuiltinModule`),
|
|
3
|
+
* never a program's: a Nimbus runtime that puts a program's process in front
|
|
4
|
+
* of the realm's (node-shims) records the realm's own under
|
|
5
|
+
* `Symbol.for('nimbus.fronts')`, and a warm isolate keeps the last program's
|
|
6
|
+
* process as the global after it ends. Its builtins carry Node's argument
|
|
7
|
+
* checks and errors for programs, which the runtime's own callers pay for
|
|
8
|
+
* on every call. Undefined where the realm has no such builtin.
|
|
9
|
+
*/
|
|
10
|
+
type BuiltinProcess = { getBuiltinModule?(id: string): unknown };
|
|
11
|
+
type Fronts = WeakMap<object, { own: unknown }>;
|
|
12
|
+
|
|
13
|
+
export function realmBuiltin<T>(id: string): T | undefined {
|
|
14
|
+
try {
|
|
15
|
+
const global = globalThis as { process?: BuiltinProcess } & Record<symbol, Fronts | undefined>;
|
|
16
|
+
const current = global.process;
|
|
17
|
+
const own = (current && global[Symbol.for('nimbus.fronts')]?.get(current)?.own) as BuiltinProcess | undefined;
|
|
18
|
+
return (own ?? current)?.getBuiltinModule?.(id) as T | undefined;
|
|
19
|
+
} catch {
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
}
|
package/src/utf8.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* utf8.ts — the UTF-8 length of a string,
|
|
2
|
+
* utf8.ts — the UTF-8 length of a string, and the order of two strings by
|
|
3
|
+
* their UTF-8 bytes, without encoding them.
|
|
3
4
|
*
|
|
4
5
|
* Bounds on names and payloads are in bytes, and the hot ones (every path a
|
|
5
6
|
* W7 wave or a listing page carries) were measured by encoding the string
|
|
@@ -7,7 +8,15 @@
|
|
|
7
8
|
* TextEncoder would write: one to four per code point, and three for a
|
|
8
9
|
* lone surrogate, which it writes as U+FFFD.
|
|
9
10
|
*/
|
|
11
|
+
|
|
12
|
+
/** From this many UTF-16 units, a string is counted natively (longUtf8Length). */
|
|
13
|
+
const LONG_UNITS = 256;
|
|
14
|
+
const encoder = new TextEncoder();
|
|
15
|
+
/** Where longUtf8Length's encodes land: written, never read. */
|
|
16
|
+
let scratch: Uint8Array | undefined;
|
|
17
|
+
|
|
10
18
|
export function utf8Length(text: string): number {
|
|
19
|
+
if (text.length >= LONG_UNITS) return longUtf8Length(text);
|
|
11
20
|
let bytes = text.length;
|
|
12
21
|
for (let index = 0; index < text.length; index++) {
|
|
13
22
|
const unit = text.charCodeAt(index);
|
|
@@ -29,3 +38,62 @@ export function utf8Length(text: string): number {
|
|
|
29
38
|
}
|
|
30
39
|
return bytes;
|
|
31
40
|
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* A long string's UTF-8 length at the engine's speed: encoded a slice at a
|
|
44
|
+
* time into one reused buffer, only the count kept. encodeInto never splits
|
|
45
|
+
* a surrogate pair, so each slice starts on a code point.
|
|
46
|
+
*/
|
|
47
|
+
function longUtf8Length(text: string): number {
|
|
48
|
+
scratch ??= new Uint8Array(256 * 1024);
|
|
49
|
+
let bytes = 0;
|
|
50
|
+
for (let read = 0; read < text.length;) {
|
|
51
|
+
const done = encoder.encodeInto(read === 0 ? text : text.slice(read), scratch);
|
|
52
|
+
read += done.read;
|
|
53
|
+
bytes += done.written;
|
|
54
|
+
}
|
|
55
|
+
return bytes;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The code point at `index` of `text`: a pair is one past U+FFFF, and a lone
|
|
60
|
+
* surrogate is its own (as WTF-8 writes it, between U+D7FF and U+E000).
|
|
61
|
+
*/
|
|
62
|
+
function codePointAt(text: string, index: number): number {
|
|
63
|
+
const unit = text.charCodeAt(index);
|
|
64
|
+
if (unit < 0xd800 || unit > 0xdbff || index + 1 >= text.length) return unit;
|
|
65
|
+
const next = text.charCodeAt(index + 1);
|
|
66
|
+
return next >= 0xdc00 && next <= 0xdfff ? 0x10000 + ((unit - 0xd800) << 10) + (next - 0xdc00) : unit;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* `a` against `b` (negative, 0, positive) as their UTF-8 bytes order them,
|
|
71
|
+
* as SQLite's `ORDER BY` of a text column and git's index do: code point
|
|
72
|
+
* order. A string UTF-8 cannot write (a lone surrogate) is ordered as WTF-8
|
|
73
|
+
* writes it, its surrogate a code point of its own: the order is total, as a
|
|
74
|
+
* listing's cursor needs, and decided at the first code point that differs,
|
|
75
|
+
* so a directory's entries follow it together. Units equal in both are
|
|
76
|
+
* passed as they are; two that differ, neither a surrogate, are their own
|
|
77
|
+
* code points; the rest, rare in a path, is surrogatesAt's.
|
|
78
|
+
*/
|
|
79
|
+
export function compareUtf8(a: string, b: string): number {
|
|
80
|
+
const n = Math.min(a.length, b.length);
|
|
81
|
+
for (let i = 0; i < n; i++) {
|
|
82
|
+
const x = a.charCodeAt(i);
|
|
83
|
+
const y = b.charCodeAt(i);
|
|
84
|
+
if (x === y) continue;
|
|
85
|
+
if ((x < 0xd800 || x > 0xdfff) && (y < 0xd800 || y > 0xdfff)) return x - y;
|
|
86
|
+
return surrogatesAt(a, b, i);
|
|
87
|
+
}
|
|
88
|
+
return a.length - b.length;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** compareUtf8 of `a` and `b`, equal before `i`, where they differ at `i` and one unit there is a surrogate. */
|
|
92
|
+
function surrogatesAt(a: string, b: string, i: number): number {
|
|
93
|
+
// A low half after a high one the two share: the code points begin at the high one.
|
|
94
|
+
const before = i > 0 ? a.charCodeAt(i - 1) : 0;
|
|
95
|
+
const x = a.charCodeAt(i);
|
|
96
|
+
const y = b.charCodeAt(i);
|
|
97
|
+
const at = before >= 0xd800 && before <= 0xdbff && ((x >= 0xdc00 && x <= 0xdfff) || (y >= 0xdc00 && y <= 0xdfff)) ? i - 1 : i;
|
|
98
|
+
return codePointAt(a, at) - codePointAt(b, at);
|
|
99
|
+
}
|
package/src/w7-frame.ts
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
import { crc32 } from './crc32.js';
|
|
17
17
|
import { CHUNK_SIZE } from './limits.js';
|
|
18
18
|
import { utf8Length } from './utf8.js';
|
|
19
|
+
import { rootClampedSegments } from './lexical-path.js';
|
|
19
20
|
|
|
20
21
|
// The batch payload types live WITH the wire format that encodes them: every
|
|
21
22
|
// producer and consumer of a W7 stream speaks exactly these records, and the
|
|
@@ -79,10 +80,14 @@ export interface BatchWritePayload {
|
|
|
79
80
|
ops?: W7Op[];
|
|
80
81
|
}
|
|
81
82
|
|
|
82
|
-
/**
|
|
83
|
+
/**
|
|
84
|
+
* One attribute change (setattr): the mode, the owner, or the times; what
|
|
85
|
+
* chmod, chown and utimes each make. An owner side that is null keeps what
|
|
86
|
+
* the file has when the change lands (chown's -1).
|
|
87
|
+
*/
|
|
83
88
|
export type W7Attrs =
|
|
84
89
|
| { mode: number }
|
|
85
|
-
| { uid: number; gid: number }
|
|
90
|
+
| { uid: number | null; gid: number | null }
|
|
86
91
|
| { atime: number; mtime: number };
|
|
87
92
|
|
|
88
93
|
/**
|
|
@@ -94,6 +99,23 @@ export type W7Attrs =
|
|
|
94
99
|
* nothing makes a missing parent. The program-order counterpart of the
|
|
95
100
|
* syscall, where `file` and `directory` are a checkout's upserts.
|
|
96
101
|
*/
|
|
102
|
+
/** What a W7 open asks of its file, as open(2)'s O_ACCMODE: to read it, to write it, or both. */
|
|
103
|
+
export type W7OpenAccess = 'read' | 'write' | 'readwrite';
|
|
104
|
+
function isOpenAccess(value: unknown): value is W7OpenAccess {
|
|
105
|
+
return value === 'read' || value === 'write' || value === 'readwrite';
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** The rights an open's access grants its description: what a call through it may do. */
|
|
109
|
+
export function openAccessRights(access: W7OpenAccess): { read: boolean; write: boolean } {
|
|
110
|
+
return { read: access !== 'write', write: access !== 'read' };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** The permission bits an open asks of the file it reaches (4 read, 2 write): its access's, and write under `truncate`, as open(2)'s. */
|
|
114
|
+
export function openAccessMask(call: { access: W7OpenAccess; truncate?: true }): number {
|
|
115
|
+
const rights = openAccessRights(call.access);
|
|
116
|
+
return (rights.read ? 4 : 0) | (rights.write || call.truncate === true ? 2 : 0);
|
|
117
|
+
}
|
|
118
|
+
|
|
97
119
|
export type W7Call =
|
|
98
120
|
/**
|
|
99
121
|
* `ino`, on a call that makes a name (writeFile, appendFile, mkdir,
|
|
@@ -125,6 +147,14 @@ export type W7Call =
|
|
|
125
147
|
| { call: 'unlink'; path: string }
|
|
126
148
|
| { call: 'rmdir'; path: string }
|
|
127
149
|
| { call: 'symlink'; path: string; target: string; ino?: number }
|
|
150
|
+
/**
|
|
151
|
+
* copyfile as the session copies a file: the file \`from\` names (a link
|
|
152
|
+
* followed) to \`path\` (a link there followed, a file there replaced),
|
|
153
|
+
* by reference within one filesystem and by its bytes across, no byte of
|
|
154
|
+
* it in the wave. The copy's mode is a copy's there: a new file the
|
|
155
|
+
* source's less the umask, a file there its own.
|
|
156
|
+
*/
|
|
157
|
+
| { call: 'copyFile'; path: string; from: string }
|
|
128
158
|
/**
|
|
129
159
|
* rm(1) as fs.rm makes it: a file or link unlinked, a directory with all
|
|
130
160
|
* it holds when `recursive` (else EISDIR's refusal, as unlink's), and a
|
|
@@ -132,22 +162,27 @@ export type W7Call =
|
|
|
132
162
|
*/
|
|
133
163
|
| { call: 'rm'; path: string; recursive?: true; force?: true }
|
|
134
164
|
/** chown(2) of the link itself (lchown): the name's own entry, never what it names. */
|
|
135
|
-
| { call: 'lchown'; path: string; uid: number; gid: number }
|
|
165
|
+
| { call: 'lchown'; path: string; uid: number | null; gid: number | null }
|
|
136
166
|
/** utimes of the link itself (lutimes). */
|
|
137
167
|
| { call: 'lutimes'; path: string; atime: number; mtime: number }
|
|
138
168
|
/**
|
|
139
|
-
* open(2) of a file
|
|
140
|
-
* session's own open decides it: a name made
|
|
141
|
-
* `umask`) or refused (EEXIST when `exclusive`,
|
|
142
|
-
* EISDIR, EACCES for any access asked that the
|
|
169
|
+
* open(2) of a file with the access `access` asks, as O_ACCMODE does
|
|
170
|
+
* (W7OpenAccess), as the session's own open decides it: a name made
|
|
171
|
+
* (`create`, `mode` less `umask`) or refused (EEXIST when `exclusive`,
|
|
172
|
+
* ENOENT without `create`, EISDIR, EACCES for any access asked that the
|
|
173
|
+
* file's mode refuses, write too under `truncate`: openAccessMask),
|
|
143
174
|
* emptied when `truncate`, a link at the name followed unless `nofollow`
|
|
144
175
|
* (ELOOP). Its answer is the file's stat. With `description` (an id the
|
|
145
176
|
* process chose), the session keeps the open description under it, its
|
|
146
|
-
* access
|
|
147
|
-
* name it, and its `close` ends it. `ino`: the number
|
|
148
|
-
* given (a delegation's, as a mkdir's).
|
|
177
|
+
* rights the access's (openAccessRights); the description's write, append
|
|
178
|
+
* and ftruncate calls name it, and its `close` ends it. `ino`: the number
|
|
179
|
+
* a name it makes is given (a delegation's, as a mkdir's). `fchmod`: the
|
|
180
|
+
* file it opened then given exactly this mode (fchmod(2), as libuv's
|
|
181
|
+
* copyfile narrows its destination before a byte reaches it), part of the
|
|
182
|
+
* open: refused (EPERM: another's file), the open is refused and keeps no
|
|
183
|
+
* description, so a write through it is EBADF.
|
|
149
184
|
*/
|
|
150
|
-
| { call: 'open'; path: string; mode: number; ino?: number; umask?: number;
|
|
185
|
+
| { call: 'open'; path: string; mode: number; access: W7OpenAccess; ino?: number; umask?: number; create?: true; truncate?: true; exclusive?: true; nofollow?: true; description?: string; fchmod?: number }
|
|
151
186
|
/** close(2) of the session's open description `description` (an `open` call's). */
|
|
152
187
|
| { call: 'close'; path: string; description: string };
|
|
153
188
|
|
|
@@ -677,7 +712,7 @@ async function* decodeRecords(
|
|
|
677
712
|
if (v3) throw new Error(`w7-frame: unknown record tag ${envelope.tag}`);
|
|
678
713
|
summary.pathCount++;
|
|
679
714
|
summary.opCount++;
|
|
680
|
-
const value = parseObject(payload, 'call', ['call', 'path'], ['mode', 'target', 'ino', 'size', 'existing', 'recursive', 'force', 'uid', 'gid', 'atime', 'mtime', 'umask', '
|
|
715
|
+
const value = parseObject(payload, 'call', ['call', 'path'], ['mode', 'target', 'from', 'ino', 'size', 'existing', 'recursive', 'force', 'uid', 'gid', 'atime', 'mtime', 'umask', 'access', 'create', 'truncate', 'exclusive', 'nofollow', 'description', 'fchmod']);
|
|
681
716
|
yield { type: 'call', call: parsePathCall(value, (path, label) => ownedPaths.claim(canonicalPath(path, label))) };
|
|
682
717
|
break;
|
|
683
718
|
}
|
|
@@ -812,8 +847,8 @@ async function* encodeFile(file: EncoderFile, state: EncoderState): AsyncGenerat
|
|
|
812
847
|
let fileCheck = 0;
|
|
813
848
|
let chunkId = 0;
|
|
814
849
|
const pieces = file.source === null ? givenChunks(file.chunks) : fixedChunks(file.inode, file.source);
|
|
850
|
+
const contentBytes = TEXT_ENCODER.encode(file.contentId);
|
|
815
851
|
for await (const data of pieces) {
|
|
816
|
-
const contentBytes = new TextEncoder().encode(file.contentId);
|
|
817
852
|
const prefix = new Uint8Array(4 + contentBytes.length + 8);
|
|
818
853
|
writeU32LE(prefix, 0, contentBytes.length);
|
|
819
854
|
prefix.set(contentBytes, 4);
|
|
@@ -880,16 +915,20 @@ function encodeMetadataRecord(
|
|
|
880
915
|
value: object,
|
|
881
916
|
state?: EncoderState,
|
|
882
917
|
): Uint8Array[] {
|
|
883
|
-
const payload =
|
|
918
|
+
const payload = TEXT_ENCODER.encode(JSON.stringify(value));
|
|
884
919
|
if (payload.byteLength > MAX_METADATA_BYTES) {
|
|
885
920
|
throw new Error(`w7-frame: metadata record exceeds ${MAX_METADATA_BYTES} bytes`);
|
|
886
921
|
}
|
|
887
|
-
|
|
922
|
+
// The header and the payload in one buffer, checked as one (crc32 composes).
|
|
923
|
+
const record = new Uint8Array(5 + payload.byteLength);
|
|
924
|
+
record[0] = tag;
|
|
925
|
+
writeU32LE(record, 1, payload.byteLength);
|
|
926
|
+
record.set(payload, 5);
|
|
888
927
|
if (state && tag !== RecordTag.BatchEnd) {
|
|
889
|
-
state.batchCheck = updateRecordCheck(state.batchCheck,
|
|
928
|
+
state.batchCheck = updateRecordCheck(state.batchCheck, record);
|
|
890
929
|
state.summary.recordCount++;
|
|
891
930
|
}
|
|
892
|
-
return [
|
|
931
|
+
return [record];
|
|
893
932
|
}
|
|
894
933
|
|
|
895
934
|
/**
|
|
@@ -1020,7 +1059,7 @@ function parseBatchBegin(bytes: Uint8Array, mode: W7Mode): BatchBeginMetadata {
|
|
|
1020
1059
|
function parseAttrs(value: Record<string, unknown>, label: string): W7Attrs {
|
|
1021
1060
|
const keys = Object.keys(value).sort().join(',');
|
|
1022
1061
|
if (keys === 'mode') return { mode: u32(value.mode, `${label} mode`) };
|
|
1023
|
-
if (keys === 'gid,uid') return { uid:
|
|
1062
|
+
if (keys === 'gid,uid') return { uid: ownerId(value.uid, `${label} uid`), gid: ownerId(value.gid, `${label} gid`) };
|
|
1024
1063
|
if (keys === 'atime,mtime') return { atime: safeInteger(value.atime, `${label} atime`), mtime: safeInteger(value.mtime, `${label} mtime`) };
|
|
1025
1064
|
throw new Error(`w7-frame: ${label} changes the mode, the owner or the times, one of them: got ${keys || 'nothing'}`);
|
|
1026
1065
|
}
|
|
@@ -1108,6 +1147,9 @@ function parsePathCall(value: Record<string, unknown>, path: (value: unknown, la
|
|
|
1108
1147
|
call: 'symlink', path: path(value.path, 'symlink path'), target: boundedString(value.target, 'symlink target', MAX_PATH_BYTES),
|
|
1109
1148
|
...(value.ino === undefined ? {} : { ino: inodeNumber(value.ino, 'symlink ino') }),
|
|
1110
1149
|
};
|
|
1150
|
+
case 'copyFile':
|
|
1151
|
+
if (keys !== 'call,from,path') break;
|
|
1152
|
+
return { call: 'copyFile', path: path(value.path, 'copyFile path'), from: path(value.from, 'copyFile from') };
|
|
1111
1153
|
case 'ftruncate':
|
|
1112
1154
|
if (keys.replace(',description', '').replace(',ino', '') !== 'call,path,size') break;
|
|
1113
1155
|
return {
|
|
@@ -1131,25 +1173,27 @@ function parsePathCall(value: Record<string, unknown>, path: (value: unknown, la
|
|
|
1131
1173
|
}
|
|
1132
1174
|
case 'lchown':
|
|
1133
1175
|
if (keys !== 'call,gid,path,uid') break;
|
|
1134
|
-
return { call: 'lchown', path: path(value.path, 'lchown path'), uid:
|
|
1176
|
+
return { call: 'lchown', path: path(value.path, 'lchown path'), uid: ownerId(value.uid, 'lchown uid'), gid: ownerId(value.gid, 'lchown gid') };
|
|
1135
1177
|
case 'open': {
|
|
1136
|
-
const flags = ['create', 'exclusive', 'nofollow', '
|
|
1178
|
+
const flags = ['create', 'exclusive', 'nofollow', 'truncate'] as const;
|
|
1137
1179
|
let required = keys;
|
|
1138
|
-
for (const flag of ['create', 'description', 'exclusive', '
|
|
1139
|
-
if (required !== 'call,mode,path') break;
|
|
1180
|
+
for (const flag of ['create', 'description', 'exclusive', 'fchmod', 'ino', 'nofollow', 'truncate', 'umask']) required = required.replace(`,${flag}`, '');
|
|
1181
|
+
if (required !== 'access,call,mode,path') break;
|
|
1140
1182
|
for (const flag of flags) {
|
|
1141
1183
|
if (value[flag] !== undefined && value[flag] !== true) throw new Error(`w7-frame: open ${flag} is true or absent`);
|
|
1142
1184
|
}
|
|
1185
|
+
const access = value.access;
|
|
1186
|
+
if (!isOpenAccess(access)) throw new Error(`w7-frame: open access is 'read', 'write' or 'readwrite': got ${String(access)}`);
|
|
1143
1187
|
return {
|
|
1144
|
-
call: 'open', path: path(value.path, 'open path'), mode: u32(value.mode, 'open mode'),
|
|
1188
|
+
call: 'open', path: path(value.path, 'open path'), mode: u32(value.mode, 'open mode'), access,
|
|
1145
1189
|
...(value.ino === undefined ? {} : { ino: inodeNumber(value.ino, 'open ino') }),
|
|
1146
1190
|
...(value.umask === undefined ? {} : { umask: umaskOf(value.umask, 'open umask') }),
|
|
1147
1191
|
...(value.create === true ? { create: true as const } : {}),
|
|
1148
1192
|
...(value.truncate === true ? { truncate: true as const } : {}),
|
|
1149
1193
|
...(value.exclusive === true ? { exclusive: true as const } : {}),
|
|
1150
1194
|
...(value.nofollow === true ? { nofollow: true as const } : {}),
|
|
1151
|
-
...(value.read === true ? { read: true as const } : {}),
|
|
1152
1195
|
...(value.description === undefined ? {} : { description: descriptionId(value.description, 'open description') }),
|
|
1196
|
+
...(value.fchmod === undefined ? {} : { fchmod: u32(value.fchmod, 'open fchmod') }),
|
|
1153
1197
|
};
|
|
1154
1198
|
}
|
|
1155
1199
|
case 'lutimes':
|
|
@@ -1384,15 +1428,7 @@ class PathOwnership {
|
|
|
1384
1428
|
}
|
|
1385
1429
|
|
|
1386
1430
|
function normalizePath(path: string): string {
|
|
1387
|
-
|
|
1388
|
-
for (const segment of path.split('/')) {
|
|
1389
|
-
if (segment === '..') {
|
|
1390
|
-
if (out.length > 0) out.pop();
|
|
1391
|
-
} else if (segment !== '' && segment !== '.') {
|
|
1392
|
-
out.push(segment);
|
|
1393
|
-
}
|
|
1394
|
-
}
|
|
1395
|
-
return out.join('/');
|
|
1431
|
+
return rootClampedSegments(path).join('/');
|
|
1396
1432
|
}
|
|
1397
1433
|
|
|
1398
1434
|
function parentPath(path: string): string {
|
|
@@ -1415,6 +1451,11 @@ function safeInteger(value: unknown, label: string): number {
|
|
|
1415
1451
|
return value as number;
|
|
1416
1452
|
}
|
|
1417
1453
|
|
|
1454
|
+
/** A chown side: a uid or gid, or null, which keeps the file's. */
|
|
1455
|
+
function ownerId(value: unknown, label: string): number | null {
|
|
1456
|
+
return value === null ? null : u32(value, label);
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1418
1459
|
function u32(value: unknown, label: string): number {
|
|
1419
1460
|
const integer = safeInteger(value, label);
|
|
1420
1461
|
if (integer > 0xffff_ffff) throw new Error(`w7-frame: ${label} exceeds uint32`);
|
package/src/wave-writer.ts
CHANGED
|
@@ -164,7 +164,7 @@ export interface WaveWriterOptions<Meta = undefined> {
|
|
|
164
164
|
root: string | null;
|
|
165
165
|
/** An existing worktree: only directories strictly below it are published. */
|
|
166
166
|
worktreeRoot?: string | null;
|
|
167
|
-
/** Record paths are relative to it (joined with '/'); omitted, they are VFS paths. */
|
|
167
|
+
/** Record paths are relative to it (joined with '/'; '' is the base itself); omitted, they are VFS paths. */
|
|
168
168
|
base?: string;
|
|
169
169
|
/** Wall-clock time (ms) after which no new wave starts. */
|
|
170
170
|
deadline?: number | null;
|
|
@@ -300,7 +300,6 @@ export class WaveWriter<Meta = undefined> {
|
|
|
300
300
|
private ownedPathBytes = 0;
|
|
301
301
|
private pin: Pin | null = null;
|
|
302
302
|
private inFlight: Promise<void> | null = null;
|
|
303
|
-
private cutQueue: Promise<unknown> = Promise.resolve();
|
|
304
303
|
private sequence = 0;
|
|
305
304
|
private failure: WaveFailure | null = null;
|
|
306
305
|
/** Owners (records' `meta`) whose records a failed wave carried. */
|
|
@@ -532,7 +531,8 @@ export class WaveWriter<Meta = undefined> {
|
|
|
532
531
|
// ── Internals ────────────────────────────────────────────────────────
|
|
533
532
|
|
|
534
533
|
private key(path: string): string {
|
|
535
|
-
|
|
534
|
+
if (!this.options.base) return path;
|
|
535
|
+
return path ? this.options.base + '/' + path : this.options.base;
|
|
536
536
|
}
|
|
537
537
|
|
|
538
538
|
private tally(): Tally {
|
|
@@ -551,13 +551,24 @@ export class WaveWriter<Meta = undefined> {
|
|
|
551
551
|
if (admit) this.owned.add(path);
|
|
552
552
|
}
|
|
553
553
|
|
|
554
|
-
/**
|
|
555
|
-
|
|
554
|
+
/**
|
|
555
|
+
* The directories a wave publishes for something at or below `dir`:
|
|
556
|
+
* `dir` and each one above it, deepest first, up to the writer's root
|
|
557
|
+
* (included) and inside its worktree root.
|
|
558
|
+
*/
|
|
559
|
+
private *chain(dir: string): Generator<string, void, undefined> {
|
|
556
560
|
const { root, worktreeRoot = null } = this.options;
|
|
557
|
-
let current =
|
|
558
|
-
|
|
559
|
-
if (
|
|
560
|
-
|
|
561
|
+
for (let current = dir; current; current = parentOf(current)) {
|
|
562
|
+
if (root && current !== root && !current.startsWith(root + '/')) return;
|
|
563
|
+
if (worktreeRoot !== null && !current.startsWith(worktreeRoot + '/')) return;
|
|
564
|
+
yield current;
|
|
565
|
+
if (current === root) return;
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** The chain from `path` upward, to the first directory already owned. */
|
|
570
|
+
private walkChain(path: string, admit: boolean, tally: Tally): void {
|
|
571
|
+
for (const current of this.chain(path)) {
|
|
561
572
|
this.counters.ownershipVisits++;
|
|
562
573
|
if (this.ownedDirectories.has(current)) break;
|
|
563
574
|
if (admit) this.ownedDirectories.add(current);
|
|
@@ -566,8 +577,6 @@ export class WaveWriter<Meta = undefined> {
|
|
|
566
577
|
tally.pathBytes += utf8Length(current);
|
|
567
578
|
if (admit) this.owned.add(current);
|
|
568
579
|
}
|
|
569
|
-
if (current === root) break;
|
|
570
|
-
current = parentOf(current);
|
|
571
580
|
}
|
|
572
581
|
}
|
|
573
582
|
|
|
@@ -636,16 +645,14 @@ export class WaveWriter<Meta = undefined> {
|
|
|
636
645
|
}
|
|
637
646
|
|
|
638
647
|
/**
|
|
639
|
-
* Send the buffered wave once the one in flight has published.
|
|
640
|
-
*
|
|
648
|
+
* Send the buffered wave once the one in flight has published. Only a
|
|
649
|
+
* mutation cuts, and mutations run one at a time, so at most one wave is
|
|
650
|
+
* in flight and one buffers.
|
|
641
651
|
*/
|
|
642
|
-
private cut(): Promise<void> {
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
}
|
|
647
|
-
|
|
648
|
-
private async cutNow(): Promise<void> {
|
|
652
|
+
private async cut(): Promise<void> {
|
|
653
|
+
// A later turn than the mutation that asks: a pin set or cleared by a
|
|
654
|
+
// microtask queued before the cut is the wave's.
|
|
655
|
+
await Promise.resolve();
|
|
649
656
|
if (this.inFlight) {
|
|
650
657
|
const waitStarted = Date.now();
|
|
651
658
|
await this.inFlight.catch(() => {});
|
|
@@ -848,16 +855,10 @@ export class WaveWriter<Meta = undefined> {
|
|
|
848
855
|
/** The directories the buffered records publish, shallowest first. */
|
|
849
856
|
private publishedDirectories(): string[] {
|
|
850
857
|
const dirs = new Set<string>();
|
|
851
|
-
const { root, worktreeRoot = null } = this.options;
|
|
852
858
|
const collect = (path: string): void => {
|
|
853
|
-
|
|
854
|
-
while (current) {
|
|
855
|
-
if (root && current !== root && !current.startsWith(root + '/')) break;
|
|
856
|
-
if (worktreeRoot !== null && !current.startsWith(worktreeRoot + '/')) break;
|
|
859
|
+
for (const current of this.chain(path)) {
|
|
857
860
|
if (dirs.has(current)) break;
|
|
858
861
|
dirs.add(current);
|
|
859
|
-
if (current === root) break;
|
|
860
|
-
current = parentOf(current);
|
|
861
862
|
}
|
|
862
863
|
};
|
|
863
864
|
for (const path of this.records.keys()) collect(parentOf(path));
|