@nimbus-sh/core 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -26
- package/dist/_shared/byte-stream.d.ts +5 -4
- package/dist/_shared/byte-stream.d.ts.map +1 -1
- package/dist/_shared/cache-stats.d.ts +23 -1
- package/dist/_shared/cache-stats.d.ts.map +1 -1
- package/dist/_shared/cache-stats.js +13 -1
- package/dist/_shared/exports-resolver.d.ts +1 -1
- package/dist/_shared/exports-resolver.js +1 -1
- package/dist/_shared/real-node-imports.d.ts +7 -0
- package/dist/_shared/real-node-imports.d.ts.map +1 -1
- package/dist/_shared/real-node-imports.js +8 -0
- package/dist/_shared/retry.js +1 -1
- package/dist/_shared/tarball.d.ts +23 -5
- package/dist/_shared/tarball.d.ts.map +1 -1
- package/dist/_shared/tarball.js +38 -5
- package/dist/_shared/vfs-write-ledger.d.ts.map +1 -1
- package/dist/_shared/vfs-write-ledger.js +22 -16
- package/dist/constants.d.ts +5 -11
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +16 -56
- package/dist/runtime/assets-loader.js +1 -1
- package/dist/runtime/comment-strip.d.ts +0 -35
- package/dist/runtime/comment-strip.d.ts.map +1 -1
- package/dist/runtime/comment-strip.js +48 -20
- package/dist/runtime/esbuild-service.d.ts +38 -42
- package/dist/runtime/esbuild-service.d.ts.map +1 -1
- package/dist/runtime/esbuild-service.js +494 -197
- package/dist/runtime/javascript-ast.d.ts.map +1 -1
- package/dist/runtime/javascript-ast.js +45 -7
- package/dist/runtime/opentui-wasm-backend.d.ts.map +1 -1
- package/dist/runtime/opentui-wasm-backend.js +18 -16
- package/dist/runtime/port-registry.d.ts +4 -2
- package/dist/runtime/port-registry.d.ts.map +1 -1
- package/dist/runtime/port-registry.js +60 -8
- package/dist/runtime/virtual-socket-kernel.generated.d.ts +1 -1
- package/dist/runtime/virtual-socket-kernel.generated.js +1 -1
- package/dist/shell/stdin-adapter.d.ts +11 -0
- package/dist/shell/stdin-adapter.d.ts.map +1 -0
- package/dist/shell/stdin-adapter.js +59 -0
- package/dist/shell/unix-commands.d.ts +2 -1
- package/dist/shell/unix-commands.d.ts.map +1 -1
- package/dist/shell/unix-commands.js +1078 -50
- package/dist/substrate/lifo/commands/io/dd.js +1 -1
- package/dist/substrate/lifo/commands/net/curl.d.ts.map +1 -1
- package/dist/substrate/lifo/commands/net/curl.js +270 -54
- package/dist/substrate/lifo/commands/system/npm.d.ts +18 -1
- package/dist/substrate/lifo/commands/system/npm.d.ts.map +1 -1
- package/dist/substrate/lifo/commands/system/npm.js +26 -43
- package/dist/substrate/lifo/commands/text/head.d.ts.map +1 -1
- package/dist/substrate/lifo/commands/text/head.js +2 -1
- package/dist/substrate/lifo/commands/text/sed.d.ts.map +1 -1
- package/dist/substrate/lifo/commands/text/sed.js +499 -117
- package/dist/substrate/lifo/commands/types.d.ts +8 -4
- package/dist/substrate/lifo/commands/types.d.ts.map +1 -1
- package/dist/substrate/lifo/node-compat/zlib.d.ts +37 -18
- package/dist/substrate/lifo/node-compat/zlib.d.ts.map +1 -1
- package/dist/substrate/lifo/node-compat/zlib.js +122 -23
- package/dist/substrate/lifo/shell/interpreter.d.ts +9 -1
- package/dist/substrate/lifo/shell/interpreter.d.ts.map +1 -1
- package/dist/substrate/lifo/shell/interpreter.js +79 -29
- package/dist/substrate/lifo/shell/pipe.d.ts +16 -0
- package/dist/substrate/lifo/shell/pipe.d.ts.map +1 -1
- package/dist/substrate/lifo/shell/pipe.js +62 -18
- package/dist/substrate/lifo/shell/terminal-stdin.d.ts +20 -3
- package/dist/substrate/lifo/shell/terminal-stdin.d.ts.map +1 -1
- package/dist/substrate/lifo/shell/terminal-stdin.js +60 -24
- package/dist/vfs/seed-project.d.ts.map +1 -1
- package/dist/vfs/seed-project.js +1 -1
- package/dist/vfs/sqlite-vfs.d.ts +21 -30
- package/dist/vfs/sqlite-vfs.d.ts.map +1 -1
- package/dist/vfs/sqlite-vfs.js +113 -29
- package/dist/workspace/nimbus-workspace.d.ts +15 -0
- package/dist/workspace/nimbus-workspace.d.ts.map +1 -1
- package/dist/workspace/nimbus-workspace.js +20 -3
- package/dist/workspace/supervisor-op.d.ts +23 -0
- package/dist/workspace/supervisor-op.d.ts.map +1 -0
- package/dist/workspace/supervisor-op.js +92 -0
- package/package.json +3 -2
- package/src/_shared/byte-stream.ts +5 -4
- package/src/_shared/cache-stats.ts +27 -1
- package/src/_shared/exports-resolver.ts +1 -1
- package/src/_shared/real-node-imports.ts +8 -0
- package/src/_shared/retry.ts +1 -1
- package/src/_shared/tarball.ts +56 -5
- package/src/_shared/vfs-write-ledger.ts +22 -16
- package/src/constants.ts +30 -60
- package/src/runtime/assets-loader.ts +1 -1
- package/src/runtime/comment-strip.ts +42 -23
- package/src/runtime/esbuild-service.ts +566 -217
- package/src/runtime/javascript-ast.ts +40 -9
- package/src/runtime/opentui-wasm-backend.ts +19 -17
- package/src/runtime/port-registry.ts +66 -8
- package/src/runtime/virtual-socket-kernel.generated.ts +1 -1
- package/src/shell/stdin-adapter.ts +58 -0
- package/src/shell/unix-commands.ts +1054 -50
- package/src/substrate/lifo/commands/io/dd.ts +2 -2
- package/src/substrate/lifo/commands/net/curl.ts +310 -58
- package/src/substrate/lifo/commands/system/npm.ts +58 -38
- package/src/substrate/lifo/commands/text/head.ts +2 -1
- package/src/substrate/lifo/commands/text/sed.ts +519 -109
- package/src/substrate/lifo/commands/types.ts +8 -5
- package/src/substrate/lifo/node-compat/zlib.ts +155 -26
- package/src/substrate/lifo/shell/interpreter.ts +74 -27
- package/src/substrate/lifo/shell/pipe.ts +65 -24
- package/src/substrate/lifo/shell/terminal-stdin.ts +66 -30
- package/src/vfs/seed-project.ts +3 -3
- package/src/vfs/sqlite-vfs.ts +128 -64
- package/src/workspace/nimbus-workspace.ts +26 -2
- package/src/workspace/supervisor-op.ts +117 -0
- package/dist/_shared/install-phase.d.ts +0 -47
- package/dist/_shared/install-phase.d.ts.map +0 -1
- package/dist/_shared/install-phase.js +0 -1
- package/dist/_shared/rpc-dispose.d.ts +0 -4
- package/dist/_shared/rpc-dispose.d.ts.map +0 -1
- package/dist/_shared/rpc-dispose.js +0 -30
- package/dist/_shared/w7-frame.d.ts +0 -79
- package/dist/_shared/w7-frame.d.ts.map +0 -1
- package/dist/_shared/w7-frame.js +0 -840
- package/dist/_shared/weighted-credit-pool.d.ts +0 -61
- package/dist/_shared/weighted-credit-pool.d.ts.map +0 -1
- package/dist/_shared/weighted-credit-pool.js +0 -196
- package/dist/observability/diag-counters.d.ts +0 -236
- package/dist/observability/diag-counters.d.ts.map +0 -1
- package/dist/observability/diag-counters.js +0 -195
- package/dist/observability/heap-estimate.d.ts +0 -225
- package/dist/observability/heap-estimate.d.ts.map +0 -1
- package/dist/observability/heap-estimate.js +0 -185
- package/dist/observability/heavy-alloc-coord.d.ts +0 -93
- package/dist/observability/heavy-alloc-coord.d.ts.map +0 -1
- package/dist/observability/heavy-alloc-coord.js +0 -181
- package/dist/observability/oom-classify.d.ts +0 -138
- package/dist/observability/oom-classify.d.ts.map +0 -1
- package/dist/observability/oom-classify.js +0 -268
- package/dist/observability/oom-discriminator.d.ts +0 -176
- package/dist/observability/oom-discriminator.d.ts.map +0 -1
- package/dist/observability/oom-discriminator.js +0 -272
- package/src/_shared/install-phase.ts +0 -56
- package/src/_shared/rpc-dispose.ts +0 -31
- package/src/_shared/w7-frame.ts +0 -1013
- package/src/_shared/weighted-credit-pool.ts +0 -244
- package/src/observability/diag-counters.ts +0 -343
- package/src/observability/heap-estimate.ts +0 -331
- package/src/observability/heavy-alloc-coord.ts +0 -241
- package/src/observability/oom-classify.ts +0 -260
- package/src/observability/oom-discriminator.ts +0 -405
package/src/_shared/w7-frame.ts
DELETED
|
@@ -1,1013 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* W7 v3 — incremental typed records for streamed bulk filesystem writes.
|
|
3
|
-
* The format is internal: every producer and consumer deploys together.
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
import { CHUNK_SIZE } from '../constants.js';
|
|
7
|
-
import type {
|
|
8
|
-
BatchChunkEntry,
|
|
9
|
-
BatchInodeEntry,
|
|
10
|
-
BatchWritePayload,
|
|
11
|
-
VfsInodeKind,
|
|
12
|
-
} from '../vfs/sqlite-vfs.js';
|
|
13
|
-
|
|
14
|
-
export const W7_MAGIC = new Uint8Array([0x4e, 0x57, 0x37, 0x03]);
|
|
15
|
-
|
|
16
|
-
const ENCODER_QUEUE_HWM = 0;
|
|
17
|
-
const MAX_METADATA_BYTES = 64 * 1024;
|
|
18
|
-
const MAX_PATH_BYTES = 64 * 1024;
|
|
19
|
-
const MAX_BATCH_ID_BYTES = 128;
|
|
20
|
-
const MAX_CONTENT_ID_BYTES = 256;
|
|
21
|
-
export const W7_MAX_PATHS_PER_BATCH = 128;
|
|
22
|
-
export const W7_MAX_OWNED_PATH_BYTES = 64 * 1024;
|
|
23
|
-
export const W7_MAX_RECORD_BYTES = 5 + 4 + MAX_CONTENT_ID_BYTES + 8 + CHUNK_SIZE;
|
|
24
|
-
|
|
25
|
-
const enum RecordTag {
|
|
26
|
-
BatchBegin = 1,
|
|
27
|
-
Delete = 2,
|
|
28
|
-
Directory = 3,
|
|
29
|
-
FileBegin = 4,
|
|
30
|
-
FileChunk = 5,
|
|
31
|
-
FileEnd = 6,
|
|
32
|
-
BatchEnd = 7,
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
const MODE = 'path-atomic-committed-prefix' as const;
|
|
36
|
-
|
|
37
|
-
interface BatchBeginMetadata {
|
|
38
|
-
id: string;
|
|
39
|
-
mode: typeof MODE;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
interface DeleteMetadata {
|
|
43
|
-
path: string;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
interface InodeMetadata {
|
|
47
|
-
path: string;
|
|
48
|
-
atime?: number;
|
|
49
|
-
mtime: number;
|
|
50
|
-
mode: number;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
interface DirectoryMetadata extends InodeMetadata {
|
|
54
|
-
kind: 'directory';
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
interface FileBeginMetadata extends InodeMetadata {
|
|
58
|
-
kind: 'file' | 'symlink';
|
|
59
|
-
contentId: string;
|
|
60
|
-
size: number;
|
|
61
|
-
chunkCount: number;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
interface FileEndMetadata {
|
|
65
|
-
contentId: string;
|
|
66
|
-
size: number;
|
|
67
|
-
chunkCount: number;
|
|
68
|
-
check: number;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export interface W7BatchSummary {
|
|
72
|
-
recordCount: number;
|
|
73
|
-
pathCount: number;
|
|
74
|
-
deleteCount: number;
|
|
75
|
-
directoryCount: number;
|
|
76
|
-
fileCount: number;
|
|
77
|
-
chunkCount: number;
|
|
78
|
-
byteCount: number;
|
|
79
|
-
check: number;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
export interface W7ChunkRetention {
|
|
83
|
-
readonly bytes: number;
|
|
84
|
-
release(): void;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
export interface W7DecodeOptions {
|
|
88
|
-
signal?: AbortSignal;
|
|
89
|
-
retainChunk?: (byteLength: number, signal?: AbortSignal) => Promise<W7ChunkRetention>;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
type W7DirectoryInode = BatchInodeEntry & { kind: 'directory'; isDir: true };
|
|
93
|
-
type W7ContentInode = BatchInodeEntry & { kind: 'file' | 'symlink'; isDir: false };
|
|
94
|
-
|
|
95
|
-
export type W7DecodedRecord =
|
|
96
|
-
| { type: 'delete'; path: string }
|
|
97
|
-
| { type: 'directory'; inode: W7DirectoryInode }
|
|
98
|
-
| {
|
|
99
|
-
type: 'file-begin';
|
|
100
|
-
streamContentId: string;
|
|
101
|
-
inode: W7ContentInode;
|
|
102
|
-
}
|
|
103
|
-
| {
|
|
104
|
-
type: 'file-chunk';
|
|
105
|
-
streamContentId: string;
|
|
106
|
-
path: string;
|
|
107
|
-
chunkId: number;
|
|
108
|
-
data: Uint8Array;
|
|
109
|
-
retention: W7ChunkRetention;
|
|
110
|
-
}
|
|
111
|
-
| {
|
|
112
|
-
type: 'file-end';
|
|
113
|
-
streamContentId: string;
|
|
114
|
-
path: string;
|
|
115
|
-
size: number;
|
|
116
|
-
chunkCount: number;
|
|
117
|
-
check: number;
|
|
118
|
-
}
|
|
119
|
-
| { type: 'batch-end'; summary: W7BatchSummary };
|
|
120
|
-
|
|
121
|
-
export interface W7DecodedStream {
|
|
122
|
-
readonly batchId: string;
|
|
123
|
-
readonly mode: typeof MODE;
|
|
124
|
-
readonly records: AsyncIterable<W7DecodedRecord>;
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
interface EncoderFile {
|
|
128
|
-
inode: W7ContentInode;
|
|
129
|
-
contentId: string;
|
|
130
|
-
chunks: BatchChunkEntry[];
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
interface EncoderState {
|
|
134
|
-
batchCheck: number;
|
|
135
|
-
summary: Omit<W7BatchSummary, 'check'>;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
/** Encode one bounded record per pull; no batch-sized metadata header exists. */
|
|
139
|
-
export function encodeWriteBatchStream(payload: BatchWritePayload): ReadableStream<Uint8Array> {
|
|
140
|
-
const batchId = crypto.randomUUID();
|
|
141
|
-
const { deletes, directories, files } = preparePayload(payload, batchId);
|
|
142
|
-
const iterator = encodeRecords(batchId, deletes, directories, files);
|
|
143
|
-
let closed = false;
|
|
144
|
-
let magicEmitted = false;
|
|
145
|
-
const source: UnderlyingByteSource = {
|
|
146
|
-
type: 'bytes',
|
|
147
|
-
pull(controller) {
|
|
148
|
-
if (closed) return;
|
|
149
|
-
try {
|
|
150
|
-
if (!magicEmitted) {
|
|
151
|
-
magicEmitted = true;
|
|
152
|
-
controller.enqueue(W7_MAGIC.slice());
|
|
153
|
-
return;
|
|
154
|
-
}
|
|
155
|
-
const next = iterator.next();
|
|
156
|
-
if (next.done) {
|
|
157
|
-
closed = true;
|
|
158
|
-
controller.close();
|
|
159
|
-
return;
|
|
160
|
-
}
|
|
161
|
-
for (const part of next.value) controller.enqueue(part);
|
|
162
|
-
} catch (error) {
|
|
163
|
-
closed = true;
|
|
164
|
-
controller.error(error);
|
|
165
|
-
}
|
|
166
|
-
},
|
|
167
|
-
cancel() {
|
|
168
|
-
closed = true;
|
|
169
|
-
iterator.return?.(undefined);
|
|
170
|
-
},
|
|
171
|
-
};
|
|
172
|
-
return new ReadableStream<Uint8Array>(source as never, {
|
|
173
|
-
highWaterMark: ENCODER_QUEUE_HWM,
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
/**
|
|
178
|
-
* Parse the v3 preamble eagerly, then expose validated operation records
|
|
179
|
-
* incrementally. Chunk credit is acquired after its bounded header validates
|
|
180
|
-
* and before its payload bytes are read or copied.
|
|
181
|
-
*/
|
|
182
|
-
export async function decodeWriteBatchStream(
|
|
183
|
-
stream: ReadableStream<Uint8Array>,
|
|
184
|
-
options: W7DecodeOptions = {},
|
|
185
|
-
): Promise<W7DecodedStream> {
|
|
186
|
-
let reader: ReadableStreamBYOBReader;
|
|
187
|
-
try {
|
|
188
|
-
reader = stream.getReader({ mode: 'byob' });
|
|
189
|
-
} catch {
|
|
190
|
-
throw new Error('w7-frame: stream must be a byte-oriented ReadableStream');
|
|
191
|
-
}
|
|
192
|
-
const buffer = new ExactByteReader(reader);
|
|
193
|
-
let handedOff = false;
|
|
194
|
-
try {
|
|
195
|
-
throwIfAborted(options.signal);
|
|
196
|
-
const magic = await buffer.readExact(W7_MAGIC.length, 'magic');
|
|
197
|
-
if (!bytesEqual(magic, W7_MAGIC)) {
|
|
198
|
-
const version = magic.length === 4
|
|
199
|
-
&& magic[0] === 0x4e && magic[1] === 0x57 && magic[2] === 0x37
|
|
200
|
-
? magic[3]
|
|
201
|
-
: null;
|
|
202
|
-
if (version !== null) {
|
|
203
|
-
throw new Error(`w7-frame: unsupported protocol version ${version}; expected 3`);
|
|
204
|
-
}
|
|
205
|
-
throw new Error(`w7-frame: bad magic, expected NW7\\x03, got ${hex(magic)}`);
|
|
206
|
-
}
|
|
207
|
-
const beginEnvelope = await readEnvelope(buffer, 'batch-begin');
|
|
208
|
-
if (beginEnvelope.tag !== RecordTag.BatchBegin) {
|
|
209
|
-
throw new Error(`w7-frame: first record must be batch-begin, got tag ${beginEnvelope.tag}`);
|
|
210
|
-
}
|
|
211
|
-
if (beginEnvelope.length > MAX_METADATA_BYTES) {
|
|
212
|
-
throw new Error(
|
|
213
|
-
`w7-frame: batch-begin length ${beginEnvelope.length} exceeds ${MAX_METADATA_BYTES}`,
|
|
214
|
-
);
|
|
215
|
-
}
|
|
216
|
-
const beginPayload = await buffer.readExact(beginEnvelope.length, 'batch-begin payload');
|
|
217
|
-
const begin = parseBatchBegin(beginPayload);
|
|
218
|
-
const initialCheck = updateRecordCheck(CRC_SEED, beginEnvelope.header, beginPayload);
|
|
219
|
-
handedOff = true;
|
|
220
|
-
return {
|
|
221
|
-
batchId: begin.id,
|
|
222
|
-
mode: begin.mode,
|
|
223
|
-
records: decodeRecords(stream, reader, buffer, options, initialCheck),
|
|
224
|
-
};
|
|
225
|
-
} catch (error) {
|
|
226
|
-
if (!handedOff) await cancelReader(reader, error);
|
|
227
|
-
throw error;
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
async function* decodeRecords(
|
|
232
|
-
stream: ReadableStream<Uint8Array>,
|
|
233
|
-
reader: ReadableStreamBYOBReader,
|
|
234
|
-
buffer: ExactByteReader,
|
|
235
|
-
options: W7DecodeOptions,
|
|
236
|
-
initialCheck: number,
|
|
237
|
-
): AsyncGenerator<W7DecodedRecord> {
|
|
238
|
-
const ownedPaths = new PathOwnership();
|
|
239
|
-
const contentIds = new Set<string>();
|
|
240
|
-
let active: {
|
|
241
|
-
metadata: FileBeginMetadata;
|
|
242
|
-
inode: W7ContentInode;
|
|
243
|
-
nextChunkId: number;
|
|
244
|
-
receivedBytes: number;
|
|
245
|
-
check: number;
|
|
246
|
-
} | null = null;
|
|
247
|
-
let batchCheck = initialCheck;
|
|
248
|
-
const summary: Omit<W7BatchSummary, 'check'> = {
|
|
249
|
-
recordCount: 1,
|
|
250
|
-
pathCount: 0,
|
|
251
|
-
deleteCount: 0,
|
|
252
|
-
directoryCount: 0,
|
|
253
|
-
fileCount: 0,
|
|
254
|
-
chunkCount: 0,
|
|
255
|
-
byteCount: 0,
|
|
256
|
-
};
|
|
257
|
-
let completed = false;
|
|
258
|
-
let failure: unknown = new DOMException('W7 consumer cancelled', 'AbortError');
|
|
259
|
-
try {
|
|
260
|
-
while (true) {
|
|
261
|
-
throwIfAborted(options.signal);
|
|
262
|
-
const envelope = await readEnvelope(buffer, 'record');
|
|
263
|
-
if (active && envelope.tag !== RecordTag.FileChunk && envelope.tag !== RecordTag.FileEnd) {
|
|
264
|
-
throw new Error(`w7-frame: file ${active.inode.path} ended without file-end`);
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
if (envelope.tag === RecordTag.FileChunk) {
|
|
268
|
-
if (!active) throw new Error('w7-frame: file-chunk without active file');
|
|
269
|
-
const prefixLength = Math.min(envelope.length, 4 + MAX_CONTENT_ID_BYTES + 8);
|
|
270
|
-
const idLengthBytes = await buffer.readExact(4, 'file-chunk content-id length');
|
|
271
|
-
const idLength = readU32LE(idLengthBytes, 0);
|
|
272
|
-
if (idLength === 0 || idLength > MAX_CONTENT_ID_BYTES) {
|
|
273
|
-
throw new Error(`w7-frame: invalid file-chunk content-id length ${idLength}`);
|
|
274
|
-
}
|
|
275
|
-
const remainingPrefixLength = idLength + 8;
|
|
276
|
-
if (4 + remainingPrefixLength > prefixLength || 4 + remainingPrefixLength > envelope.length) {
|
|
277
|
-
throw new Error('w7-frame: malformed file-chunk record length');
|
|
278
|
-
}
|
|
279
|
-
const rest = await buffer.readExact(remainingPrefixLength, 'file-chunk header');
|
|
280
|
-
const contentId = decodeText(rest.subarray(0, idLength), 'file-chunk content id');
|
|
281
|
-
const chunkId = readU32LE(rest, idLength);
|
|
282
|
-
const dataLength = readU32LE(rest, idLength + 4);
|
|
283
|
-
if (envelope.length !== 4 + idLength + 8 + dataLength) {
|
|
284
|
-
throw new Error('w7-frame: file-chunk payload length mismatch');
|
|
285
|
-
}
|
|
286
|
-
if (contentId !== active.metadata.contentId) {
|
|
287
|
-
throw new Error(`w7-frame: file-chunk content id ${contentId} does not own ${active.inode.path}`);
|
|
288
|
-
}
|
|
289
|
-
if (chunkId !== active.nextChunkId) {
|
|
290
|
-
throw new Error(
|
|
291
|
-
`w7-frame: ${active.inode.path}: expected chunk ${active.nextChunkId}, got ${chunkId}`,
|
|
292
|
-
);
|
|
293
|
-
}
|
|
294
|
-
if (chunkId >= active.inode.chunkCount) {
|
|
295
|
-
throw new Error(`w7-frame: ${active.inode.path}: chunk ${chunkId} is out of range`);
|
|
296
|
-
}
|
|
297
|
-
const expectedBytes = Math.min(
|
|
298
|
-
CHUNK_SIZE,
|
|
299
|
-
active.inode.size - (chunkId * CHUNK_SIZE),
|
|
300
|
-
);
|
|
301
|
-
if (dataLength !== expectedBytes || dataLength > CHUNK_SIZE) {
|
|
302
|
-
throw new Error(
|
|
303
|
-
`w7-frame: ${active.inode.path}: chunk ${chunkId} has ${dataLength} bytes; expected ${expectedBytes}`,
|
|
304
|
-
);
|
|
305
|
-
}
|
|
306
|
-
const headerPayload = concatBytes(idLengthBytes, rest);
|
|
307
|
-
let retention: W7ChunkRetention | null = null;
|
|
308
|
-
try {
|
|
309
|
-
retention = options.retainChunk
|
|
310
|
-
? await options.retainChunk(dataLength, options.signal)
|
|
311
|
-
: noopRetention(dataLength);
|
|
312
|
-
throwIfAborted(options.signal);
|
|
313
|
-
const data = await buffer.readExact(dataLength, 'file-chunk data');
|
|
314
|
-
batchCheck = updateRecordCheck(batchCheck, envelope.header, headerPayload, data);
|
|
315
|
-
active.check = crc32Update(active.check, data);
|
|
316
|
-
active.nextChunkId++;
|
|
317
|
-
active.receivedBytes += dataLength;
|
|
318
|
-
summary.recordCount++;
|
|
319
|
-
summary.chunkCount++;
|
|
320
|
-
summary.byteCount += dataLength;
|
|
321
|
-
const record: W7DecodedRecord = {
|
|
322
|
-
type: 'file-chunk',
|
|
323
|
-
streamContentId: contentId,
|
|
324
|
-
path: active.inode.path,
|
|
325
|
-
chunkId,
|
|
326
|
-
data,
|
|
327
|
-
retention,
|
|
328
|
-
};
|
|
329
|
-
retention = null;
|
|
330
|
-
yield record;
|
|
331
|
-
} finally {
|
|
332
|
-
retention?.release();
|
|
333
|
-
}
|
|
334
|
-
continue;
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
if (envelope.length > MAX_METADATA_BYTES) {
|
|
338
|
-
throw new Error(`w7-frame: metadata record length ${envelope.length} exceeds ${MAX_METADATA_BYTES}`);
|
|
339
|
-
}
|
|
340
|
-
const payload = await buffer.readExact(envelope.length, 'record payload');
|
|
341
|
-
if (envelope.tag !== RecordTag.BatchEnd) {
|
|
342
|
-
batchCheck = updateRecordCheck(batchCheck, envelope.header, payload);
|
|
343
|
-
summary.recordCount++;
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
switch (envelope.tag) {
|
|
347
|
-
case RecordTag.Delete: {
|
|
348
|
-
const metadata = parseDelete(payload);
|
|
349
|
-
claimPath(ownedPaths, metadata.path);
|
|
350
|
-
summary.pathCount++;
|
|
351
|
-
summary.deleteCount++;
|
|
352
|
-
yield { type: 'delete', path: metadata.path };
|
|
353
|
-
break;
|
|
354
|
-
}
|
|
355
|
-
case RecordTag.Directory: {
|
|
356
|
-
const metadata = parseDirectory(payload);
|
|
357
|
-
claimPath(ownedPaths, metadata.path);
|
|
358
|
-
summary.pathCount++;
|
|
359
|
-
summary.directoryCount++;
|
|
360
|
-
yield { type: 'directory', inode: directoryInode(metadata) };
|
|
361
|
-
break;
|
|
362
|
-
}
|
|
363
|
-
case RecordTag.FileBegin: {
|
|
364
|
-
const metadata = parseFileBegin(payload);
|
|
365
|
-
claimPath(ownedPaths, metadata.path);
|
|
366
|
-
if (contentIds.has(metadata.contentId)) {
|
|
367
|
-
throw new Error(`w7-frame: duplicate stream content id ${metadata.contentId}`);
|
|
368
|
-
}
|
|
369
|
-
contentIds.add(metadata.contentId);
|
|
370
|
-
const inode = fileInode(metadata);
|
|
371
|
-
summary.pathCount++;
|
|
372
|
-
summary.fileCount++;
|
|
373
|
-
active = {
|
|
374
|
-
metadata,
|
|
375
|
-
inode,
|
|
376
|
-
nextChunkId: 0,
|
|
377
|
-
receivedBytes: 0,
|
|
378
|
-
check: CRC_SEED,
|
|
379
|
-
};
|
|
380
|
-
yield { type: 'file-begin', streamContentId: metadata.contentId, inode };
|
|
381
|
-
break;
|
|
382
|
-
}
|
|
383
|
-
case RecordTag.FileEnd: {
|
|
384
|
-
if (!active) throw new Error('w7-frame: file-end without active file');
|
|
385
|
-
const metadata = parseFileEnd(payload);
|
|
386
|
-
const actualCheck = crc32Finish(active.check);
|
|
387
|
-
if (metadata.contentId !== active.metadata.contentId) {
|
|
388
|
-
throw new Error(`w7-frame: file-end content id mismatch for ${active.inode.path}`);
|
|
389
|
-
}
|
|
390
|
-
if (metadata.size !== active.receivedBytes || metadata.size !== active.inode.size) {
|
|
391
|
-
throw new Error(`w7-frame: file-end byte total mismatch for ${active.inode.path}`);
|
|
392
|
-
}
|
|
393
|
-
if (metadata.chunkCount !== active.nextChunkId
|
|
394
|
-
|| metadata.chunkCount !== active.inode.chunkCount) {
|
|
395
|
-
throw new Error(`w7-frame: file-end chunk count mismatch for ${active.inode.path}`);
|
|
396
|
-
}
|
|
397
|
-
if (metadata.check !== actualCheck) {
|
|
398
|
-
throw new Error(`w7-frame: file-end check mismatch for ${active.inode.path}`);
|
|
399
|
-
}
|
|
400
|
-
const record: W7DecodedRecord = {
|
|
401
|
-
type: 'file-end',
|
|
402
|
-
streamContentId: metadata.contentId,
|
|
403
|
-
path: active.inode.path,
|
|
404
|
-
size: metadata.size,
|
|
405
|
-
chunkCount: metadata.chunkCount,
|
|
406
|
-
check: metadata.check,
|
|
407
|
-
};
|
|
408
|
-
active = null;
|
|
409
|
-
yield record;
|
|
410
|
-
break;
|
|
411
|
-
}
|
|
412
|
-
case RecordTag.BatchEnd: {
|
|
413
|
-
if (active) throw new Error(`w7-frame: batch-end while file ${active.inode.path} is active`);
|
|
414
|
-
const actual = parseBatchEnd(payload);
|
|
415
|
-
const expected = { ...summary, check: crc32Finish(batchCheck) };
|
|
416
|
-
if (!sameSummary(actual, expected)) {
|
|
417
|
-
throw new Error(
|
|
418
|
-
`w7-frame: batch-end summary mismatch; expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,
|
|
419
|
-
);
|
|
420
|
-
}
|
|
421
|
-
await buffer.ensureEof(stream);
|
|
422
|
-
completed = true;
|
|
423
|
-
yield { type: 'batch-end', summary: actual };
|
|
424
|
-
return;
|
|
425
|
-
}
|
|
426
|
-
case RecordTag.BatchBegin:
|
|
427
|
-
throw new Error('w7-frame: duplicate batch-begin');
|
|
428
|
-
default:
|
|
429
|
-
throw new Error(`w7-frame: unknown record tag ${envelope.tag}`);
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
} catch (error) {
|
|
433
|
-
failure = error;
|
|
434
|
-
throw error;
|
|
435
|
-
} finally {
|
|
436
|
-
if (completed) {
|
|
437
|
-
try { reader.releaseLock(); } catch { /* already released */ }
|
|
438
|
-
} else {
|
|
439
|
-
await cancelReader(reader, failure);
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
function* encodeRecords(
|
|
445
|
-
batchId: string,
|
|
446
|
-
deletes: string[],
|
|
447
|
-
directories: W7DirectoryInode[],
|
|
448
|
-
files: EncoderFile[],
|
|
449
|
-
): Generator<Uint8Array[]> {
|
|
450
|
-
const state: EncoderState = {
|
|
451
|
-
batchCheck: CRC_SEED,
|
|
452
|
-
summary: {
|
|
453
|
-
recordCount: 0,
|
|
454
|
-
pathCount: 0,
|
|
455
|
-
deleteCount: 0,
|
|
456
|
-
directoryCount: 0,
|
|
457
|
-
fileCount: 0,
|
|
458
|
-
chunkCount: 0,
|
|
459
|
-
byteCount: 0,
|
|
460
|
-
},
|
|
461
|
-
};
|
|
462
|
-
yield encodeMetadataRecord(RecordTag.BatchBegin, { id: batchId, mode: MODE }, state);
|
|
463
|
-
for (const path of deletes) {
|
|
464
|
-
state.summary.pathCount++;
|
|
465
|
-
state.summary.deleteCount++;
|
|
466
|
-
yield encodeMetadataRecord(RecordTag.Delete, { path }, state);
|
|
467
|
-
}
|
|
468
|
-
for (const inode of directories) {
|
|
469
|
-
state.summary.pathCount++;
|
|
470
|
-
state.summary.directoryCount++;
|
|
471
|
-
yield encodeMetadataRecord(RecordTag.Directory, {
|
|
472
|
-
...inodeMetadata(inode),
|
|
473
|
-
kind: inode.kind,
|
|
474
|
-
}, state);
|
|
475
|
-
}
|
|
476
|
-
for (const file of files) {
|
|
477
|
-
state.summary.pathCount++;
|
|
478
|
-
state.summary.fileCount++;
|
|
479
|
-
yield encodeMetadataRecord(RecordTag.FileBegin, {
|
|
480
|
-
...inodeMetadata(file.inode),
|
|
481
|
-
kind: file.inode.kind,
|
|
482
|
-
contentId: file.contentId,
|
|
483
|
-
size: file.inode.size,
|
|
484
|
-
chunkCount: file.inode.chunkCount,
|
|
485
|
-
}, state);
|
|
486
|
-
let fileCheck = CRC_SEED;
|
|
487
|
-
for (const chunk of file.chunks) {
|
|
488
|
-
const data = chunk.data;
|
|
489
|
-
const contentBytes = new TextEncoder().encode(file.contentId);
|
|
490
|
-
const prefix = new Uint8Array(4 + contentBytes.length + 8);
|
|
491
|
-
writeU32LE(prefix, 0, contentBytes.length);
|
|
492
|
-
prefix.set(contentBytes, 4);
|
|
493
|
-
writeU32LE(prefix, 4 + contentBytes.length, chunk.chunkId);
|
|
494
|
-
writeU32LE(prefix, 8 + contentBytes.length, data.byteLength);
|
|
495
|
-
const header = recordHeader(RecordTag.FileChunk, prefix.byteLength + data.byteLength);
|
|
496
|
-
state.batchCheck = updateRecordCheck(state.batchCheck, header, prefix, data);
|
|
497
|
-
state.summary.recordCount++;
|
|
498
|
-
state.summary.chunkCount++;
|
|
499
|
-
state.summary.byteCount += data.byteLength;
|
|
500
|
-
fileCheck = crc32Update(fileCheck, data);
|
|
501
|
-
yield [concatBytes(header, prefix), data];
|
|
502
|
-
}
|
|
503
|
-
yield encodeMetadataRecord(RecordTag.FileEnd, {
|
|
504
|
-
contentId: file.contentId,
|
|
505
|
-
size: file.inode.size,
|
|
506
|
-
chunkCount: file.inode.chunkCount,
|
|
507
|
-
check: crc32Finish(fileCheck),
|
|
508
|
-
}, state);
|
|
509
|
-
}
|
|
510
|
-
const end: W7BatchSummary = {
|
|
511
|
-
...state.summary,
|
|
512
|
-
check: crc32Finish(state.batchCheck),
|
|
513
|
-
};
|
|
514
|
-
yield encodeMetadataRecord(RecordTag.BatchEnd, end);
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
function encodeMetadataRecord(
|
|
518
|
-
tag: RecordTag,
|
|
519
|
-
value: object,
|
|
520
|
-
state?: EncoderState,
|
|
521
|
-
): Uint8Array[] {
|
|
522
|
-
const payload = new TextEncoder().encode(JSON.stringify(value));
|
|
523
|
-
if (payload.byteLength > MAX_METADATA_BYTES) {
|
|
524
|
-
throw new Error(`w7-frame: metadata record exceeds ${MAX_METADATA_BYTES} bytes`);
|
|
525
|
-
}
|
|
526
|
-
const header = recordHeader(tag, payload.byteLength);
|
|
527
|
-
if (state && tag !== RecordTag.BatchEnd) {
|
|
528
|
-
state.batchCheck = updateRecordCheck(state.batchCheck, header, payload);
|
|
529
|
-
state.summary.recordCount++;
|
|
530
|
-
}
|
|
531
|
-
return [concatBytes(header, payload)];
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
function preparePayload(
|
|
535
|
-
payload: BatchWritePayload,
|
|
536
|
-
batchId: string,
|
|
537
|
-
): { deletes: string[]; directories: W7DirectoryInode[]; files: EncoderFile[] } {
|
|
538
|
-
if (!payload || !Array.isArray(payload.inodes) || !Array.isArray(payload.chunks)) {
|
|
539
|
-
throw new Error('w7-frame: payload must contain inode and chunk arrays');
|
|
540
|
-
}
|
|
541
|
-
const ownedPaths = new PathOwnership();
|
|
542
|
-
const deletes = [...(payload.deletePaths ?? [])];
|
|
543
|
-
for (const path of deletes) claimPath(ownedPaths, canonicalPath(path, 'delete path'));
|
|
544
|
-
const chunksByPath = new Map<string, BatchChunkEntry[]>();
|
|
545
|
-
for (const chunk of payload.chunks) {
|
|
546
|
-
const path = canonicalPath(chunk.path, 'chunk path');
|
|
547
|
-
const list = chunksByPath.get(path);
|
|
548
|
-
if (list) list.push(chunk);
|
|
549
|
-
else chunksByPath.set(path, [chunk]);
|
|
550
|
-
}
|
|
551
|
-
const directories: W7DirectoryInode[] = [];
|
|
552
|
-
const files: EncoderFile[] = [];
|
|
553
|
-
let fileIndex = 0;
|
|
554
|
-
for (const inode of payload.inodes) {
|
|
555
|
-
const path = canonicalPath(inode.path, 'inode path');
|
|
556
|
-
if (path !== inode.path) throw new Error(`w7-frame: noncanonical inode path ${inode.path}`);
|
|
557
|
-
claimPath(ownedPaths, path);
|
|
558
|
-
const normalizedInode = normalizeInode(inode);
|
|
559
|
-
const fileChunks = chunksByPath.get(path) ?? [];
|
|
560
|
-
if (normalizedInode.kind === 'directory') {
|
|
561
|
-
if (fileChunks.length > 0) throw new Error(`w7-frame: directory ${path} has chunks`);
|
|
562
|
-
directories.push(normalizedInode);
|
|
563
|
-
} else {
|
|
564
|
-
validateChunks(normalizedInode, fileChunks);
|
|
565
|
-
files.push({
|
|
566
|
-
inode: normalizedInode,
|
|
567
|
-
contentId: `${batchId}:${fileIndex++}`,
|
|
568
|
-
chunks: fileChunks,
|
|
569
|
-
});
|
|
570
|
-
}
|
|
571
|
-
chunksByPath.delete(path);
|
|
572
|
-
}
|
|
573
|
-
if (chunksByPath.size > 0) {
|
|
574
|
-
throw new Error(`w7-frame: chunk has no inode: ${chunksByPath.keys().next().value}`);
|
|
575
|
-
}
|
|
576
|
-
return { deletes, directories, files };
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
function parseBatchBegin(bytes: Uint8Array): BatchBeginMetadata {
|
|
580
|
-
const value = parseObject(bytes, 'batch-begin', ['id', 'mode']);
|
|
581
|
-
const id = boundedString(value.id, 'batch id', MAX_BATCH_ID_BYTES);
|
|
582
|
-
if (value.mode !== MODE) throw new Error(`w7-frame: unsupported batch mode ${String(value.mode)}`);
|
|
583
|
-
return { id, mode: MODE };
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
function parseDelete(bytes: Uint8Array): DeleteMetadata {
|
|
587
|
-
const value = parseObject(bytes, 'delete', ['path']);
|
|
588
|
-
return { path: canonicalPath(value.path, 'delete path') };
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
function parseDirectory(bytes: Uint8Array): DirectoryMetadata {
|
|
592
|
-
const value = parseObject(bytes, 'directory', ['path', 'kind', 'mtime', 'mode'], ['atime']);
|
|
593
|
-
if (value.kind !== 'directory') {
|
|
594
|
-
throw new Error(`w7-frame: unsupported directory kind ${String(value.kind)}`);
|
|
595
|
-
}
|
|
596
|
-
return { ...parseInodeMetadata(value, 'directory'), kind: 'directory' };
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
function parseFileBegin(bytes: Uint8Array): FileBeginMetadata {
|
|
600
|
-
const value = parseObject(
|
|
601
|
-
bytes,
|
|
602
|
-
'file-begin',
|
|
603
|
-
['path', 'kind', 'contentId', 'size', 'chunkCount', 'mtime', 'mode'],
|
|
604
|
-
['atime'],
|
|
605
|
-
);
|
|
606
|
-
const base = parseInodeMetadata(value, 'file-begin');
|
|
607
|
-
if (value.kind !== 'file' && value.kind !== 'symlink') {
|
|
608
|
-
throw new Error(`w7-frame: unsupported file-begin kind ${String(value.kind)}`);
|
|
609
|
-
}
|
|
610
|
-
const contentId = boundedString(value.contentId, 'stream content id', MAX_CONTENT_ID_BYTES);
|
|
611
|
-
if (!/^[A-Za-z0-9._:-]+$/.test(contentId)) {
|
|
612
|
-
throw new Error(`w7-frame: invalid stream content id ${contentId}`);
|
|
613
|
-
}
|
|
614
|
-
const size = safeInteger(value.size, 'file size');
|
|
615
|
-
const chunkCount = u32(value.chunkCount, 'file chunk count');
|
|
616
|
-
const expected = size === 0 ? 0 : Math.ceil(size / CHUNK_SIZE);
|
|
617
|
-
if (chunkCount !== expected) {
|
|
618
|
-
throw new Error(`w7-frame: ${base.path}: expected ${expected} chunks, got ${chunkCount}`);
|
|
619
|
-
}
|
|
620
|
-
return { ...base, kind: value.kind, contentId, size, chunkCount };
|
|
621
|
-
}
|
|
622
|
-
|
|
623
|
-
function parseFileEnd(bytes: Uint8Array): FileEndMetadata {
|
|
624
|
-
const value = parseObject(bytes, 'file-end', ['contentId', 'size', 'chunkCount', 'check']);
|
|
625
|
-
return {
|
|
626
|
-
contentId: boundedString(value.contentId, 'stream content id', MAX_CONTENT_ID_BYTES),
|
|
627
|
-
size: safeInteger(value.size, 'file-end size'),
|
|
628
|
-
chunkCount: u32(value.chunkCount, 'file-end chunk count'),
|
|
629
|
-
check: u32(value.check, 'file-end check'),
|
|
630
|
-
};
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
function parseBatchEnd(bytes: Uint8Array): W7BatchSummary {
|
|
634
|
-
const keys = [
|
|
635
|
-
'recordCount', 'pathCount', 'deleteCount', 'directoryCount',
|
|
636
|
-
'fileCount', 'chunkCount', 'byteCount', 'check',
|
|
637
|
-
];
|
|
638
|
-
const value = parseObject(bytes, 'batch-end', keys);
|
|
639
|
-
return {
|
|
640
|
-
recordCount: safeInteger(value.recordCount, 'batch record count'),
|
|
641
|
-
pathCount: safeInteger(value.pathCount, 'batch path count'),
|
|
642
|
-
deleteCount: safeInteger(value.deleteCount, 'batch delete count'),
|
|
643
|
-
directoryCount: safeInteger(value.directoryCount, 'batch directory count'),
|
|
644
|
-
fileCount: safeInteger(value.fileCount, 'batch file count'),
|
|
645
|
-
chunkCount: safeInteger(value.chunkCount, 'batch chunk count'),
|
|
646
|
-
byteCount: safeInteger(value.byteCount, 'batch byte count'),
|
|
647
|
-
check: u32(value.check, 'batch check'),
|
|
648
|
-
};
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
function parseInodeMetadata(value: Record<string, unknown>, label: string): InodeMetadata {
|
|
652
|
-
return {
|
|
653
|
-
path: canonicalPath(value.path, `${label} path`),
|
|
654
|
-
...(value.atime === undefined ? {} : { atime: safeInteger(value.atime, `${label} atime`) }),
|
|
655
|
-
mtime: safeInteger(value.mtime, `${label} mtime`),
|
|
656
|
-
mode: u32(value.mode, `${label} mode`),
|
|
657
|
-
};
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
function parseObject(
|
|
661
|
-
bytes: Uint8Array,
|
|
662
|
-
label: string,
|
|
663
|
-
required: readonly string[],
|
|
664
|
-
optional: readonly string[] = [],
|
|
665
|
-
): Record<string, unknown> {
|
|
666
|
-
let value: unknown;
|
|
667
|
-
try {
|
|
668
|
-
value = JSON.parse(decodeText(bytes, label));
|
|
669
|
-
} catch (error) {
|
|
670
|
-
if (error instanceof Error && error.message.startsWith('w7-frame:')) throw error;
|
|
671
|
-
throw new Error(`w7-frame: invalid ${label} JSON: ${errorMessage(error)}`);
|
|
672
|
-
}
|
|
673
|
-
if (!isObject(value) || Array.isArray(value)) throw new Error(`w7-frame: ${label} must be an object`);
|
|
674
|
-
const allowed = new Set([...required, ...optional]);
|
|
675
|
-
for (const key of required) {
|
|
676
|
-
if (!Object.hasOwn(value, key)) throw new Error(`w7-frame: ${label} missing ${key}`);
|
|
677
|
-
}
|
|
678
|
-
for (const key of Object.keys(value)) {
|
|
679
|
-
if (!allowed.has(key)) throw new Error(`w7-frame: ${label} has unknown field ${key}`);
|
|
680
|
-
}
|
|
681
|
-
return value;
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
function normalizeInode(inode: BatchInodeEntry): W7DirectoryInode | W7ContentInode {
|
|
685
|
-
canonicalPath(inode.path, 'inode path');
|
|
686
|
-
if (inode.parentPath !== parentPath(inode.path)) {
|
|
687
|
-
throw new Error(`w7-frame: ${inode.path}: noncanonical parent path ${inode.parentPath}`);
|
|
688
|
-
}
|
|
689
|
-
safeInteger(inode.size, `${inode.path} size`);
|
|
690
|
-
u32(inode.chunkCount, `${inode.path} chunk count`);
|
|
691
|
-
safeInteger(inode.mtime, `${inode.path} mtime`);
|
|
692
|
-
if (inode.atime !== undefined) safeInteger(inode.atime, `${inode.path} atime`);
|
|
693
|
-
u32(inode.mode, `${inode.path} mode`);
|
|
694
|
-
const rawKind: unknown = inode.kind ?? (inode.isDir ? 'directory' : 'file');
|
|
695
|
-
if (rawKind !== 'file' && rawKind !== 'directory' && rawKind !== 'symlink') {
|
|
696
|
-
throw new Error(`w7-frame: unsupported inode kind ${String(rawKind)}`);
|
|
697
|
-
}
|
|
698
|
-
const kind: VfsInodeKind = rawKind;
|
|
699
|
-
if (kind === 'directory' && !inode.isDir) {
|
|
700
|
-
throw new Error(`w7-frame: directory inode ${inode.path} must be a directory`);
|
|
701
|
-
}
|
|
702
|
-
if (kind !== 'directory' && inode.isDir) {
|
|
703
|
-
throw new Error(`w7-frame: ${kind} inode ${inode.path} cannot be a directory`);
|
|
704
|
-
}
|
|
705
|
-
if (kind === 'directory' && (inode.size !== 0 || inode.chunkCount !== 0)) {
|
|
706
|
-
throw new Error(`w7-frame: directory ${inode.path} must have zero size and chunks`);
|
|
707
|
-
}
|
|
708
|
-
if (kind !== 'directory') {
|
|
709
|
-
const expected = inode.size === 0 ? 0 : Math.ceil(inode.size / CHUNK_SIZE);
|
|
710
|
-
if (inode.chunkCount !== expected) {
|
|
711
|
-
throw new Error(`w7-frame: ${inode.path}: expected ${expected} chunks, got ${inode.chunkCount}`);
|
|
712
|
-
}
|
|
713
|
-
return { ...inode, kind, isDir: false };
|
|
714
|
-
}
|
|
715
|
-
return { ...inode, kind, isDir: true };
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
function validateChunks(inode: BatchInodeEntry, chunks: BatchChunkEntry[]): void {
|
|
719
|
-
if (chunks.length !== inode.chunkCount) {
|
|
720
|
-
throw new Error(`w7-frame: ${inode.path}: expected ${inode.chunkCount} chunks, got ${chunks.length}`);
|
|
721
|
-
}
|
|
722
|
-
chunks.sort((left, right) => left.chunkId - right.chunkId);
|
|
723
|
-
for (let index = 0; index < chunks.length; index++) {
|
|
724
|
-
const chunk = chunks[index];
|
|
725
|
-
if (chunk.chunkId !== index) {
|
|
726
|
-
throw new Error(`w7-frame: ${inode.path}: expected chunk ${index}, got ${chunk.chunkId}`);
|
|
727
|
-
}
|
|
728
|
-
const expected = Math.min(CHUNK_SIZE, inode.size - (index * CHUNK_SIZE));
|
|
729
|
-
if (!(chunk.data instanceof Uint8Array) || chunk.data.byteLength !== expected) {
|
|
730
|
-
throw new Error(`w7-frame: ${inode.path}: chunk ${index} must contain ${expected} bytes`);
|
|
731
|
-
}
|
|
732
|
-
}
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
function directoryInode(metadata: DirectoryMetadata): W7DirectoryInode {
|
|
736
|
-
return {
|
|
737
|
-
path: metadata.path,
|
|
738
|
-
parentPath: parentPath(metadata.path),
|
|
739
|
-
kind: metadata.kind,
|
|
740
|
-
isDir: true,
|
|
741
|
-
size: 0,
|
|
742
|
-
atime: metadata.atime,
|
|
743
|
-
mtime: metadata.mtime,
|
|
744
|
-
mode: metadata.mode,
|
|
745
|
-
chunkCount: 0,
|
|
746
|
-
};
|
|
747
|
-
}
|
|
748
|
-
|
|
749
|
-
function fileInode(metadata: FileBeginMetadata): W7ContentInode {
|
|
750
|
-
return {
|
|
751
|
-
path: metadata.path,
|
|
752
|
-
parentPath: parentPath(metadata.path),
|
|
753
|
-
kind: metadata.kind,
|
|
754
|
-
isDir: false,
|
|
755
|
-
size: metadata.size,
|
|
756
|
-
atime: metadata.atime,
|
|
757
|
-
mtime: metadata.mtime,
|
|
758
|
-
mode: metadata.mode,
|
|
759
|
-
chunkCount: metadata.chunkCount,
|
|
760
|
-
};
|
|
761
|
-
}
|
|
762
|
-
|
|
763
|
-
function inodeMetadata(inode: BatchInodeEntry): InodeMetadata {
|
|
764
|
-
return {
|
|
765
|
-
path: inode.path,
|
|
766
|
-
...(inode.atime === undefined ? {} : { atime: inode.atime }),
|
|
767
|
-
mtime: inode.mtime,
|
|
768
|
-
mode: inode.mode,
|
|
769
|
-
};
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
function canonicalPath(value: unknown, label: string): string {
|
|
773
|
-
const path = boundedString(value, label, MAX_PATH_BYTES);
|
|
774
|
-
if (path.includes('\0')) throw new Error(`w7-frame: ${label} contains NUL`);
|
|
775
|
-
const normalized = normalizePath(path);
|
|
776
|
-
if (!path || normalized !== path) throw new Error(`w7-frame: noncanonical ${label}: ${path}`);
|
|
777
|
-
return path;
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
class PathOwnership {
|
|
781
|
-
private readonly paths = new Set<string>();
|
|
782
|
-
private pathBytes = 0;
|
|
783
|
-
|
|
784
|
-
claim(path: string): void {
|
|
785
|
-
if (this.paths.has(path)) throw new Error(`w7-frame: duplicate path ownership: ${path}`);
|
|
786
|
-
if (this.paths.size >= W7_MAX_PATHS_PER_BATCH) {
|
|
787
|
-
throw new Error(`w7-frame: batch exceeds ${W7_MAX_PATHS_PER_BATCH} owned paths`);
|
|
788
|
-
}
|
|
789
|
-
const nextPathBytes = this.pathBytes + new TextEncoder().encode(path).byteLength;
|
|
790
|
-
if (nextPathBytes > W7_MAX_OWNED_PATH_BYTES) {
|
|
791
|
-
throw new Error(
|
|
792
|
-
`w7-frame: owned path bytes exceed ${W7_MAX_OWNED_PATH_BYTES}`,
|
|
793
|
-
);
|
|
794
|
-
}
|
|
795
|
-
this.paths.add(path);
|
|
796
|
-
this.pathBytes = nextPathBytes;
|
|
797
|
-
}
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
function claimPath(paths: PathOwnership, path: string): void {
|
|
801
|
-
paths.claim(path);
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
function normalizePath(path: string): string {
|
|
805
|
-
const out: string[] = [];
|
|
806
|
-
for (const segment of path.split('/')) {
|
|
807
|
-
if (segment === '..') {
|
|
808
|
-
if (out.length > 0) out.pop();
|
|
809
|
-
} else if (segment !== '' && segment !== '.') {
|
|
810
|
-
out.push(segment);
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
|
-
return out.join('/');
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
function parentPath(path: string): string {
|
|
817
|
-
const index = path.lastIndexOf('/');
|
|
818
|
-
return index < 0 ? '' : path.slice(0, index);
|
|
819
|
-
}
|
|
820
|
-
|
|
821
|
-
function boundedString(value: unknown, label: string, maxBytes: number): string {
|
|
822
|
-
if (typeof value !== 'string' || value.length === 0) {
|
|
823
|
-
throw new Error(`w7-frame: ${label} must be a non-empty string`);
|
|
824
|
-
}
|
|
825
|
-
const length = new TextEncoder().encode(value).byteLength;
|
|
826
|
-
if (length > maxBytes) throw new Error(`w7-frame: ${label} exceeds ${maxBytes} bytes`);
|
|
827
|
-
return value;
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
function safeInteger(value: unknown, label: string): number {
|
|
831
|
-
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
|
832
|
-
throw new Error(`w7-frame: ${label} must be a non-negative safe integer`);
|
|
833
|
-
}
|
|
834
|
-
return value as number;
|
|
835
|
-
}
|
|
836
|
-
|
|
837
|
-
function u32(value: unknown, label: string): number {
|
|
838
|
-
const integer = safeInteger(value, label);
|
|
839
|
-
if (integer > 0xffff_ffff) throw new Error(`w7-frame: ${label} exceeds uint32`);
|
|
840
|
-
return integer;
|
|
841
|
-
}
|
|
842
|
-
|
|
843
|
-
function sameSummary(left: W7BatchSummary, right: W7BatchSummary): boolean {
|
|
844
|
-
return left.recordCount === right.recordCount
|
|
845
|
-
&& left.pathCount === right.pathCount
|
|
846
|
-
&& left.deleteCount === right.deleteCount
|
|
847
|
-
&& left.directoryCount === right.directoryCount
|
|
848
|
-
&& left.fileCount === right.fileCount
|
|
849
|
-
&& left.chunkCount === right.chunkCount
|
|
850
|
-
&& left.byteCount === right.byteCount
|
|
851
|
-
&& left.check === right.check;
|
|
852
|
-
}
|
|
853
|
-
|
|
854
|
-
async function readEnvelope(buffer: ExactByteReader, label: string): Promise<{
|
|
855
|
-
tag: number;
|
|
856
|
-
length: number;
|
|
857
|
-
header: Uint8Array;
|
|
858
|
-
}> {
|
|
859
|
-
const header = await buffer.readExact(5, `${label} header`);
|
|
860
|
-
return { tag: header[0], length: readU32LE(header, 1), header };
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
function recordHeader(tag: RecordTag, length: number): Uint8Array {
|
|
864
|
-
const header = new Uint8Array(5);
|
|
865
|
-
header[0] = tag;
|
|
866
|
-
writeU32LE(header, 1, length);
|
|
867
|
-
return header;
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
function writeU32LE(out: Uint8Array, offset: number, value: number): void {
|
|
871
|
-
out[offset] = value & 0xff;
|
|
872
|
-
out[offset + 1] = (value >>> 8) & 0xff;
|
|
873
|
-
out[offset + 2] = (value >>> 16) & 0xff;
|
|
874
|
-
out[offset + 3] = (value >>> 24) & 0xff;
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
function readU32LE(bytes: Uint8Array, offset: number): number {
|
|
878
|
-
return (
|
|
879
|
-
bytes[offset]
|
|
880
|
-
| (bytes[offset + 1] << 8)
|
|
881
|
-
| (bytes[offset + 2] << 16)
|
|
882
|
-
| (bytes[offset + 3] << 24)
|
|
883
|
-
) >>> 0;
|
|
884
|
-
}
|
|
885
|
-
|
|
886
|
-
function updateRecordCheck(seed: number, ...parts: Uint8Array[]): number {
|
|
887
|
-
let check = seed;
|
|
888
|
-
for (const part of parts) check = crc32Update(check, part);
|
|
889
|
-
return check;
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
const CRC_SEED = 0xffff_ffff;
|
|
893
|
-
let crcTable: Uint32Array | null = null;
|
|
894
|
-
|
|
895
|
-
function crc32Update(check: number, bytes: Uint8Array): number {
|
|
896
|
-
crcTable ??= createCrcTable();
|
|
897
|
-
const table = crcTable;
|
|
898
|
-
let value = check;
|
|
899
|
-
for (const byte of bytes) value = table[(value ^ byte) & 0xff] ^ (value >>> 8);
|
|
900
|
-
return value >>> 0;
|
|
901
|
-
}
|
|
902
|
-
|
|
903
|
-
function crc32Finish(check: number): number {
|
|
904
|
-
return (check ^ 0xffff_ffff) >>> 0;
|
|
905
|
-
}
|
|
906
|
-
|
|
907
|
-
function createCrcTable(): Uint32Array {
|
|
908
|
-
const table = new Uint32Array(256);
|
|
909
|
-
for (let index = 0; index < table.length; index++) {
|
|
910
|
-
let value = index;
|
|
911
|
-
for (let bit = 0; bit < 8; bit++) {
|
|
912
|
-
value = (value & 1) !== 0 ? 0xedb8_8320 ^ (value >>> 1) : value >>> 1;
|
|
913
|
-
}
|
|
914
|
-
table[index] = value >>> 0;
|
|
915
|
-
}
|
|
916
|
-
return table;
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
function noopRetention(bytes: number): W7ChunkRetention {
|
|
920
|
-
return { bytes, release() {} };
|
|
921
|
-
}
|
|
922
|
-
|
|
923
|
-
function decodeText(bytes: Uint8Array, label: string): string {
|
|
924
|
-
try {
|
|
925
|
-
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
926
|
-
} catch (error) {
|
|
927
|
-
throw new Error(`w7-frame: invalid UTF-8 in ${label}: ${errorMessage(error)}`);
|
|
928
|
-
}
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
function concatBytes(...parts: Uint8Array[]): Uint8Array {
|
|
932
|
-
const output = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));
|
|
933
|
-
let offset = 0;
|
|
934
|
-
for (const part of parts) {
|
|
935
|
-
output.set(part, offset);
|
|
936
|
-
offset += part.byteLength;
|
|
937
|
-
}
|
|
938
|
-
return output;
|
|
939
|
-
}
|
|
940
|
-
|
|
941
|
-
function bytesEqual(left: Uint8Array, right: Uint8Array): boolean {
|
|
942
|
-
return left.length === right.length && left.every((byte, index) => byte === right[index]);
|
|
943
|
-
}
|
|
944
|
-
|
|
945
|
-
function hex(bytes: Uint8Array): string {
|
|
946
|
-
return Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join(' ');
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
function isObject(value: unknown): value is Record<string, unknown> {
|
|
950
|
-
return typeof value === 'object' && value !== null;
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
function errorMessage(error: unknown): string {
|
|
954
|
-
return error instanceof Error ? error.message : String(error);
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
function throwIfAborted(signal?: AbortSignal): void {
|
|
958
|
-
if (!signal?.aborted) return;
|
|
959
|
-
throw new DOMException(
|
|
960
|
-
signal.reason instanceof Error ? signal.reason.message : String(signal.reason ?? 'Aborted'),
|
|
961
|
-
'AbortError',
|
|
962
|
-
);
|
|
963
|
-
}
|
|
964
|
-
|
|
965
|
-
async function cancelReader(
|
|
966
|
-
reader: ReadableStreamBYOBReader,
|
|
967
|
-
reason: unknown,
|
|
968
|
-
): Promise<void> {
|
|
969
|
-
try { await reader.cancel(reason); } catch { /* preserve primary failure */ }
|
|
970
|
-
try { reader.releaseLock(); } catch { /* already released */ }
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
class ExactByteReader {
|
|
974
|
-
private done = false;
|
|
975
|
-
|
|
976
|
-
constructor(private readonly reader: ReadableStreamBYOBReader) {}
|
|
977
|
-
|
|
978
|
-
async readExact(length: number, label: string): Promise<Uint8Array> {
|
|
979
|
-
if (length === 0) return new Uint8Array(0);
|
|
980
|
-
const output = new Uint8Array(length);
|
|
981
|
-
let offset = 0;
|
|
982
|
-
while (offset < length) {
|
|
983
|
-
if (this.done) {
|
|
984
|
-
throw new Error(
|
|
985
|
-
`w7-frame: stream ended ${offset} bytes into expected ${length}-byte ${label}`,
|
|
986
|
-
);
|
|
987
|
-
}
|
|
988
|
-
const next = await this.reader.read(new Uint8Array(length - offset));
|
|
989
|
-
if (next.done) this.done = true;
|
|
990
|
-
else if (next.value.byteLength > 0) {
|
|
991
|
-
output.set(next.value, offset);
|
|
992
|
-
offset += next.value.byteLength;
|
|
993
|
-
}
|
|
994
|
-
}
|
|
995
|
-
return output;
|
|
996
|
-
}
|
|
997
|
-
|
|
998
|
-
async ensureEof(stream: ReadableStream<Uint8Array>): Promise<void> {
|
|
999
|
-
if (this.done) return;
|
|
1000
|
-
this.reader.releaseLock();
|
|
1001
|
-
const reader = stream.getReader();
|
|
1002
|
-
try {
|
|
1003
|
-
const next = await reader.read();
|
|
1004
|
-
if (!next.done && next.value.byteLength > 0) {
|
|
1005
|
-
await reader.cancel(new Error('w7-frame: trailing bytes after batch-end'));
|
|
1006
|
-
throw new Error('w7-frame: trailing bytes after batch-end');
|
|
1007
|
-
}
|
|
1008
|
-
this.done = true;
|
|
1009
|
-
} finally {
|
|
1010
|
-
try { reader.releaseLock(); } catch { /* already released */ }
|
|
1011
|
-
}
|
|
1012
|
-
}
|
|
1013
|
-
}
|