@nimbus-sh/worker 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esbuild-wasm-bundle.generated.d.ts +1 -1
- package/dist/esbuild-wasm-bundle.generated.js +1 -1
- package/dist/facets/cirrus-real.js +1 -1
- package/dist/facets/manager.d.ts +8 -8
- package/dist/facets/manager.d.ts.map +1 -1
- package/dist/facets/manager.js +23 -18
- package/dist/facets/opencode-staging.d.ts.map +1 -1
- package/dist/facets/opencode-staging.js +3 -4
- package/dist/facets/real-vite-hmr.js +1 -1
- package/dist/facets/vite-dev-server.d.ts +4 -4
- package/dist/facets/vite-dev-server.js +5 -5
- package/dist/git/network-facet.d.ts.map +1 -1
- package/dist/git/network-facet.js +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/loaders/child-process/spawn-facet.js +1 -1
- package/dist/loaders/child-process/spawn-pool.d.ts.map +1 -1
- package/dist/loaders/child-process/spawn-pool.js +2 -2
- package/dist/loaders/generated-workers.d.ts +2 -2
- package/dist/loaders/generated-workers.d.ts.map +1 -1
- package/dist/loaders/generated-workers.js +3 -3
- package/dist/loaders/npm-resolve-preamble.d.ts +3 -3
- package/dist/loaders/npm-resolve-preamble.js +3 -3
- package/dist/loaders/pre-bundle-preamble.d.ts +5 -5
- package/dist/loaders/pre-bundle-preamble.js +6 -6
- package/dist/npm/install-batch-facet.d.ts +1 -1
- package/dist/npm/install-batch-facet.js +1 -1
- package/dist/npm/installer.d.ts +6 -6
- package/dist/npm/installer.d.ts.map +1 -1
- package/dist/npm/installer.js +28 -27
- package/dist/npm/pre-bundle-facet.d.ts +2 -2
- package/dist/npm/pre-bundle-facet.js +5 -5
- package/dist/npm/r2-cache.d.ts +1 -1
- package/dist/npm/resolve-facet.d.ts +1 -1
- package/dist/npm/resolve-facet.js +1 -1
- package/dist/npm/resolve-one-facet.d.ts +5 -5
- package/dist/npm/resolve-one-facet.js +5 -5
- package/dist/router/index.js +2 -2
- package/dist/router/remote-api.js +1 -1
- package/dist/runtime/bun-repl.d.ts +1 -1
- package/dist/runtime/bun-repl.js +3 -3
- package/dist/runtime/esbuild-wasm-bytes.js +1 -1
- package/dist/runtime/facet-loader-host.d.ts +2 -2
- package/dist/runtime/facet-loader-host.js +3 -3
- package/dist/runtime/node-repl.js +2 -2
- package/dist/runtime/node-shims-artifact.js +1 -1
- package/dist/runtime/node-shims.js +2 -1
- package/dist/runtime/opencode-artifact.js +1 -1
- package/dist/runtime/opentui-wasm-bytes.js +1 -1
- package/dist/runtime/python-repl.js +3 -3
- package/dist/runtime/ruby-repl.js +2 -2
- package/dist/runtime/sqlite-wasm-bytes.js +1 -1
- package/dist/session/diag.js +1 -1
- package/dist/session/hibernation.d.ts +8 -8
- package/dist/session/hibernation.d.ts.map +1 -1
- package/dist/session/hibernation.js +15 -15
- package/dist/session/init-phases.d.ts +2 -3
- package/dist/session/init-phases.d.ts.map +1 -1
- package/dist/session/init-phases.js +3 -2
- package/dist/session/init.d.ts.map +1 -1
- package/dist/session/init.js +5 -4
- package/dist/session/keys.d.ts.map +1 -1
- package/dist/session/keys.js +4 -3
- package/dist/session/nimbus-session.d.ts +4 -15
- package/dist/session/nimbus-session.d.ts.map +1 -1
- package/dist/session/nimbus-session.js +16 -28
- package/dist/session/programmatic.d.ts +0 -2
- package/dist/session/programmatic.d.ts.map +1 -1
- package/dist/session/programmatic.js +8 -9
- package/dist/session/routes.d.ts.map +1 -1
- package/dist/session/routes.js +19 -18
- package/dist/session/rpc.d.ts +7 -7
- package/dist/session/rpc.d.ts.map +1 -1
- package/dist/session/rpc.js +20 -19
- package/dist/session/start-real-vite.js +1 -1
- package/dist/session/supervisor-rpc.d.ts.map +1 -1
- package/dist/session/supervisor-rpc.js +7 -9
- package/dist/session/ws.d.ts +1 -3
- package/dist/session/ws.d.ts.map +1 -1
- package/dist/session/ws.js +4 -3
- package/package.json +4 -3
- package/scripts/bundle-esbuild-wasm.mjs +2 -2
- package/scripts/bundle-facet-workers.mjs +5 -7
- package/scripts/bundle-sqlite-wasm.mjs +1 -1
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Produced by scripts/bundle-facet-workers.mjs from:
|
|
5
5
|
* - @nimbus-sh/core src/_shared/tarball-stream.ts (streaming tar primitives)
|
|
6
|
-
* - @nimbus-sh/
|
|
6
|
+
* - @nimbus-sh/platform src/w7-frame.ts (W7 streaming bulk-write encoder)
|
|
7
7
|
*
|
|
8
8
|
* Consumed by src/loaders/loader-pool.ts callers via the `preamble`
|
|
9
9
|
* option. The preamble is injected at the top of every generated
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* W7_MAGIC, W7_MAX_RECORD_BYTES.
|
|
17
17
|
*
|
|
18
18
|
* Tar size: 4.15 KiB
|
|
19
|
-
* W7 size:
|
|
19
|
+
* W7 size: 28.97 KiB
|
|
20
20
|
*/
|
|
21
21
|
export const TAR_STREAM_PREAMBLE = "// ../core/src/_shared/tarball-stream.ts\nvar MAX_FILE_BYTES = 2e7;\nfunction canonicalTarName(name) {\n const out = [];\n for (const seg of name.split(\"/\")) {\n if (seg === \"..\") {\n if (out.length === 0) return \"\";\n out.pop();\n } else if (seg !== \"\" && seg !== \".\") {\n out.push(seg);\n }\n }\n return out.join(\"/\");\n}\nfunction parseTarHeader(block) {\n if (block[0] === 0) return null;\n let name = \"\";\n for (let i = 0; i < 100 && block[i] !== 0; i++) {\n name += String.fromCharCode(block[i]);\n }\n let prefix = \"\";\n for (let i = 345; i < 500 && block[i] !== 0; i++) {\n prefix += String.fromCharCode(block[i]);\n }\n if (prefix) name = prefix + \"/\" + name;\n name = name.replace(/^package\\//, \"\");\n name = canonicalTarName(name);\n let sizeStr = \"\";\n for (let i = 124; i < 136 && block[i] !== 0; i++) {\n sizeStr += String.fromCharCode(block[i]);\n }\n const size = parseInt(sizeStr.trim(), 8) || 0;\n const typeFlag = block[156];\n return { name, size, typeFlag };\n}\nasync function* readableStreamToAsyncIterable(rs) {\n const reader = rs.getReader();\n try {\n while (true) {\n const { value, done } = await reader.read();\n if (done) return;\n if (value && value.length > 0) yield value;\n }\n } finally {\n try {\n reader.releaseLock();\n } catch {\n }\n }\n}\nasync function* streamTarEntries(source, onSkip) {\n let carry = new Uint8Array(0);\n let state = { kind: \"header\" };\n function concat(a, b) {\n if (a.length === 0) return b;\n if (b.length === 0) return a;\n const out = new Uint8Array(a.length + b.length);\n out.set(a, 0);\n out.set(b, a.length);\n return out;\n }\n for await (const chunkRaw of source) {\n let buf = concat(carry, chunkRaw);\n let cursor = 0;\n while (true) {\n if (state.kind === \"header\") {\n if (buf.length - cursor < 512) break;\n const header = buf.subarray(cursor, cursor + 512);\n const parsed = parseTarHeader(header);\n cursor += 512;\n if (!parsed) return;\n const { name, size, typeFlag } = parsed;\n const pad = size === 0 ? 0 : (512 - size % 512) % 512;\n const isRegularFile = typeFlag === 48 || typeFlag === 0;\n if (size === 0) {\n if (isRegularFile && name) {\n yield { name, data: new Uint8Array(0) };\n }\n state = { kind: \"header\" };\n continue;\n }\n if (!isRegularFile || !name || size > MAX_FILE_BYTES) {\n if (onSkip) {\n const reason = !isRegularFile ? \"non-regular\" : !name ? \"no-name\" : \"too-large\";\n try {\n onSkip(name, size, reason);\n } catch {\n }\n }\n state = { kind: \"skip\", remaining: size + pad };\n continue;\n }\n state = {\n kind: \"file\",\n name,\n remaining: size,\n fileBuf: new Uint8Array(size),\n fileOffset: 0,\n pad,\n skip: false\n };\n continue;\n }\n if (state.kind === \"file\") {\n const avail2 = buf.length - cursor;\n if (avail2 === 0) break;\n if (state.remaining > 0) {\n const take2 = Math.min(state.remaining, avail2);\n state.fileBuf.set(buf.subarray(cursor, cursor + take2), state.fileOffset);\n state.fileOffset += take2;\n state.remaining -= take2;\n cursor += take2;\n if (state.remaining > 0) break;\n }\n if (state.pad > 0) {\n const avail22 = buf.length - cursor;\n if (avail22 === 0) break;\n const take2 = Math.min(state.pad, avail22);\n state.pad -= take2;\n cursor += take2;\n if (state.pad > 0) break;\n }\n yield { name: state.name, data: state.fileBuf };\n state = { kind: \"header\" };\n continue;\n }\n const avail = buf.length - cursor;\n if (avail === 0) break;\n const take = Math.min(state.remaining, avail);\n cursor += take;\n state.remaining -= take;\n if (state.remaining > 0) break;\n state = { kind: \"header\" };\n }\n if (cursor >= buf.length) {\n carry = new Uint8Array(0);\n } else if (cursor === 0) {\n carry = buf;\n } else {\n carry = buf.slice(cursor);\n }\n }\n}";
|
|
22
|
-
export const W7_FRAME_PREAMBLE = "// ../core/src/constants.ts\nvar CHUNK_SIZE = 65536;\nvar VFS_CAPACITY = 10 * 1024 * 1024 * 1024;\nvar MAX_TX_BLOB_BYTES = 1 * 1024 * 1024;\nvar MAX_GLOBAL_WRITE_STREAM_CREDIT_BYTES = 8 * 1024 * 1024;\nvar MAX_RPC_SAFE_PAYLOAD_BYTES = 28 * 1024 * 1024;\nvar FS_READ_BATCH_REQUEST_BYTES = 4 * 1024 * 1024;\nvar VFS_BUNDLE_MAX_BYTES = 24 * 1024 * 1024;\nvar BUNDLE_MAX_ENCODED_BYTES = 22 * 1024 * 1024;\nvar PREFETCH_CACHE_MAX_BYTES = 16 * 1024 * 1024;\nvar CWD_SNAPSHOT_MAX_FILE_BYTES = 2 * 1024 * 1024;\nvar SUPERVISOR_HEAP_CEILING_BYTES = 64 * 1024 * 1024;\nvar SUPERVISOR_IN_FLIGHT_ALLOCATION_BUDGET_BYTES = 40 * 1024 * 1024;\nvar SUPERVISOR_READ_RESERVE_BYTES = 1024 * 1024;\n\n// ../core/src/_shared/w7-frame.ts\nvar W7_MAGIC = new Uint8Array([78, 87, 55, 3]);\nvar ENCODER_QUEUE_HWM = 0;\nvar MAX_METADATA_BYTES = 64 * 1024;\nvar MAX_PATH_BYTES = 64 * 1024;\nvar MAX_BATCH_ID_BYTES = 128;\nvar MAX_CONTENT_ID_BYTES = 256;\nvar W7_MAX_PATHS_PER_BATCH = 128;\nvar W7_MAX_OWNED_PATH_BYTES = 64 * 1024;\nvar W7_MAX_RECORD_BYTES = 5 + 4 + MAX_CONTENT_ID_BYTES + 8 + CHUNK_SIZE;\nvar MODE = \"path-atomic-committed-prefix\";\nfunction encodeWriteBatchStream(payload) {\n const batchId = crypto.randomUUID();\n const { deletes, directories, files } = preparePayload(payload, batchId);\n const iterator = encodeRecords(batchId, deletes, directories, files);\n let closed = false;\n let magicEmitted = false;\n const source = {\n type: \"bytes\",\n pull(controller) {\n if (closed) return;\n try {\n if (!magicEmitted) {\n magicEmitted = true;\n controller.enqueue(W7_MAGIC.slice());\n return;\n }\n const next = iterator.next();\n if (next.done) {\n closed = true;\n controller.close();\n return;\n }\n for (const part of next.value) controller.enqueue(part);\n } catch (error) {\n closed = true;\n controller.error(error);\n }\n },\n cancel() {\n closed = true;\n iterator.return?.(void 0);\n }\n };\n return new ReadableStream(source, {\n highWaterMark: ENCODER_QUEUE_HWM\n });\n}\nasync function decodeWriteBatchStream(stream, options = {}) {\n let reader;\n try {\n reader = stream.getReader({ mode: \"byob\" });\n } catch {\n throw new Error(\"w7-frame: stream must be a byte-oriented ReadableStream\");\n }\n const buffer = new ExactByteReader(reader);\n let handedOff = false;\n try {\n throwIfAborted(options.signal);\n const magic = await buffer.readExact(W7_MAGIC.length, \"magic\");\n if (!bytesEqual(magic, W7_MAGIC)) {\n const version = magic.length === 4 && magic[0] === 78 && magic[1] === 87 && magic[2] === 55 ? magic[3] : null;\n if (version !== null) {\n throw new Error(`w7-frame: unsupported protocol version ${version}; expected 3`);\n }\n throw new Error(`w7-frame: bad magic, expected NW7\\\\x03, got ${hex(magic)}`);\n }\n const beginEnvelope = await readEnvelope(buffer, \"batch-begin\");\n if (beginEnvelope.tag !== 1 /* BatchBegin */) {\n throw new Error(`w7-frame: first record must be batch-begin, got tag ${beginEnvelope.tag}`);\n }\n if (beginEnvelope.length > MAX_METADATA_BYTES) {\n throw new Error(\n `w7-frame: batch-begin length ${beginEnvelope.length} exceeds ${MAX_METADATA_BYTES}`\n );\n }\n const beginPayload = await buffer.readExact(beginEnvelope.length, \"batch-begin payload\");\n const begin = parseBatchBegin(beginPayload);\n const initialCheck = updateRecordCheck(CRC_SEED, beginEnvelope.header, beginPayload);\n handedOff = true;\n return {\n batchId: begin.id,\n mode: begin.mode,\n records: decodeRecords(stream, reader, buffer, options, initialCheck)\n };\n } catch (error) {\n if (!handedOff) await cancelReader(reader, error);\n throw error;\n }\n}\nasync function* decodeRecords(stream, reader, buffer, options, initialCheck) {\n const ownedPaths = new PathOwnership();\n const contentIds = /* @__PURE__ */ new Set();\n let active = null;\n let batchCheck = initialCheck;\n const summary = {\n recordCount: 1,\n pathCount: 0,\n deleteCount: 0,\n directoryCount: 0,\n fileCount: 0,\n chunkCount: 0,\n byteCount: 0\n };\n let completed = false;\n let failure = new DOMException(\"W7 consumer cancelled\", \"AbortError\");\n try {\n while (true) {\n throwIfAborted(options.signal);\n const envelope = await readEnvelope(buffer, \"record\");\n if (active && envelope.tag !== 5 /* FileChunk */ && envelope.tag !== 6 /* FileEnd */) {\n throw new Error(`w7-frame: file ${active.inode.path} ended without file-end`);\n }\n if (envelope.tag === 5 /* FileChunk */) {\n if (!active) throw new Error(\"w7-frame: file-chunk without active file\");\n const prefixLength = Math.min(envelope.length, 4 + MAX_CONTENT_ID_BYTES + 8);\n const idLengthBytes = await buffer.readExact(4, \"file-chunk content-id length\");\n const idLength = readU32LE(idLengthBytes, 0);\n if (idLength === 0 || idLength > MAX_CONTENT_ID_BYTES) {\n throw new Error(`w7-frame: invalid file-chunk content-id length ${idLength}`);\n }\n const remainingPrefixLength = idLength + 8;\n if (4 + remainingPrefixLength > prefixLength || 4 + remainingPrefixLength > envelope.length) {\n throw new Error(\"w7-frame: malformed file-chunk record length\");\n }\n const rest = await buffer.readExact(remainingPrefixLength, \"file-chunk header\");\n const contentId = decodeText(rest.subarray(0, idLength), \"file-chunk content id\");\n const chunkId = readU32LE(rest, idLength);\n const dataLength = readU32LE(rest, idLength + 4);\n if (envelope.length !== 4 + idLength + 8 + dataLength) {\n throw new Error(\"w7-frame: file-chunk payload length mismatch\");\n }\n if (contentId !== active.metadata.contentId) {\n throw new Error(`w7-frame: file-chunk content id ${contentId} does not own ${active.inode.path}`);\n }\n if (chunkId !== active.nextChunkId) {\n throw new Error(\n `w7-frame: ${active.inode.path}: expected chunk ${active.nextChunkId}, got ${chunkId}`\n );\n }\n if (chunkId >= active.inode.chunkCount) {\n throw new Error(`w7-frame: ${active.inode.path}: chunk ${chunkId} is out of range`);\n }\n const expectedBytes = Math.min(\n CHUNK_SIZE,\n active.inode.size - chunkId * CHUNK_SIZE\n );\n if (dataLength !== expectedBytes || dataLength > CHUNK_SIZE) {\n throw new Error(\n `w7-frame: ${active.inode.path}: chunk ${chunkId} has ${dataLength} bytes; expected ${expectedBytes}`\n );\n }\n const headerPayload = concatBytes(idLengthBytes, rest);\n let retention = null;\n try {\n retention = options.retainChunk ? await options.retainChunk(dataLength, options.signal) : noopRetention(dataLength);\n throwIfAborted(options.signal);\n const data = await buffer.readExact(dataLength, \"file-chunk data\");\n batchCheck = updateRecordCheck(batchCheck, envelope.header, headerPayload, data);\n active.check = crc32Update(active.check, data);\n active.nextChunkId++;\n active.receivedBytes += dataLength;\n summary.recordCount++;\n summary.chunkCount++;\n summary.byteCount += dataLength;\n const record = {\n type: \"file-chunk\",\n streamContentId: contentId,\n path: active.inode.path,\n chunkId,\n data,\n retention\n };\n retention = null;\n yield record;\n } finally {\n retention?.release();\n }\n continue;\n }\n if (envelope.length > MAX_METADATA_BYTES) {\n throw new Error(`w7-frame: metadata record length ${envelope.length} exceeds ${MAX_METADATA_BYTES}`);\n }\n const payload = await buffer.readExact(envelope.length, \"record payload\");\n if (envelope.tag !== 7 /* BatchEnd */) {\n batchCheck = updateRecordCheck(batchCheck, envelope.header, payload);\n summary.recordCount++;\n }\n switch (envelope.tag) {\n case 2 /* Delete */: {\n const metadata = parseDelete(payload);\n claimPath(ownedPaths, metadata.path);\n summary.pathCount++;\n summary.deleteCount++;\n yield { type: \"delete\", path: metadata.path };\n break;\n }\n case 3 /* Directory */: {\n const metadata = parseDirectory(payload);\n claimPath(ownedPaths, metadata.path);\n summary.pathCount++;\n summary.directoryCount++;\n yield { type: \"directory\", inode: directoryInode(metadata) };\n break;\n }\n case 4 /* FileBegin */: {\n const metadata = parseFileBegin(payload);\n claimPath(ownedPaths, metadata.path);\n if (contentIds.has(metadata.contentId)) {\n throw new Error(`w7-frame: duplicate stream content id ${metadata.contentId}`);\n }\n contentIds.add(metadata.contentId);\n const inode = fileInode(metadata);\n summary.pathCount++;\n summary.fileCount++;\n active = {\n metadata,\n inode,\n nextChunkId: 0,\n receivedBytes: 0,\n check: CRC_SEED\n };\n yield { type: \"file-begin\", streamContentId: metadata.contentId, inode };\n break;\n }\n case 6 /* FileEnd */: {\n if (!active) throw new Error(\"w7-frame: file-end without active file\");\n const metadata = parseFileEnd(payload);\n const actualCheck = crc32Finish(active.check);\n if (metadata.contentId !== active.metadata.contentId) {\n throw new Error(`w7-frame: file-end content id mismatch for ${active.inode.path}`);\n }\n if (metadata.size !== active.receivedBytes || metadata.size !== active.inode.size) {\n throw new Error(`w7-frame: file-end byte total mismatch for ${active.inode.path}`);\n }\n if (metadata.chunkCount !== active.nextChunkId || metadata.chunkCount !== active.inode.chunkCount) {\n throw new Error(`w7-frame: file-end chunk count mismatch for ${active.inode.path}`);\n }\n if (metadata.check !== actualCheck) {\n throw new Error(`w7-frame: file-end check mismatch for ${active.inode.path}`);\n }\n const record = {\n type: \"file-end\",\n streamContentId: metadata.contentId,\n path: active.inode.path,\n size: metadata.size,\n chunkCount: metadata.chunkCount,\n check: metadata.check\n };\n active = null;\n yield record;\n break;\n }\n case 7 /* BatchEnd */: {\n if (active) throw new Error(`w7-frame: batch-end while file ${active.inode.path} is active`);\n const actual = parseBatchEnd(payload);\n const expected = { ...summary, check: crc32Finish(batchCheck) };\n if (!sameSummary(actual, expected)) {\n throw new Error(\n `w7-frame: batch-end summary mismatch; expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`\n );\n }\n await buffer.ensureEof(stream);\n completed = true;\n yield { type: \"batch-end\", summary: actual };\n return;\n }\n case 1 /* BatchBegin */:\n throw new Error(\"w7-frame: duplicate batch-begin\");\n default:\n throw new Error(`w7-frame: unknown record tag ${envelope.tag}`);\n }\n }\n } catch (error) {\n failure = error;\n throw error;\n } finally {\n if (completed) {\n try {\n reader.releaseLock();\n } catch {\n }\n } else {\n await cancelReader(reader, failure);\n }\n }\n}\nfunction* encodeRecords(batchId, deletes, directories, files) {\n const state = {\n batchCheck: CRC_SEED,\n summary: {\n recordCount: 0,\n pathCount: 0,\n deleteCount: 0,\n directoryCount: 0,\n fileCount: 0,\n chunkCount: 0,\n byteCount: 0\n }\n };\n yield encodeMetadataRecord(1 /* BatchBegin */, { id: batchId, mode: MODE }, state);\n for (const path of deletes) {\n state.summary.pathCount++;\n state.summary.deleteCount++;\n yield encodeMetadataRecord(2 /* Delete */, { path }, state);\n }\n for (const inode of directories) {\n state.summary.pathCount++;\n state.summary.directoryCount++;\n yield encodeMetadataRecord(3 /* Directory */, {\n ...inodeMetadata(inode),\n kind: inode.kind\n }, state);\n }\n for (const file of files) {\n state.summary.pathCount++;\n state.summary.fileCount++;\n yield encodeMetadataRecord(4 /* FileBegin */, {\n ...inodeMetadata(file.inode),\n kind: file.inode.kind,\n contentId: file.contentId,\n size: file.inode.size,\n chunkCount: file.inode.chunkCount\n }, state);\n let fileCheck = CRC_SEED;\n for (const chunk of file.chunks) {\n const data = chunk.data;\n const contentBytes = new TextEncoder().encode(file.contentId);\n const prefix = new Uint8Array(4 + contentBytes.length + 8);\n writeU32LE(prefix, 0, contentBytes.length);\n prefix.set(contentBytes, 4);\n writeU32LE(prefix, 4 + contentBytes.length, chunk.chunkId);\n writeU32LE(prefix, 8 + contentBytes.length, data.byteLength);\n const header = recordHeader(5 /* FileChunk */, prefix.byteLength + data.byteLength);\n state.batchCheck = updateRecordCheck(state.batchCheck, header, prefix, data);\n state.summary.recordCount++;\n state.summary.chunkCount++;\n state.summary.byteCount += data.byteLength;\n fileCheck = crc32Update(fileCheck, data);\n yield [concatBytes(header, prefix), data];\n }\n yield encodeMetadataRecord(6 /* FileEnd */, {\n contentId: file.contentId,\n size: file.inode.size,\n chunkCount: file.inode.chunkCount,\n check: crc32Finish(fileCheck)\n }, state);\n }\n const end = {\n ...state.summary,\n check: crc32Finish(state.batchCheck)\n };\n yield encodeMetadataRecord(7 /* BatchEnd */, end);\n}\nfunction encodeMetadataRecord(tag, value, state) {\n const payload = new TextEncoder().encode(JSON.stringify(value));\n if (payload.byteLength > MAX_METADATA_BYTES) {\n throw new Error(`w7-frame: metadata record exceeds ${MAX_METADATA_BYTES} bytes`);\n }\n const header = recordHeader(tag, payload.byteLength);\n if (state && tag !== 7 /* BatchEnd */) {\n state.batchCheck = updateRecordCheck(state.batchCheck, header, payload);\n state.summary.recordCount++;\n }\n return [concatBytes(header, payload)];\n}\nfunction preparePayload(payload, batchId) {\n if (!payload || !Array.isArray(payload.inodes) || !Array.isArray(payload.chunks)) {\n throw new Error(\"w7-frame: payload must contain inode and chunk arrays\");\n }\n const ownedPaths = new PathOwnership();\n const deletes = [...payload.deletePaths ?? []];\n for (const path of deletes) claimPath(ownedPaths, canonicalPath(path, \"delete path\"));\n const chunksByPath = /* @__PURE__ */ new Map();\n for (const chunk of payload.chunks) {\n const path = canonicalPath(chunk.path, \"chunk path\");\n const list = chunksByPath.get(path);\n if (list) list.push(chunk);\n else chunksByPath.set(path, [chunk]);\n }\n const directories = [];\n const files = [];\n let fileIndex = 0;\n for (const inode of payload.inodes) {\n const path = canonicalPath(inode.path, \"inode path\");\n if (path !== inode.path) throw new Error(`w7-frame: noncanonical inode path ${inode.path}`);\n claimPath(ownedPaths, path);\n const normalizedInode = normalizeInode(inode);\n const fileChunks = chunksByPath.get(path) ?? [];\n if (normalizedInode.kind === \"directory\") {\n if (fileChunks.length > 0) throw new Error(`w7-frame: directory ${path} has chunks`);\n directories.push(normalizedInode);\n } else {\n validateChunks(normalizedInode, fileChunks);\n files.push({\n inode: normalizedInode,\n contentId: `${batchId}:${fileIndex++}`,\n chunks: fileChunks\n });\n }\n chunksByPath.delete(path);\n }\n if (chunksByPath.size > 0) {\n throw new Error(`w7-frame: chunk has no inode: ${chunksByPath.keys().next().value}`);\n }\n return { deletes, directories, files };\n}\nfunction parseBatchBegin(bytes) {\n const value = parseObject(bytes, \"batch-begin\", [\"id\", \"mode\"]);\n const id = boundedString(value.id, \"batch id\", MAX_BATCH_ID_BYTES);\n if (value.mode !== MODE) throw new Error(`w7-frame: unsupported batch mode ${String(value.mode)}`);\n return { id, mode: MODE };\n}\nfunction parseDelete(bytes) {\n const value = parseObject(bytes, \"delete\", [\"path\"]);\n return { path: canonicalPath(value.path, \"delete path\") };\n}\nfunction parseDirectory(bytes) {\n const value = parseObject(bytes, \"directory\", [\"path\", \"kind\", \"mtime\", \"mode\"], [\"atime\"]);\n if (value.kind !== \"directory\") {\n throw new Error(`w7-frame: unsupported directory kind ${String(value.kind)}`);\n }\n return { ...parseInodeMetadata(value, \"directory\"), kind: \"directory\" };\n}\nfunction parseFileBegin(bytes) {\n const value = parseObject(\n bytes,\n \"file-begin\",\n [\"path\", \"kind\", \"contentId\", \"size\", \"chunkCount\", \"mtime\", \"mode\"],\n [\"atime\"]\n );\n const base = parseInodeMetadata(value, \"file-begin\");\n if (value.kind !== \"file\" && value.kind !== \"symlink\") {\n throw new Error(`w7-frame: unsupported file-begin kind ${String(value.kind)}`);\n }\n const contentId = boundedString(value.contentId, \"stream content id\", MAX_CONTENT_ID_BYTES);\n if (!/^[A-Za-z0-9._:-]+$/.test(contentId)) {\n throw new Error(`w7-frame: invalid stream content id ${contentId}`);\n }\n const size = safeInteger(value.size, \"file size\");\n const chunkCount = u32(value.chunkCount, \"file chunk count\");\n const expected = size === 0 ? 0 : Math.ceil(size / CHUNK_SIZE);\n if (chunkCount !== expected) {\n throw new Error(`w7-frame: ${base.path}: expected ${expected} chunks, got ${chunkCount}`);\n }\n return { ...base, kind: value.kind, contentId, size, chunkCount };\n}\nfunction parseFileEnd(bytes) {\n const value = parseObject(bytes, \"file-end\", [\"contentId\", \"size\", \"chunkCount\", \"check\"]);\n return {\n contentId: boundedString(value.contentId, \"stream content id\", MAX_CONTENT_ID_BYTES),\n size: safeInteger(value.size, \"file-end size\"),\n chunkCount: u32(value.chunkCount, \"file-end chunk count\"),\n check: u32(value.check, \"file-end check\")\n };\n}\nfunction parseBatchEnd(bytes) {\n const keys = [\n \"recordCount\",\n \"pathCount\",\n \"deleteCount\",\n \"directoryCount\",\n \"fileCount\",\n \"chunkCount\",\n \"byteCount\",\n \"check\"\n ];\n const value = parseObject(bytes, \"batch-end\", keys);\n return {\n recordCount: safeInteger(value.recordCount, \"batch record count\"),\n pathCount: safeInteger(value.pathCount, \"batch path count\"),\n deleteCount: safeInteger(value.deleteCount, \"batch delete count\"),\n directoryCount: safeInteger(value.directoryCount, \"batch directory count\"),\n fileCount: safeInteger(value.fileCount, \"batch file count\"),\n chunkCount: safeInteger(value.chunkCount, \"batch chunk count\"),\n byteCount: safeInteger(value.byteCount, \"batch byte count\"),\n check: u32(value.check, \"batch check\")\n };\n}\nfunction parseInodeMetadata(value, label) {\n return {\n path: canonicalPath(value.path, `${label} path`),\n ...value.atime === void 0 ? {} : { atime: safeInteger(value.atime, `${label} atime`) },\n mtime: safeInteger(value.mtime, `${label} mtime`),\n mode: u32(value.mode, `${label} mode`)\n };\n}\nfunction parseObject(bytes, label, required, optional = []) {\n let value;\n try {\n value = JSON.parse(decodeText(bytes, label));\n } catch (error) {\n if (error instanceof Error && error.message.startsWith(\"w7-frame:\")) throw error;\n throw new Error(`w7-frame: invalid ${label} JSON: ${errorMessage(error)}`);\n }\n if (!isObject(value) || Array.isArray(value)) throw new Error(`w7-frame: ${label} must be an object`);\n const allowed = /* @__PURE__ */ new Set([...required, ...optional]);\n for (const key of required) {\n if (!Object.hasOwn(value, key)) throw new Error(`w7-frame: ${label} missing ${key}`);\n }\n for (const key of Object.keys(value)) {\n if (!allowed.has(key)) throw new Error(`w7-frame: ${label} has unknown field ${key}`);\n }\n return value;\n}\nfunction normalizeInode(inode) {\n canonicalPath(inode.path, \"inode path\");\n if (inode.parentPath !== parentPath(inode.path)) {\n throw new Error(`w7-frame: ${inode.path}: noncanonical parent path ${inode.parentPath}`);\n }\n safeInteger(inode.size, `${inode.path} size`);\n u32(inode.chunkCount, `${inode.path} chunk count`);\n safeInteger(inode.mtime, `${inode.path} mtime`);\n if (inode.atime !== void 0) safeInteger(inode.atime, `${inode.path} atime`);\n u32(inode.mode, `${inode.path} mode`);\n const rawKind = inode.kind ?? (inode.isDir ? \"directory\" : \"file\");\n if (rawKind !== \"file\" && rawKind !== \"directory\" && rawKind !== \"symlink\") {\n throw new Error(`w7-frame: unsupported inode kind ${String(rawKind)}`);\n }\n const kind = rawKind;\n if (kind === \"directory\" && !inode.isDir) {\n throw new Error(`w7-frame: directory inode ${inode.path} must be a directory`);\n }\n if (kind !== \"directory\" && inode.isDir) {\n throw new Error(`w7-frame: ${kind} inode ${inode.path} cannot be a directory`);\n }\n if (kind === \"directory\" && (inode.size !== 0 || inode.chunkCount !== 0)) {\n throw new Error(`w7-frame: directory ${inode.path} must have zero size and chunks`);\n }\n if (kind !== \"directory\") {\n const expected = inode.size === 0 ? 0 : Math.ceil(inode.size / CHUNK_SIZE);\n if (inode.chunkCount !== expected) {\n throw new Error(`w7-frame: ${inode.path}: expected ${expected} chunks, got ${inode.chunkCount}`);\n }\n return { ...inode, kind, isDir: false };\n }\n return { ...inode, kind, isDir: true };\n}\nfunction validateChunks(inode, chunks) {\n if (chunks.length !== inode.chunkCount) {\n throw new Error(`w7-frame: ${inode.path}: expected ${inode.chunkCount} chunks, got ${chunks.length}`);\n }\n chunks.sort((left, right) => left.chunkId - right.chunkId);\n for (let index = 0; index < chunks.length; index++) {\n const chunk = chunks[index];\n if (chunk.chunkId !== index) {\n throw new Error(`w7-frame: ${inode.path}: expected chunk ${index}, got ${chunk.chunkId}`);\n }\n const expected = Math.min(CHUNK_SIZE, inode.size - index * CHUNK_SIZE);\n if (!(chunk.data instanceof Uint8Array) || chunk.data.byteLength !== expected) {\n throw new Error(`w7-frame: ${inode.path}: chunk ${index} must contain ${expected} bytes`);\n }\n }\n}\nfunction directoryInode(metadata) {\n return {\n path: metadata.path,\n parentPath: parentPath(metadata.path),\n kind: metadata.kind,\n isDir: true,\n size: 0,\n atime: metadata.atime,\n mtime: metadata.mtime,\n mode: metadata.mode,\n chunkCount: 0\n };\n}\nfunction fileInode(metadata) {\n return {\n path: metadata.path,\n parentPath: parentPath(metadata.path),\n kind: metadata.kind,\n isDir: false,\n size: metadata.size,\n atime: metadata.atime,\n mtime: metadata.mtime,\n mode: metadata.mode,\n chunkCount: metadata.chunkCount\n };\n}\nfunction inodeMetadata(inode) {\n return {\n path: inode.path,\n ...inode.atime === void 0 ? {} : { atime: inode.atime },\n mtime: inode.mtime,\n mode: inode.mode\n };\n}\nfunction canonicalPath(value, label) {\n const path = boundedString(value, label, MAX_PATH_BYTES);\n if (path.includes(\"\\0\")) throw new Error(`w7-frame: ${label} contains NUL`);\n const normalized = normalizePath(path);\n if (!path || normalized !== path) throw new Error(`w7-frame: noncanonical ${label}: ${path}`);\n return path;\n}\nvar PathOwnership = class {\n paths = /* @__PURE__ */ new Set();\n pathBytes = 0;\n claim(path) {\n if (this.paths.has(path)) throw new Error(`w7-frame: duplicate path ownership: ${path}`);\n if (this.paths.size >= W7_MAX_PATHS_PER_BATCH) {\n throw new Error(`w7-frame: batch exceeds ${W7_MAX_PATHS_PER_BATCH} owned paths`);\n }\n const nextPathBytes = this.pathBytes + new TextEncoder().encode(path).byteLength;\n if (nextPathBytes > W7_MAX_OWNED_PATH_BYTES) {\n throw new Error(\n `w7-frame: owned path bytes exceed ${W7_MAX_OWNED_PATH_BYTES}`\n );\n }\n this.paths.add(path);\n this.pathBytes = nextPathBytes;\n }\n};\nfunction claimPath(paths, path) {\n paths.claim(path);\n}\nfunction normalizePath(path) {\n const out = [];\n for (const segment of path.split(\"/\")) {\n if (segment === \"..\") {\n if (out.length > 0) out.pop();\n } else if (segment !== \"\" && segment !== \".\") {\n out.push(segment);\n }\n }\n return out.join(\"/\");\n}\nfunction parentPath(path) {\n const index = path.lastIndexOf(\"/\");\n return index < 0 ? \"\" : path.slice(0, index);\n}\nfunction boundedString(value, label, maxBytes) {\n if (typeof value !== \"string\" || value.length === 0) {\n throw new Error(`w7-frame: ${label} must be a non-empty string`);\n }\n const length = new TextEncoder().encode(value).byteLength;\n if (length > maxBytes) throw new Error(`w7-frame: ${label} exceeds ${maxBytes} bytes`);\n return value;\n}\nfunction safeInteger(value, label) {\n if (!Number.isSafeInteger(value) || value < 0) {\n throw new Error(`w7-frame: ${label} must be a non-negative safe integer`);\n }\n return value;\n}\nfunction u32(value, label) {\n const integer = safeInteger(value, label);\n if (integer > 4294967295) throw new Error(`w7-frame: ${label} exceeds uint32`);\n return integer;\n}\nfunction sameSummary(left, right) {\n return left.recordCount === right.recordCount && left.pathCount === right.pathCount && left.deleteCount === right.deleteCount && left.directoryCount === right.directoryCount && left.fileCount === right.fileCount && left.chunkCount === right.chunkCount && left.byteCount === right.byteCount && left.check === right.check;\n}\nasync function readEnvelope(buffer, label) {\n const header = await buffer.readExact(5, `${label} header`);\n return { tag: header[0], length: readU32LE(header, 1), header };\n}\nfunction recordHeader(tag, length) {\n const header = new Uint8Array(5);\n header[0] = tag;\n writeU32LE(header, 1, length);\n return header;\n}\nfunction writeU32LE(out, offset, value) {\n out[offset] = value & 255;\n out[offset + 1] = value >>> 8 & 255;\n out[offset + 2] = value >>> 16 & 255;\n out[offset + 3] = value >>> 24 & 255;\n}\nfunction readU32LE(bytes, offset) {\n return (bytes[offset] | bytes[offset + 1] << 8 | bytes[offset + 2] << 16 | bytes[offset + 3] << 24) >>> 0;\n}\nfunction updateRecordCheck(seed, ...parts) {\n let check = seed;\n for (const part of parts) check = crc32Update(check, part);\n return check;\n}\nvar CRC_SEED = 4294967295;\nvar crcTable = null;\nfunction crc32Update(check, bytes) {\n crcTable ??= createCrcTable();\n const table = crcTable;\n let value = check;\n for (const byte of bytes) value = table[(value ^ byte) & 255] ^ value >>> 8;\n return value >>> 0;\n}\nfunction crc32Finish(check) {\n return (check ^ 4294967295) >>> 0;\n}\nfunction createCrcTable() {\n const table = new Uint32Array(256);\n for (let index = 0; index < table.length; index++) {\n let value = index;\n for (let bit = 0; bit < 8; bit++) {\n value = (value & 1) !== 0 ? 3988292384 ^ value >>> 1 : value >>> 1;\n }\n table[index] = value >>> 0;\n }\n return table;\n}\nfunction noopRetention(bytes) {\n return { bytes, release() {\n } };\n}\nfunction decodeText(bytes, label) {\n try {\n return new TextDecoder(\"utf-8\", { fatal: true }).decode(bytes);\n } catch (error) {\n throw new Error(`w7-frame: invalid UTF-8 in ${label}: ${errorMessage(error)}`);\n }\n}\nfunction concatBytes(...parts) {\n const output = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));\n let offset = 0;\n for (const part of parts) {\n output.set(part, offset);\n offset += part.byteLength;\n }\n return output;\n}\nfunction bytesEqual(left, right) {\n return left.length === right.length && left.every((byte, index) => byte === right[index]);\n}\nfunction hex(bytes) {\n return Array.from(bytes, (byte) => byte.toString(16).padStart(2, \"0\")).join(\" \");\n}\nfunction isObject(value) {\n return typeof value === \"object\" && value !== null;\n}\nfunction errorMessage(error) {\n return error instanceof Error ? error.message : String(error);\n}\nfunction throwIfAborted(signal) {\n if (!signal?.aborted) return;\n throw new DOMException(\n signal.reason instanceof Error ? signal.reason.message : String(signal.reason ?? \"Aborted\"),\n \"AbortError\"\n );\n}\nasync function cancelReader(reader, reason) {\n try {\n await reader.cancel(reason);\n } catch {\n }\n try {\n reader.releaseLock();\n } catch {\n }\n}\nvar ExactByteReader = class {\n constructor(reader) {\n this.reader = reader;\n }\n reader;\n done = false;\n async readExact(length, label) {\n if (length === 0) return new Uint8Array(0);\n const output = new Uint8Array(length);\n let offset = 0;\n while (offset < length) {\n if (this.done) {\n throw new Error(\n `w7-frame: stream ended ${offset} bytes into expected ${length}-byte ${label}`\n );\n }\n const next = await this.reader.read(new Uint8Array(length - offset));\n if (next.done) this.done = true;\n else if (next.value.byteLength > 0) {\n output.set(next.value, offset);\n offset += next.value.byteLength;\n }\n }\n return output;\n }\n async ensureEof(stream) {\n if (this.done) return;\n this.reader.releaseLock();\n const reader = stream.getReader();\n try {\n const next = await reader.read();\n if (!next.done && next.value.byteLength > 0) {\n await reader.cancel(new Error(\"w7-frame: trailing bytes after batch-end\"));\n throw new Error(\"w7-frame: trailing bytes after batch-end\");\n }\n this.done = true;\n } finally {\n try {\n reader.releaseLock();\n } catch {\n }\n }\n }\n};";
|
|
22
|
+
export const W7_FRAME_PREAMBLE = "// ../platform/src/limits.ts\nvar CHUNK_SIZE = 65536;\nvar MAX_TX_BLOB_BYTES = 1 * 1024 * 1024;\nvar SQLITE_MAX_STATEMENT_BYTES = 100 * 1024;\nvar MAX_RPC_SAFE_PAYLOAD_BYTES = 28 * 1024 * 1024;\nvar SUPERVISOR_HEAP_CEILING_BYTES = 64 * 1024 * 1024;\nvar SUPERVISOR_IN_FLIGHT_ALLOCATION_BUDGET_BYTES = 40 * 1024 * 1024;\nvar SUPERVISOR_READ_RESERVE_BYTES = 1024 * 1024;\nvar MAX_GLOBAL_WRITE_STREAM_CREDIT_BYTES = 8 * 1024 * 1024;\n\n// ../platform/src/w7-frame.ts\nvar W7_MAGIC = new Uint8Array([78, 87, 55, 3]);\nvar ENCODER_QUEUE_HWM = 0;\nvar MAX_METADATA_BYTES = 64 * 1024;\nvar MAX_PATH_BYTES = 64 * 1024;\nvar MAX_BATCH_ID_BYTES = 128;\nvar MAX_CONTENT_ID_BYTES = 256;\nvar W7_MAX_PATHS_PER_BATCH = 128;\nvar W7_MAX_OWNED_PATH_BYTES = 64 * 1024;\nvar W7_MAX_RECORD_BYTES = 5 + 4 + MAX_CONTENT_ID_BYTES + 8 + CHUNK_SIZE;\nvar MODE = \"path-atomic-committed-prefix\";\nfunction encodeWriteBatchStream(payload) {\n const batchId = crypto.randomUUID();\n const { deletes, directories, files } = preparePayload(payload, batchId);\n const iterator = encodeRecords(batchId, deletes, directories, files);\n let closed = false;\n let magicEmitted = false;\n const source = {\n type: \"bytes\",\n pull(controller) {\n if (closed) return;\n try {\n if (!magicEmitted) {\n magicEmitted = true;\n controller.enqueue(W7_MAGIC.slice());\n return;\n }\n const next = iterator.next();\n if (next.done) {\n closed = true;\n controller.close();\n return;\n }\n for (const part of next.value) controller.enqueue(part);\n } catch (error) {\n closed = true;\n controller.error(error);\n }\n },\n cancel() {\n closed = true;\n iterator.return?.(void 0);\n }\n };\n return new ReadableStream(source, {\n highWaterMark: ENCODER_QUEUE_HWM\n });\n}\nasync function decodeWriteBatchStream(stream, options = {}) {\n let reader;\n try {\n reader = stream.getReader({ mode: \"byob\" });\n } catch {\n throw new Error(\"w7-frame: stream must be a byte-oriented ReadableStream\");\n }\n const buffer = new ExactByteReader(reader);\n let handedOff = false;\n try {\n throwIfAborted(options.signal);\n const magic = await buffer.readExact(W7_MAGIC.length, \"magic\");\n if (!bytesEqual(magic, W7_MAGIC)) {\n const version = magic.length === 4 && magic[0] === 78 && magic[1] === 87 && magic[2] === 55 ? magic[3] : null;\n if (version !== null) {\n throw new Error(`w7-frame: unsupported protocol version ${version}; expected 3`);\n }\n throw new Error(`w7-frame: bad magic, expected NW7\\\\x03, got ${hex(magic)}`);\n }\n const beginEnvelope = await readEnvelope(buffer, \"batch-begin\");\n if (beginEnvelope.tag !== 1 /* BatchBegin */) {\n throw new Error(`w7-frame: first record must be batch-begin, got tag ${beginEnvelope.tag}`);\n }\n if (beginEnvelope.length > MAX_METADATA_BYTES) {\n throw new Error(\n `w7-frame: batch-begin length ${beginEnvelope.length} exceeds ${MAX_METADATA_BYTES}`\n );\n }\n const beginPayload = await buffer.readExact(beginEnvelope.length, \"batch-begin payload\");\n const begin = parseBatchBegin(beginPayload);\n const initialCheck = updateRecordCheck(CRC_SEED, beginEnvelope.header, beginPayload);\n handedOff = true;\n return {\n batchId: begin.id,\n mode: begin.mode,\n records: decodeRecords(stream, reader, buffer, options, initialCheck)\n };\n } catch (error) {\n if (!handedOff) await cancelReader(reader, error);\n throw error;\n }\n}\nasync function* decodeRecords(stream, reader, buffer, options, initialCheck) {\n const ownedPaths = new PathOwnership();\n const contentIds = /* @__PURE__ */ new Set();\n let active = null;\n let batchCheck = initialCheck;\n const summary = {\n recordCount: 1,\n pathCount: 0,\n deleteCount: 0,\n directoryCount: 0,\n fileCount: 0,\n chunkCount: 0,\n byteCount: 0\n };\n let completed = false;\n let failure = new DOMException(\"W7 consumer cancelled\", \"AbortError\");\n try {\n while (true) {\n throwIfAborted(options.signal);\n const envelope = await readEnvelope(buffer, \"record\");\n if (active && envelope.tag !== 5 /* FileChunk */ && envelope.tag !== 6 /* FileEnd */) {\n throw new Error(`w7-frame: file ${active.inode.path} ended without file-end`);\n }\n if (envelope.tag === 5 /* FileChunk */) {\n if (!active) throw new Error(\"w7-frame: file-chunk without active file\");\n const prefixLength = Math.min(envelope.length, 4 + MAX_CONTENT_ID_BYTES + 8);\n const idLengthBytes = await buffer.readExact(4, \"file-chunk content-id length\");\n const idLength = readU32LE(idLengthBytes, 0);\n if (idLength === 0 || idLength > MAX_CONTENT_ID_BYTES) {\n throw new Error(`w7-frame: invalid file-chunk content-id length ${idLength}`);\n }\n const remainingPrefixLength = idLength + 8;\n if (4 + remainingPrefixLength > prefixLength || 4 + remainingPrefixLength > envelope.length) {\n throw new Error(\"w7-frame: malformed file-chunk record length\");\n }\n const rest = await buffer.readExact(remainingPrefixLength, \"file-chunk header\");\n const contentId = decodeText(rest.subarray(0, idLength), \"file-chunk content id\");\n const chunkId = readU32LE(rest, idLength);\n const dataLength = readU32LE(rest, idLength + 4);\n if (envelope.length !== 4 + idLength + 8 + dataLength) {\n throw new Error(\"w7-frame: file-chunk payload length mismatch\");\n }\n if (contentId !== active.metadata.contentId) {\n throw new Error(`w7-frame: file-chunk content id ${contentId} does not own ${active.inode.path}`);\n }\n if (chunkId !== active.nextChunkId) {\n throw new Error(\n `w7-frame: ${active.inode.path}: expected chunk ${active.nextChunkId}, got ${chunkId}`\n );\n }\n if (chunkId >= active.inode.chunkCount) {\n throw new Error(`w7-frame: ${active.inode.path}: chunk ${chunkId} is out of range`);\n }\n const expectedBytes = Math.min(\n CHUNK_SIZE,\n active.inode.size - chunkId * CHUNK_SIZE\n );\n if (dataLength !== expectedBytes || dataLength > CHUNK_SIZE) {\n throw new Error(\n `w7-frame: ${active.inode.path}: chunk ${chunkId} has ${dataLength} bytes; expected ${expectedBytes}`\n );\n }\n const headerPayload = concatBytes(idLengthBytes, rest);\n let retention = null;\n try {\n retention = options.retainChunk ? await options.retainChunk(dataLength, options.signal) : noopRetention(dataLength);\n throwIfAborted(options.signal);\n const data = await buffer.readExact(dataLength, \"file-chunk data\");\n batchCheck = updateRecordCheck(batchCheck, envelope.header, headerPayload, data);\n active.check = crc32Update(active.check, data);\n active.nextChunkId++;\n active.receivedBytes += dataLength;\n summary.recordCount++;\n summary.chunkCount++;\n summary.byteCount += dataLength;\n const record = {\n type: \"file-chunk\",\n streamContentId: contentId,\n path: active.inode.path,\n chunkId,\n data,\n retention\n };\n retention = null;\n yield record;\n } finally {\n retention?.release();\n }\n continue;\n }\n if (envelope.length > MAX_METADATA_BYTES) {\n throw new Error(`w7-frame: metadata record length ${envelope.length} exceeds ${MAX_METADATA_BYTES}`);\n }\n const payload = await buffer.readExact(envelope.length, \"record payload\");\n if (envelope.tag !== 7 /* BatchEnd */) {\n batchCheck = updateRecordCheck(batchCheck, envelope.header, payload);\n summary.recordCount++;\n }\n switch (envelope.tag) {\n case 2 /* Delete */: {\n const metadata = parseDelete(payload);\n claimPath(ownedPaths, metadata.path);\n summary.pathCount++;\n summary.deleteCount++;\n yield { type: \"delete\", path: metadata.path };\n break;\n }\n case 3 /* Directory */: {\n const metadata = parseDirectory(payload);\n claimPath(ownedPaths, metadata.path);\n summary.pathCount++;\n summary.directoryCount++;\n yield { type: \"directory\", inode: directoryInode(metadata) };\n break;\n }\n case 4 /* FileBegin */: {\n const metadata = parseFileBegin(payload);\n claimPath(ownedPaths, metadata.path);\n if (contentIds.has(metadata.contentId)) {\n throw new Error(`w7-frame: duplicate stream content id ${metadata.contentId}`);\n }\n contentIds.add(metadata.contentId);\n const inode = fileInode(metadata);\n summary.pathCount++;\n summary.fileCount++;\n active = {\n metadata,\n inode,\n nextChunkId: 0,\n receivedBytes: 0,\n check: CRC_SEED\n };\n yield { type: \"file-begin\", streamContentId: metadata.contentId, inode };\n break;\n }\n case 6 /* FileEnd */: {\n if (!active) throw new Error(\"w7-frame: file-end without active file\");\n const metadata = parseFileEnd(payload);\n const actualCheck = crc32Finish(active.check);\n if (metadata.contentId !== active.metadata.contentId) {\n throw new Error(`w7-frame: file-end content id mismatch for ${active.inode.path}`);\n }\n if (metadata.size !== active.receivedBytes || metadata.size !== active.inode.size) {\n throw new Error(`w7-frame: file-end byte total mismatch for ${active.inode.path}`);\n }\n if (metadata.chunkCount !== active.nextChunkId || metadata.chunkCount !== active.inode.chunkCount) {\n throw new Error(`w7-frame: file-end chunk count mismatch for ${active.inode.path}`);\n }\n if (metadata.check !== actualCheck) {\n throw new Error(`w7-frame: file-end check mismatch for ${active.inode.path}`);\n }\n const record = {\n type: \"file-end\",\n streamContentId: metadata.contentId,\n path: active.inode.path,\n size: metadata.size,\n chunkCount: metadata.chunkCount,\n check: metadata.check\n };\n active = null;\n yield record;\n break;\n }\n case 7 /* BatchEnd */: {\n if (active) throw new Error(`w7-frame: batch-end while file ${active.inode.path} is active`);\n const actual = parseBatchEnd(payload);\n const expected = { ...summary, check: crc32Finish(batchCheck) };\n if (!sameSummary(actual, expected)) {\n throw new Error(\n `w7-frame: batch-end summary mismatch; expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`\n );\n }\n await buffer.ensureEof(stream);\n completed = true;\n yield { type: \"batch-end\", summary: actual };\n return;\n }\n case 1 /* BatchBegin */:\n throw new Error(\"w7-frame: duplicate batch-begin\");\n default:\n throw new Error(`w7-frame: unknown record tag ${envelope.tag}`);\n }\n }\n } catch (error) {\n failure = error;\n throw error;\n } finally {\n if (completed) {\n try {\n reader.releaseLock();\n } catch {\n }\n } else {\n await cancelReader(reader, failure);\n }\n }\n}\nfunction* encodeRecords(batchId, deletes, directories, files) {\n const state = {\n batchCheck: CRC_SEED,\n summary: {\n recordCount: 0,\n pathCount: 0,\n deleteCount: 0,\n directoryCount: 0,\n fileCount: 0,\n chunkCount: 0,\n byteCount: 0\n }\n };\n yield encodeMetadataRecord(1 /* BatchBegin */, { id: batchId, mode: MODE }, state);\n for (const path of deletes) {\n state.summary.pathCount++;\n state.summary.deleteCount++;\n yield encodeMetadataRecord(2 /* Delete */, { path }, state);\n }\n for (const inode of directories) {\n state.summary.pathCount++;\n state.summary.directoryCount++;\n yield encodeMetadataRecord(3 /* Directory */, {\n ...inodeMetadata(inode),\n kind: inode.kind\n }, state);\n }\n for (const file of files) {\n state.summary.pathCount++;\n state.summary.fileCount++;\n yield encodeMetadataRecord(4 /* FileBegin */, {\n ...inodeMetadata(file.inode),\n kind: file.inode.kind,\n contentId: file.contentId,\n size: file.inode.size,\n chunkCount: file.inode.chunkCount\n }, state);\n let fileCheck = CRC_SEED;\n for (const chunk of file.chunks) {\n const data = chunk.data;\n const contentBytes = new TextEncoder().encode(file.contentId);\n const prefix = new Uint8Array(4 + contentBytes.length + 8);\n writeU32LE(prefix, 0, contentBytes.length);\n prefix.set(contentBytes, 4);\n writeU32LE(prefix, 4 + contentBytes.length, chunk.chunkId);\n writeU32LE(prefix, 8 + contentBytes.length, data.byteLength);\n const header = recordHeader(5 /* FileChunk */, prefix.byteLength + data.byteLength);\n state.batchCheck = updateRecordCheck(state.batchCheck, header, prefix, data);\n state.summary.recordCount++;\n state.summary.chunkCount++;\n state.summary.byteCount += data.byteLength;\n fileCheck = crc32Update(fileCheck, data);\n yield [concatBytes(header, prefix), data];\n }\n yield encodeMetadataRecord(6 /* FileEnd */, {\n contentId: file.contentId,\n size: file.inode.size,\n chunkCount: file.inode.chunkCount,\n check: crc32Finish(fileCheck)\n }, state);\n }\n const end = {\n ...state.summary,\n check: crc32Finish(state.batchCheck)\n };\n yield encodeMetadataRecord(7 /* BatchEnd */, end);\n}\nfunction encodeMetadataRecord(tag, value, state) {\n const payload = new TextEncoder().encode(JSON.stringify(value));\n if (payload.byteLength > MAX_METADATA_BYTES) {\n throw new Error(`w7-frame: metadata record exceeds ${MAX_METADATA_BYTES} bytes`);\n }\n const header = recordHeader(tag, payload.byteLength);\n if (state && tag !== 7 /* BatchEnd */) {\n state.batchCheck = updateRecordCheck(state.batchCheck, header, payload);\n state.summary.recordCount++;\n }\n return [concatBytes(header, payload)];\n}\nfunction preparePayload(payload, batchId) {\n if (!payload || !Array.isArray(payload.inodes) || !Array.isArray(payload.chunks)) {\n throw new Error(\"w7-frame: payload must contain inode and chunk arrays\");\n }\n const ownedPaths = new PathOwnership();\n const deletes = [...payload.deletePaths ?? []];\n for (const path of deletes) claimPath(ownedPaths, canonicalPath(path, \"delete path\"));\n const chunksByPath = /* @__PURE__ */ new Map();\n for (const chunk of payload.chunks) {\n const path = canonicalPath(chunk.path, \"chunk path\");\n const list = chunksByPath.get(path);\n if (list) list.push(chunk);\n else chunksByPath.set(path, [chunk]);\n }\n const directories = [];\n const files = [];\n let fileIndex = 0;\n for (const inode of payload.inodes) {\n const path = canonicalPath(inode.path, \"inode path\");\n if (path !== inode.path) throw new Error(`w7-frame: noncanonical inode path ${inode.path}`);\n claimPath(ownedPaths, path);\n const normalizedInode = normalizeInode(inode);\n const fileChunks = chunksByPath.get(path) ?? [];\n if (normalizedInode.kind === \"directory\") {\n if (fileChunks.length > 0) throw new Error(`w7-frame: directory ${path} has chunks`);\n directories.push(normalizedInode);\n } else {\n validateChunks(normalizedInode, fileChunks);\n files.push({\n inode: normalizedInode,\n contentId: `${batchId}:${fileIndex++}`,\n chunks: fileChunks\n });\n }\n chunksByPath.delete(path);\n }\n if (chunksByPath.size > 0) {\n throw new Error(`w7-frame: chunk has no inode: ${chunksByPath.keys().next().value}`);\n }\n return { deletes, directories, files };\n}\nfunction parseBatchBegin(bytes) {\n const value = parseObject(bytes, \"batch-begin\", [\"id\", \"mode\"]);\n const id = boundedString(value.id, \"batch id\", MAX_BATCH_ID_BYTES);\n if (value.mode !== MODE) throw new Error(`w7-frame: unsupported batch mode ${String(value.mode)}`);\n return { id, mode: MODE };\n}\nfunction parseDelete(bytes) {\n const value = parseObject(bytes, \"delete\", [\"path\"]);\n return { path: canonicalPath(value.path, \"delete path\") };\n}\nfunction parseDirectory(bytes) {\n const value = parseObject(bytes, \"directory\", [\"path\", \"kind\", \"mtime\", \"mode\"], [\"atime\"]);\n if (value.kind !== \"directory\") {\n throw new Error(`w7-frame: unsupported directory kind ${String(value.kind)}`);\n }\n return { ...parseInodeMetadata(value, \"directory\"), kind: \"directory\" };\n}\nfunction parseFileBegin(bytes) {\n const value = parseObject(\n bytes,\n \"file-begin\",\n [\"path\", \"kind\", \"contentId\", \"size\", \"chunkCount\", \"mtime\", \"mode\"],\n [\"atime\"]\n );\n const base = parseInodeMetadata(value, \"file-begin\");\n if (value.kind !== \"file\" && value.kind !== \"symlink\") {\n throw new Error(`w7-frame: unsupported file-begin kind ${String(value.kind)}`);\n }\n const contentId = boundedString(value.contentId, \"stream content id\", MAX_CONTENT_ID_BYTES);\n if (!/^[A-Za-z0-9._:-]+$/.test(contentId)) {\n throw new Error(`w7-frame: invalid stream content id ${contentId}`);\n }\n const size = safeInteger(value.size, \"file size\");\n const chunkCount = u32(value.chunkCount, \"file chunk count\");\n const expected = size === 0 ? 0 : Math.ceil(size / CHUNK_SIZE);\n if (chunkCount !== expected) {\n throw new Error(`w7-frame: ${base.path}: expected ${expected} chunks, got ${chunkCount}`);\n }\n return { ...base, kind: value.kind, contentId, size, chunkCount };\n}\nfunction parseFileEnd(bytes) {\n const value = parseObject(bytes, \"file-end\", [\"contentId\", \"size\", \"chunkCount\", \"check\"]);\n return {\n contentId: boundedString(value.contentId, \"stream content id\", MAX_CONTENT_ID_BYTES),\n size: safeInteger(value.size, \"file-end size\"),\n chunkCount: u32(value.chunkCount, \"file-end chunk count\"),\n check: u32(value.check, \"file-end check\")\n };\n}\nfunction parseBatchEnd(bytes) {\n const keys = [\n \"recordCount\",\n \"pathCount\",\n \"deleteCount\",\n \"directoryCount\",\n \"fileCount\",\n \"chunkCount\",\n \"byteCount\",\n \"check\"\n ];\n const value = parseObject(bytes, \"batch-end\", keys);\n return {\n recordCount: safeInteger(value.recordCount, \"batch record count\"),\n pathCount: safeInteger(value.pathCount, \"batch path count\"),\n deleteCount: safeInteger(value.deleteCount, \"batch delete count\"),\n directoryCount: safeInteger(value.directoryCount, \"batch directory count\"),\n fileCount: safeInteger(value.fileCount, \"batch file count\"),\n chunkCount: safeInteger(value.chunkCount, \"batch chunk count\"),\n byteCount: safeInteger(value.byteCount, \"batch byte count\"),\n check: u32(value.check, \"batch check\")\n };\n}\nfunction parseInodeMetadata(value, label) {\n return {\n path: canonicalPath(value.path, `${label} path`),\n ...value.atime === void 0 ? {} : { atime: safeInteger(value.atime, `${label} atime`) },\n mtime: safeInteger(value.mtime, `${label} mtime`),\n mode: u32(value.mode, `${label} mode`)\n };\n}\nfunction parseObject(bytes, label, required, optional = []) {\n let value;\n try {\n value = JSON.parse(decodeText(bytes, label));\n } catch (error) {\n if (error instanceof Error && error.message.startsWith(\"w7-frame:\")) throw error;\n throw new Error(`w7-frame: invalid ${label} JSON: ${errorMessage(error)}`);\n }\n if (!isObject(value) || Array.isArray(value)) throw new Error(`w7-frame: ${label} must be an object`);\n const allowed = /* @__PURE__ */ new Set([...required, ...optional]);\n for (const key of required) {\n if (!Object.hasOwn(value, key)) throw new Error(`w7-frame: ${label} missing ${key}`);\n }\n for (const key of Object.keys(value)) {\n if (!allowed.has(key)) throw new Error(`w7-frame: ${label} has unknown field ${key}`);\n }\n return value;\n}\nfunction normalizeInode(inode) {\n canonicalPath(inode.path, \"inode path\");\n if (inode.parentPath !== parentPath(inode.path)) {\n throw new Error(`w7-frame: ${inode.path}: noncanonical parent path ${inode.parentPath}`);\n }\n safeInteger(inode.size, `${inode.path} size`);\n u32(inode.chunkCount, `${inode.path} chunk count`);\n safeInteger(inode.mtime, `${inode.path} mtime`);\n if (inode.atime !== void 0) safeInteger(inode.atime, `${inode.path} atime`);\n u32(inode.mode, `${inode.path} mode`);\n const rawKind = inode.kind ?? (inode.isDir ? \"directory\" : \"file\");\n if (rawKind !== \"file\" && rawKind !== \"directory\" && rawKind !== \"symlink\") {\n throw new Error(`w7-frame: unsupported inode kind ${String(rawKind)}`);\n }\n const kind = rawKind;\n if (kind === \"directory\" && !inode.isDir) {\n throw new Error(`w7-frame: directory inode ${inode.path} must be a directory`);\n }\n if (kind !== \"directory\" && inode.isDir) {\n throw new Error(`w7-frame: ${kind} inode ${inode.path} cannot be a directory`);\n }\n if (kind === \"directory\" && (inode.size !== 0 || inode.chunkCount !== 0)) {\n throw new Error(`w7-frame: directory ${inode.path} must have zero size and chunks`);\n }\n if (kind !== \"directory\") {\n const expected = inode.size === 0 ? 0 : Math.ceil(inode.size / CHUNK_SIZE);\n if (inode.chunkCount !== expected) {\n throw new Error(`w7-frame: ${inode.path}: expected ${expected} chunks, got ${inode.chunkCount}`);\n }\n return { ...inode, kind, isDir: false };\n }\n return { ...inode, kind, isDir: true };\n}\nfunction validateChunks(inode, chunks) {\n if (chunks.length !== inode.chunkCount) {\n throw new Error(`w7-frame: ${inode.path}: expected ${inode.chunkCount} chunks, got ${chunks.length}`);\n }\n chunks.sort((left, right) => left.chunkId - right.chunkId);\n for (let index = 0; index < chunks.length; index++) {\n const chunk = chunks[index];\n if (chunk.chunkId !== index) {\n throw new Error(`w7-frame: ${inode.path}: expected chunk ${index}, got ${chunk.chunkId}`);\n }\n const expected = Math.min(CHUNK_SIZE, inode.size - index * CHUNK_SIZE);\n if (!(chunk.data instanceof Uint8Array) || chunk.data.byteLength !== expected) {\n throw new Error(`w7-frame: ${inode.path}: chunk ${index} must contain ${expected} bytes`);\n }\n }\n}\nfunction directoryInode(metadata) {\n return {\n path: metadata.path,\n parentPath: parentPath(metadata.path),\n kind: metadata.kind,\n isDir: true,\n size: 0,\n atime: metadata.atime,\n mtime: metadata.mtime,\n mode: metadata.mode,\n chunkCount: 0\n };\n}\nfunction fileInode(metadata) {\n return {\n path: metadata.path,\n parentPath: parentPath(metadata.path),\n kind: metadata.kind,\n isDir: false,\n size: metadata.size,\n atime: metadata.atime,\n mtime: metadata.mtime,\n mode: metadata.mode,\n chunkCount: metadata.chunkCount\n };\n}\nfunction inodeMetadata(inode) {\n return {\n path: inode.path,\n ...inode.atime === void 0 ? {} : { atime: inode.atime },\n mtime: inode.mtime,\n mode: inode.mode\n };\n}\nfunction canonicalPath(value, label) {\n const path = boundedString(value, label, MAX_PATH_BYTES);\n if (path.includes(\"\\0\")) throw new Error(`w7-frame: ${label} contains NUL`);\n const normalized = normalizePath(path);\n if (!path || normalized !== path) throw new Error(`w7-frame: noncanonical ${label}: ${path}`);\n return path;\n}\nvar PathOwnership = class {\n paths = /* @__PURE__ */ new Set();\n pathBytes = 0;\n claim(path) {\n if (this.paths.has(path)) throw new Error(`w7-frame: duplicate path ownership: ${path}`);\n if (this.paths.size >= W7_MAX_PATHS_PER_BATCH) {\n throw new Error(`w7-frame: batch exceeds ${W7_MAX_PATHS_PER_BATCH} owned paths`);\n }\n const nextPathBytes = this.pathBytes + new TextEncoder().encode(path).byteLength;\n if (nextPathBytes > W7_MAX_OWNED_PATH_BYTES) {\n throw new Error(\n `w7-frame: owned path bytes exceed ${W7_MAX_OWNED_PATH_BYTES}`\n );\n }\n this.paths.add(path);\n this.pathBytes = nextPathBytes;\n }\n};\nfunction claimPath(paths, path) {\n paths.claim(path);\n}\nfunction normalizePath(path) {\n const out = [];\n for (const segment of path.split(\"/\")) {\n if (segment === \"..\") {\n if (out.length > 0) out.pop();\n } else if (segment !== \"\" && segment !== \".\") {\n out.push(segment);\n }\n }\n return out.join(\"/\");\n}\nfunction parentPath(path) {\n const index = path.lastIndexOf(\"/\");\n return index < 0 ? \"\" : path.slice(0, index);\n}\nfunction boundedString(value, label, maxBytes) {\n if (typeof value !== \"string\" || value.length === 0) {\n throw new Error(`w7-frame: ${label} must be a non-empty string`);\n }\n const length = new TextEncoder().encode(value).byteLength;\n if (length > maxBytes) throw new Error(`w7-frame: ${label} exceeds ${maxBytes} bytes`);\n return value;\n}\nfunction safeInteger(value, label) {\n if (!Number.isSafeInteger(value) || value < 0) {\n throw new Error(`w7-frame: ${label} must be a non-negative safe integer`);\n }\n return value;\n}\nfunction u32(value, label) {\n const integer = safeInteger(value, label);\n if (integer > 4294967295) throw new Error(`w7-frame: ${label} exceeds uint32`);\n return integer;\n}\nfunction sameSummary(left, right) {\n return left.recordCount === right.recordCount && left.pathCount === right.pathCount && left.deleteCount === right.deleteCount && left.directoryCount === right.directoryCount && left.fileCount === right.fileCount && left.chunkCount === right.chunkCount && left.byteCount === right.byteCount && left.check === right.check;\n}\nasync function readEnvelope(buffer, label) {\n const header = await buffer.readExact(5, `${label} header`);\n return { tag: header[0], length: readU32LE(header, 1), header };\n}\nfunction recordHeader(tag, length) {\n const header = new Uint8Array(5);\n header[0] = tag;\n writeU32LE(header, 1, length);\n return header;\n}\nfunction writeU32LE(out, offset, value) {\n out[offset] = value & 255;\n out[offset + 1] = value >>> 8 & 255;\n out[offset + 2] = value >>> 16 & 255;\n out[offset + 3] = value >>> 24 & 255;\n}\nfunction readU32LE(bytes, offset) {\n return (bytes[offset] | bytes[offset + 1] << 8 | bytes[offset + 2] << 16 | bytes[offset + 3] << 24) >>> 0;\n}\nfunction updateRecordCheck(seed, ...parts) {\n let check = seed;\n for (const part of parts) check = crc32Update(check, part);\n return check;\n}\nvar CRC_SEED = 4294967295;\nvar crcTable = null;\nfunction crc32Update(check, bytes) {\n crcTable ??= createCrcTable();\n const table = crcTable;\n let value = check;\n for (const byte of bytes) value = table[(value ^ byte) & 255] ^ value >>> 8;\n return value >>> 0;\n}\nfunction crc32Finish(check) {\n return (check ^ 4294967295) >>> 0;\n}\nfunction createCrcTable() {\n const table = new Uint32Array(256);\n for (let index = 0; index < table.length; index++) {\n let value = index;\n for (let bit = 0; bit < 8; bit++) {\n value = (value & 1) !== 0 ? 3988292384 ^ value >>> 1 : value >>> 1;\n }\n table[index] = value >>> 0;\n }\n return table;\n}\nfunction noopRetention(bytes) {\n return { bytes, release() {\n } };\n}\nfunction decodeText(bytes, label) {\n try {\n return new TextDecoder(\"utf-8\", { fatal: true, ignoreBOM: false }).decode(bytes);\n } catch (error) {\n throw new Error(`w7-frame: invalid UTF-8 in ${label}: ${errorMessage(error)}`);\n }\n}\nfunction concatBytes(...parts) {\n const output = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0));\n let offset = 0;\n for (const part of parts) {\n output.set(part, offset);\n offset += part.byteLength;\n }\n return output;\n}\nfunction bytesEqual(left, right) {\n return left.length === right.length && left.every((byte, index) => byte === right[index]);\n}\nfunction hex(bytes) {\n return Array.from(bytes, (byte) => byte.toString(16).padStart(2, \"0\")).join(\" \");\n}\nfunction isObject(value) {\n return typeof value === \"object\" && value !== null;\n}\nfunction errorMessage(error) {\n return error instanceof Error ? error.message : String(error);\n}\nfunction throwIfAborted(signal) {\n if (!signal?.aborted) return;\n throw new DOMException(\n signal.reason instanceof Error ? signal.reason.message : String(signal.reason ?? \"Aborted\"),\n \"AbortError\"\n );\n}\nasync function cancelReader(reader, reason) {\n try {\n await reader.cancel(reason);\n } catch {\n }\n try {\n reader.releaseLock();\n } catch {\n }\n}\nvar ExactByteReader = class {\n constructor(reader) {\n this.reader = reader;\n }\n reader;\n done = false;\n async readExact(length, label) {\n if (length === 0) return new Uint8Array(0);\n const output = new Uint8Array(length);\n let offset = 0;\n while (offset < length) {\n if (this.done) {\n throw new Error(\n `w7-frame: stream ended ${offset} bytes into expected ${length}-byte ${label}`\n );\n }\n const next = await this.reader.read(new Uint8Array(length - offset));\n if (next.done) this.done = true;\n else if (next.value.byteLength > 0) {\n output.set(next.value, offset);\n offset += next.value.byteLength;\n }\n }\n return output;\n }\n async ensureEof(stream) {\n if (this.done) return;\n this.reader.releaseLock();\n const reader = stream.getReader();\n try {\n const next = await reader.read();\n if (!next.done && next.value.byteLength > 0) {\n await reader.cancel(new Error(\"w7-frame: trailing bytes after batch-end\"));\n throw new Error(\"w7-frame: trailing bytes after batch-end\");\n }\n this.done = true;\n } finally {\n try {\n reader.releaseLock();\n } catch {\n }\n }\n }\n};";
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* npm-resolve-preamble.ts — preamble injected into
|
|
2
|
+
* npm-resolve-preamble.ts — preamble injected into IsolatePool isolates
|
|
3
3
|
* that run src/npm/resolve-facet.ts and src/npm/resolve-one-facet.ts.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
5
|
+
* IsolatePool serialises the user function via fn.toString() and runs
|
|
6
6
|
* it inside a dynamic worker. Names referenced by the function at module
|
|
7
7
|
* scope are NOT in that worker's lexical scope at runtime — they must be
|
|
8
8
|
* re-declared in the preamble.
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
* byte-equivalent — divergence would mean the facet picks different
|
|
31
31
|
* versions than the in-supervisor path.
|
|
32
32
|
*
|
|
33
|
-
* Preamble bytes are part of the loader-cache key for
|
|
33
|
+
* Preamble bytes are part of the loader-cache key for IsolatePool —
|
|
34
34
|
* any edit invalidates the warm slot and forces a re-load on next
|
|
35
35
|
* dispatch. Acceptable cost for a one-shot resolver phase.
|
|
36
36
|
*/
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* npm-resolve-preamble.ts — preamble injected into
|
|
2
|
+
* npm-resolve-preamble.ts — preamble injected into IsolatePool isolates
|
|
3
3
|
* that run src/npm/resolve-facet.ts and src/npm/resolve-one-facet.ts.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
5
|
+
* IsolatePool serialises the user function via fn.toString() and runs
|
|
6
6
|
* it inside a dynamic worker. Names referenced by the function at module
|
|
7
7
|
* scope are NOT in that worker's lexical scope at runtime — they must be
|
|
8
8
|
* re-declared in the preamble.
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
* byte-equivalent — divergence would mean the facet picks different
|
|
31
31
|
* versions than the in-supervisor path.
|
|
32
32
|
*
|
|
33
|
-
* Preamble bytes are part of the loader-cache key for
|
|
33
|
+
* Preamble bytes are part of the loader-cache key for IsolatePool —
|
|
34
34
|
* any edit invalidates the warm slot and forces a re-load on next
|
|
35
35
|
* dispatch. Acceptable cost for a one-shot resolver phase.
|
|
36
36
|
*/
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* pre-bundle-preamble.ts — preamble injected into
|
|
2
|
+
* pre-bundle-preamble.ts — preamble injected into IsolatePool isolates
|
|
3
3
|
* that run src/pre-bundle-facet.ts.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
5
|
+
* IsolatePool serialises the user function via fn.toString() and runs
|
|
6
6
|
* it inside a dynamic worker built from inline modules. Names referenced
|
|
7
7
|
* by the function at the module-top-level scope are NOT in that worker's
|
|
8
8
|
* lexical scope at runtime — they must be re-declared in the preamble.
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
*
|
|
17
17
|
* NOT in the preamble:
|
|
18
18
|
* - ESBUILD_WASM_BASE64 / wasm BYTES — the wasm Module is shipped via
|
|
19
|
-
*
|
|
19
|
+
* IsolatePool's `wasmModules` option (LOADER `modules` map
|
|
20
20
|
* entry shape `{ wasm: ArrayBuffer }`). Workerd compiles it at
|
|
21
21
|
* module-load (startup phase) and the pool's generated worker.js
|
|
22
22
|
* exposes the resulting WebAssembly.Module on
|
|
@@ -27,14 +27,14 @@
|
|
|
27
27
|
* we inline its source here so the facet doesn't need to fault back
|
|
28
28
|
* to the supervisor for it.
|
|
29
29
|
*
|
|
30
|
-
* Preamble bytes are part of the loader-cache key for
|
|
30
|
+
* Preamble bytes are part of the loader-cache key for IsolatePool.
|
|
31
31
|
* Changing this file invalidates all warm slots in the pre-bundle pool;
|
|
32
32
|
* fine — esbuild boot is the dominant cost and re-paying it once on a
|
|
33
33
|
* deploy is acceptable.
|
|
34
34
|
*/
|
|
35
35
|
/**
|
|
36
36
|
* Preamble string injected ahead of the prebundleOne function in every
|
|
37
|
-
* pre-bundle facet isolate. Must be passed via
|
|
37
|
+
* pre-bundle facet isolate. Must be passed via IsolatePool's
|
|
38
38
|
* `preamble` option.
|
|
39
39
|
*/
|
|
40
40
|
export declare const PRE_BUNDLE_PREAMBLE: string;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* pre-bundle-preamble.ts — preamble injected into
|
|
2
|
+
* pre-bundle-preamble.ts — preamble injected into IsolatePool isolates
|
|
3
3
|
* that run src/pre-bundle-facet.ts.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
5
|
+
* IsolatePool serialises the user function via fn.toString() and runs
|
|
6
6
|
* it inside a dynamic worker built from inline modules. Names referenced
|
|
7
7
|
* by the function at the module-top-level scope are NOT in that worker's
|
|
8
8
|
* lexical scope at runtime — they must be re-declared in the preamble.
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
*
|
|
17
17
|
* NOT in the preamble:
|
|
18
18
|
* - ESBUILD_WASM_BASE64 / wasm BYTES — the wasm Module is shipped via
|
|
19
|
-
*
|
|
19
|
+
* IsolatePool's `wasmModules` option (LOADER `modules` map
|
|
20
20
|
* entry shape `{ wasm: ArrayBuffer }`). Workerd compiles it at
|
|
21
21
|
* module-load (startup phase) and the pool's generated worker.js
|
|
22
22
|
* exposes the resulting WebAssembly.Module on
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
* we inline its source here so the facet doesn't need to fault back
|
|
28
28
|
* to the supervisor for it.
|
|
29
29
|
*
|
|
30
|
-
* Preamble bytes are part of the loader-cache key for
|
|
30
|
+
* Preamble bytes are part of the loader-cache key for IsolatePool.
|
|
31
31
|
* Changing this file invalidates all warm slots in the pre-bundle pool;
|
|
32
32
|
* fine — esbuild boot is the dominant cost and re-paying it once on a
|
|
33
33
|
* deploy is acceptable.
|
|
@@ -37,7 +37,7 @@ import { getExportsResolverJS } from '@nimbus-sh/core/_shared/exports-resolver.j
|
|
|
37
37
|
// NOTE: wasm BYTES deliberately NOT in this preamble. They live in
|
|
38
38
|
// env.ASSETS at /_assets/esbuild-<version>.wasm and are fetched by
|
|
39
39
|
// the supervisor at pool-construction time
|
|
40
|
-
// (src/esbuild-wasm-bytes.ts), then handed to
|
|
40
|
+
// (src/esbuild-wasm-bytes.ts), then handed to IsolatePool's
|
|
41
41
|
// `wasmModules` option which registers them in the LOADER `modules`
|
|
42
42
|
// map as `{ wasm: ArrayBuffer }`. workerd compiles during the
|
|
43
43
|
// worker's module-load phase (where wasm code generation is
|
|
@@ -68,7 +68,7 @@ import { getExportsResolverJS } from '@nimbus-sh/core/_shared/exports-resolver.j
|
|
|
68
68
|
const RESOLVER_HELPERS_SRC = getExportsResolverJS();
|
|
69
69
|
/**
|
|
70
70
|
* Preamble string injected ahead of the prebundleOne function in every
|
|
71
|
-
* pre-bundle facet isolate. Must be passed via
|
|
71
|
+
* pre-bundle facet isolate. Must be passed via IsolatePool's
|
|
72
72
|
* `preamble` option.
|
|
73
73
|
*/
|
|
74
74
|
export const PRE_BUNDLE_PREAMBLE = `
|
|
@@ -67,7 +67,7 @@ export interface InstallBatchResult {
|
|
|
67
67
|
perPackage: InstallBatchPerPackage[];
|
|
68
68
|
/** Wall-clock ms inside the facet (whole batch). */
|
|
69
69
|
elapsed: number;
|
|
70
|
-
/** Counter snapshot at end of batch. Mirrors
|
|
70
|
+
/** Counter snapshot at end of batch. Mirrors @nimbus-sh/platform/diag-counters.js shape
|
|
71
71
|
* for the install-facet subset (commit 3 surfaces these in /api/_diag/memory). */
|
|
72
72
|
facetCounters: {
|
|
73
73
|
tarballsCompleted: number;
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
*/
|
|
39
39
|
// ── Facet function ──────────────────────────────────────────────────────
|
|
40
40
|
//
|
|
41
|
-
// Runs inside a
|
|
41
|
+
// Runs inside a IsolatePool isolate. Serialised via fn.toString();
|
|
42
42
|
// the helpers it references at top-level scope (streamTarEntries,
|
|
43
43
|
// readableStreamToAsyncIterable, MAX_FILE_BYTES) are NOT in the facet's
|
|
44
44
|
// lexical scope — the pool injects them via the preamble. No static
|
package/dist/npm/installer.d.ts
CHANGED
|
@@ -24,7 +24,7 @@ import type { EsbuildService } from '@nimbus-sh/core/runtime/esbuild-service.js'
|
|
|
24
24
|
import { NpmCache } from './cache.js';
|
|
25
25
|
import { type FetchFn } from './resolver.js';
|
|
26
26
|
import { type NpmLogEmitter } from './npm-log.js';
|
|
27
|
-
import type { InstallPhase } from '@nimbus-sh/
|
|
27
|
+
import type { InstallPhase } from '@nimbus-sh/platform/install-phase.js';
|
|
28
28
|
export interface InstallProgress {
|
|
29
29
|
phase: InstallPhase;
|
|
30
30
|
resolved: number;
|
|
@@ -91,7 +91,7 @@ export declare class NpmInstaller {
|
|
|
91
91
|
/**
|
|
92
92
|
* F-2 frontier-coordinator path. Replaces the single-resolve-facet
|
|
93
93
|
* dispatch with a per-package fanout: each BFS layer becomes ONE
|
|
94
|
-
* `
|
|
94
|
+
* `Fanout.submitMany` call, layer N+1 builds from the
|
|
95
95
|
* resolved metadata of layer N.
|
|
96
96
|
*
|
|
97
97
|
* Topology auto-routes per layer:
|
|
@@ -115,12 +115,12 @@ export declare class NpmInstaller {
|
|
|
115
115
|
*/
|
|
116
116
|
private resolveTreeViaFanout;
|
|
117
117
|
/**
|
|
118
|
-
* Batch install via two-tier fan-out (
|
|
118
|
+
* Batch install via two-tier fan-out (Fanout).
|
|
119
119
|
*
|
|
120
120
|
* Shard count is `min(specs.length, INSTALL_PEER_CAP)`, and the topology
|
|
121
121
|
* follows from it:
|
|
122
122
|
* shardCount < IN_DO_THRESHOLD (5) → in-DO fanout in-DO
|
|
123
|
-
* 1
|
|
123
|
+
* 1 IsolatePool with concurrency = shardCount, capped at
|
|
124
124
|
* 4 by V8 invariant. Each shard is one facet running its own
|
|
125
125
|
* installPackagesInFacet.
|
|
126
126
|
* shardCount >= IN_DO_THRESHOLD → peer-DO fanout peer-DO
|
|
@@ -132,7 +132,7 @@ export declare class NpmInstaller {
|
|
|
132
132
|
* `shard-${i}` task key deterministically (tests can predict
|
|
133
133
|
* placement).
|
|
134
134
|
*
|
|
135
|
-
* Pre-fix lineage: this site previously ran ONE
|
|
135
|
+
* Pre-fix lineage: this site previously ran ONE IsolatePool
|
|
136
136
|
* with concurrency=1, internal pLimit(3) — the explicit "collapses
|
|
137
137
|
* what was 4 concurrent dynamic workers (pool.map slots) into 1"
|
|
138
138
|
* Two-tier topology re-expands the fan-out without re-introducing
|
|
@@ -225,7 +225,7 @@ export declare class NpmInstaller {
|
|
|
225
225
|
/**
|
|
226
226
|
* P5 (production reliability) — deterministic supervisor-heap estimate in MiB.
|
|
227
227
|
*
|
|
228
|
-
* Routes through
|
|
228
|
+
* Routes through @nimbus-sh/platform/heap-estimate.js which sums KNOWN
|
|
229
229
|
* supervisor heap allocation sources from runtime counters that
|
|
230
230
|
* ARE accurate inside a DO context (DiagCounters singleton +
|
|
231
231
|
* SqliteVFS.getStats()). This replaces a previous use of
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"installer.d.ts","sourceRoot":"","sources":["../../src/npm/installer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EACV,SAAS,
|
|
1
|
+
{"version":3,"file":"installer.d.ts","sourceRoot":"","sources":["../../src/npm/installer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EACV,SAAS,EAGV,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAEjF,OAAO,EAAE,QAAQ,EAAsB,MAAM,YAAY,CAAC;AAC1D,OAAO,EAEiC,KAAK,OAAO,EACnD,MAAM,eAAe,CAAC;AAEvB,OAAO,EAEL,KAAK,aAAa,EACnB,MAAM,cAAc,CAAC;AAuEtB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sCAAsC,CAAC;AAEzE,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAaD,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAID,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAY;IAClC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAkB;IACtC,OAAO,CAAC,KAAK,CAAW;IACxB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,GAAG,CAAiC;IAC5C,OAAO,CAAC,GAAG,CAAM;IACjB,OAAO,CAAC,UAAU,CAAsC;IACxD;;;;;;OAMG;IACH,OAAO,CAAC,OAAO,CAAsB;IACrC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAA2B;gBAGvC,GAAG,EAAE,SAAS,EACd,GAAG,EAAE,UAAU,EACf,IAAI,CAAC,EAAE;QACL,OAAO,CAAC,EAAE,cAAc,CAAC;QACzB,GAAG,CAAC,EAAE,kBAAkB,CAAC;QACzB,GAAG,CAAC,EAAE,GAAG,CAAC;QACV,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;QACnC,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB;IAYH,4EAA4E;IAC5E,IAAI,QAAQ,IAAI,QAAQ,CAAuB;IAI/C;;;;;OAKG;IACG,OAAO,CACX,UAAU,EAAE,MAAM,EAClB,IAAI,CAAC,EAAE;QACL,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,aAAa,CAAC;KACxB,GACA,OAAO,CAAC,gBAAgB,CAAC;YAkBd,aAAa;IAoV3B;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;YACW,oBAAoB;IAiVlC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;YACW,kBAAkB;IAyNhC;;OAEG;IACH;;;;;;;OAOG;YACW,oBAAoB;YAgCpB,UAAU;IA6CxB;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe;IAiCvB;;OAEG;IACH,OAAO,CAAC,eAAe;IAgCvB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAqC1B;;OAEG;IACH,OAAO,CAAC,aAAa;IAqBrB;;OAEG;YACW,QAAQ;YA4DR,kBAAkB;IAehC,OAAO,CAAC,iBAAiB;IAuCzB;;;;;;;;;;OAUG;YACW,oBAAoB;IA8jBlC;;;;;;;;;;OAUG;IACH,OAAO,CAAC,eAAe;IAgFvB;;OAEG;IACH;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,uBAAuB;IAsD/B,2EAA2E;IAC3E,OAAO,CAAC,kBAAkB;IAI1B;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,OAAO,CAAC,0BAA0B;CAenC"}
|
package/dist/npm/installer.js
CHANGED
|
@@ -27,21 +27,22 @@ import { packumentUrl } from './r2-cache.js';
|
|
|
27
27
|
import { npmAddedLine, npmHttpCacheLine, npmHttpFetchLine, npmTitleLine, } from './npm-log.js';
|
|
28
28
|
import { applySwaps, findRejects, lookupSwap, lookupReject, shouldSkipPackage, shouldWarnSkipTransitive, isOptionalNativeBinding, formatSwapNotice, RegistryRejectError, emitRegistryEvent, } from '../facets/wasm-swap-registry.js';
|
|
29
29
|
import { resolvePackageEntry } from '@nimbus-sh/core/_shared/exports-resolver.js';
|
|
30
|
-
import { encodeWriteBatchStream } from '@nimbus-sh/
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
30
|
+
import { encodeWriteBatchStream } from '@nimbus-sh/platform/w7-frame.js';
|
|
31
|
+
import { IsolatePool } from '@nimbus-sh/fabric/isolate-pool.js';
|
|
32
|
+
import { Fanout, IN_DO_THRESHOLD } from '@nimbus-sh/fabric/fanout.js';
|
|
33
33
|
import { TAR_STREAM_PREAMBLE, W7_FRAME_PREAMBLE } from '../loaders/generated-workers.js';
|
|
34
34
|
import { installPackagesInFacet, } from './install-batch-facet.js';
|
|
35
|
-
import { setInstallPhase, recordInstallFacetCounters, recordPreBundleSummary, recordR2RaceCounters,
|
|
36
|
-
import {
|
|
37
|
-
import {
|
|
35
|
+
import { setInstallPhase, recordInstallFacetCounters, recordPreBundleSummary, recordR2RaceCounters, readDiagCounters, } from '@nimbus-sh/platform/diag-counters.js';
|
|
36
|
+
import { recordCacheStatEvents } from '@nimbus-sh/core/_shared/cache-stats.js';
|
|
37
|
+
import { estimateSupervisorHeap } from '@nimbus-sh/platform/heap-estimate.js';
|
|
38
|
+
import { describeError } from '@nimbus-sh/platform/oom-classify.js';
|
|
38
39
|
import { resolveOnePackumentInFacet, } from './resolve-one-facet.js';
|
|
39
40
|
import { NPM_RESOLVE_PREAMBLE } from '../loaders/npm-resolve-preamble.js';
|
|
40
41
|
import { prebundleOne, buildSliceForSpecifierWithCap, externalsForSpecifier, } from './pre-bundle-facet.js';
|
|
41
42
|
import { PRE_BUNDLE_PREAMBLE } from '../loaders/pre-bundle-preamble.js';
|
|
42
43
|
import { fetchEsbuildWasmBytes } from '../runtime/esbuild-wasm-bytes.js';
|
|
43
|
-
import { CHUNK_SIZE, PRE_BUNDLE_CONCURRENCY, PRE_BUNDLE_SLICE_CAP_BYTES, SUPERVISOR_IN_FLIGHT_ALLOCATION_BUDGET_BYTES, } from '@nimbus-sh/
|
|
44
|
-
import { acquireSupervisorAllocation } from '@nimbus-sh/
|
|
44
|
+
import { CHUNK_SIZE, PRE_BUNDLE_CONCURRENCY, PRE_BUNDLE_SLICE_CAP_BYTES, SUPERVISOR_IN_FLIGHT_ALLOCATION_BUDGET_BYTES, } from '@nimbus-sh/platform/limits.js';
|
|
45
|
+
import { acquireSupervisorAllocation } from '@nimbus-sh/platform/heavy-alloc-coord.js';
|
|
45
46
|
import { countPackageFiles, BARREL_PKG_FILE_THRESHOLD, packageNameFromSpecifier } from '@nimbus-sh/core/runtime/barrel-detect.js';
|
|
46
47
|
import { scanNamedImports, namedImportSignature, buildSyntheticEntry, buildScopedSliceForSynthetic, syntheticEntryPath, } from '../runtime/barrel-synthesizer.js';
|
|
47
48
|
import { enc } from '@nimbus-sh/core/_shared/bytes.js';
|
|
@@ -146,7 +147,7 @@ export class NpmInstaller {
|
|
|
146
147
|
phases['lock-check'] = Date.now() - phaseStart;
|
|
147
148
|
// ── Phase 1: Resolve ──────────────────────────────────────────
|
|
148
149
|
// Frontier-coordinator path. Each BFS layer dispatches to
|
|
149
|
-
//
|
|
150
|
+
// Fanout.submitMany — width <5 in-DO (in-DO fanout),
|
|
150
151
|
// width ≥5 peer-DO (peer-DO fanout). Per-package task body is
|
|
151
152
|
// self-contained (resolveOnePackumentInFacet), supervisor builds
|
|
152
153
|
// layer N+1 from layer N's edges. Missing env.LOADER throws at
|
|
@@ -222,7 +223,7 @@ export class NpmInstaller {
|
|
|
222
223
|
setInstallPhase('fetch');
|
|
223
224
|
// Fetch + extract + write new packages.
|
|
224
225
|
//
|
|
225
|
-
// Single fetch path: one
|
|
226
|
+
// Single fetch path: one IsolatePool isolate (the batch facet)
|
|
226
227
|
// runs the entire install. The facet streams tarballs through gunzip+tar
|
|
227
228
|
// and coalesces package-owned paths into shared writeBatchStream waves;
|
|
228
229
|
// every owner awaits each wave it contributed to.
|
|
@@ -419,7 +420,7 @@ export class NpmInstaller {
|
|
|
419
420
|
/**
|
|
420
421
|
* F-2 frontier-coordinator path. Replaces the single-resolve-facet
|
|
421
422
|
* dispatch with a per-package fanout: each BFS layer becomes ONE
|
|
422
|
-
* `
|
|
423
|
+
* `Fanout.submitMany` call, layer N+1 builds from the
|
|
423
424
|
* resolved metadata of layer N.
|
|
424
425
|
*
|
|
425
426
|
* Topology auto-routes per layer:
|
|
@@ -486,7 +487,7 @@ export class NpmInstaller {
|
|
|
486
487
|
const RESOLVE_PEER_CAP = 8;
|
|
487
488
|
// F-2 fanout pool. One construction reused across every layer; the
|
|
488
489
|
// pool is stateless across submitMany calls.
|
|
489
|
-
const fanoutPool = new
|
|
490
|
+
const fanoutPool = new Fanout(this.env, this.ctx, {
|
|
490
491
|
tag: 'npm-resolve-fanout',
|
|
491
492
|
// 5 minutes per layer is generous; typical layers complete in
|
|
492
493
|
// 1-3 s. Per-task this gates each packument fetch + R2 race.
|
|
@@ -561,8 +562,8 @@ export class NpmInstaller {
|
|
|
561
562
|
};
|
|
562
563
|
return { key: name, args: taskSpec };
|
|
563
564
|
});
|
|
564
|
-
// Dispatch the layer.
|
|
565
|
-
// <5 → in-DO fanout in-DO (
|
|
565
|
+
// Dispatch the layer. Fanout routes:
|
|
566
|
+
// <5 → in-DO fanout in-DO (IsolatePool), concurrency = layer.length (capped at 4)
|
|
566
567
|
// ≥5 → peer-DO fanout peer-DO, N peers = min(layer.length, 32)
|
|
567
568
|
let results;
|
|
568
569
|
const layerT0 = Date.now();
|
|
@@ -769,12 +770,12 @@ export class NpmInstaller {
|
|
|
769
770
|
return { resolved, unresolved };
|
|
770
771
|
}
|
|
771
772
|
/**
|
|
772
|
-
* Batch install via two-tier fan-out (
|
|
773
|
+
* Batch install via two-tier fan-out (Fanout).
|
|
773
774
|
*
|
|
774
775
|
* Shard count is `min(specs.length, INSTALL_PEER_CAP)`, and the topology
|
|
775
776
|
* follows from it:
|
|
776
777
|
* shardCount < IN_DO_THRESHOLD (5) → in-DO fanout in-DO
|
|
777
|
-
* 1
|
|
778
|
+
* 1 IsolatePool with concurrency = shardCount, capped at
|
|
778
779
|
* 4 by V8 invariant. Each shard is one facet running its own
|
|
779
780
|
* installPackagesInFacet.
|
|
780
781
|
* shardCount >= IN_DO_THRESHOLD → peer-DO fanout peer-DO
|
|
@@ -786,7 +787,7 @@ export class NpmInstaller {
|
|
|
786
787
|
* `shard-${i}` task key deterministically (tests can predict
|
|
787
788
|
* placement).
|
|
788
789
|
*
|
|
789
|
-
* Pre-fix lineage: this site previously ran ONE
|
|
790
|
+
* Pre-fix lineage: this site previously ran ONE IsolatePool
|
|
790
791
|
* with concurrency=1, internal pLimit(3) — the explicit "collapses
|
|
791
792
|
* what was 4 concurrent dynamic workers (pool.map slots) into 1"
|
|
792
793
|
* Two-tier topology re-expands the fan-out without re-introducing
|
|
@@ -851,7 +852,7 @@ export class NpmInstaller {
|
|
|
851
852
|
// Peer shards dispatch in bounded phases and each phase is a barrier, so
|
|
852
853
|
// the phase profile is what distinguishes shard work from barrier count.
|
|
853
854
|
const phaseProfile = [];
|
|
854
|
-
const fanoutPool = new
|
|
855
|
+
const fanoutPool = new Fanout(this.env, this.ctx, {
|
|
855
856
|
tag: 'npm-install-batch',
|
|
856
857
|
// Whole-batch timeout. With per-shard parallelism of N=8 peer
|
|
857
858
|
// DOs each running pLimit(3), Mossaic-class 456 packages
|
|
@@ -869,7 +870,7 @@ export class NpmInstaller {
|
|
|
869
870
|
});
|
|
870
871
|
const tasks = nonEmptyShards.map((shardSpecs, shardIdx) => ({
|
|
871
872
|
// Stable-id router key. Same shardIdx → same peer DO across
|
|
872
|
-
// runs. Tests can predict placement via
|
|
873
|
+
// runs. Tests can predict placement via Fanout's
|
|
873
874
|
// `peerSiblingId(key, peerCount)` helper.
|
|
874
875
|
key: `shard-${shardIdx}`,
|
|
875
876
|
args: { packages: shardSpecs, concurrency: 3 },
|
|
@@ -974,7 +975,7 @@ export class NpmInstaller {
|
|
|
974
975
|
}
|
|
975
976
|
catch (e) {
|
|
976
977
|
// Final catch — preserved from the pre-fix shape so the error
|
|
977
|
-
// log line shape stays consistent.
|
|
978
|
+
// log line shape stays consistent. Fanout's internal
|
|
978
979
|
// pools dispose themselves at the end of each submitMany call,
|
|
979
980
|
// so no explicit dispose() is needed here.
|
|
980
981
|
// The earlier inner-catch already logged + threw; if we reach
|
|
@@ -1304,7 +1305,7 @@ export class NpmInstaller {
|
|
|
1304
1305
|
* JSX elements get rejected as "alien" — silent render failure.
|
|
1305
1306
|
*/
|
|
1306
1307
|
async prebundleUsedModules(projDir, installed) {
|
|
1307
|
-
// Pre-bundle now runs in
|
|
1308
|
+
// Pre-bundle now runs in IsolatePool isolates (src/pre-bundle-facet.ts);
|
|
1308
1309
|
// each facet ships its own bundled esbuild-wasm via the preamble. The
|
|
1309
1310
|
// supervisor's EsbuildService is no longer on the bundle path — it
|
|
1310
1311
|
// still serves the transform path (TS/JSX → JS) which is small and
|
|
@@ -1444,7 +1445,7 @@ export class NpmInstaller {
|
|
|
1444
1445
|
// C'.1 deterministic estimator. process.memoryUsage() returns 0
|
|
1445
1446
|
// for every field inside a Durable Object class context (only
|
|
1446
1447
|
// dynamic-worker isolates under nodejs_compat get the real
|
|
1447
|
-
// implementation — see
|
|
1448
|
+
// implementation — see @nimbus-sh/platform/diag-counters.js:4 and
|
|
1448
1449
|
// heap-estimate.ts:6). The previous "supervisor heap 0.0 MiB"
|
|
1449
1450
|
// line was actively misleading: it printed every time, regardless
|
|
1450
1451
|
// of what was actually in the heap, and could not be used to
|
|
@@ -1515,7 +1516,7 @@ export class NpmInstaller {
|
|
|
1515
1516
|
// (where it should live). No supervisor-side caching — see
|
|
1516
1517
|
// src/esbuild-wasm-bytes.ts for the full architectural rationale.
|
|
1517
1518
|
//
|
|
1518
|
-
// Bytes are shipped into each facet via
|
|
1519
|
+
// Bytes are shipped into each facet via IsolatePool's
|
|
1519
1520
|
// `wasmModules` option which workerd registers in the LOADER
|
|
1520
1521
|
// `modules` map as `{ wasm: ArrayBuffer }`. Workerd compiles at
|
|
1521
1522
|
// module-load (startup phase, where wasm code generation is
|
|
@@ -1562,12 +1563,12 @@ export class NpmInstaller {
|
|
|
1562
1563
|
if (wasmBytes.byteLength > maxRetainedWasmBytes) {
|
|
1563
1564
|
throw new RangeError(`pre-bundle wasm payload ${wasmBytes.byteLength} exceeds the ${maxRetainedWasmBytes}-byte retained budget`);
|
|
1564
1565
|
}
|
|
1565
|
-
//
|
|
1566
|
+
// IsolatePool keeps the constructor-time module bytes until
|
|
1566
1567
|
// dispose(), so retain their exact credit rather than treating
|
|
1567
1568
|
// construction as a handoff that immediately frees the ArrayBuffer.
|
|
1568
1569
|
setupAllocation.shrinkTo(wasmBytes.byteLength);
|
|
1569
1570
|
try {
|
|
1570
|
-
pool = new
|
|
1571
|
+
pool = new IsolatePool(this.env, this.ctx, {
|
|
1571
1572
|
concurrency: PRE_BUNDLE_CONCURRENCY,
|
|
1572
1573
|
timeoutMs: 60_000,
|
|
1573
1574
|
retries: 0,
|
|
@@ -1994,7 +1995,7 @@ export class NpmInstaller {
|
|
|
1994
1995
|
/**
|
|
1995
1996
|
* P5 (production reliability) — deterministic supervisor-heap estimate in MiB.
|
|
1996
1997
|
*
|
|
1997
|
-
* Routes through
|
|
1998
|
+
* Routes through @nimbus-sh/platform/heap-estimate.js which sums KNOWN
|
|
1998
1999
|
* supervisor heap allocation sources from runtime counters that
|
|
1999
2000
|
* ARE accurate inside a DO context (DiagCounters singleton +
|
|
2000
2001
|
* SqliteVFS.getStats()). This replaces a previous use of
|
|
@@ -2035,7 +2036,7 @@ export class NpmInstaller {
|
|
|
2035
2036
|
// process.memoryUsage() which returns 0 for every field inside a
|
|
2036
2037
|
// Durable Object class context (only dynamic-worker isolates under
|
|
2037
2038
|
// nodejs_compat get the real implementation — see
|
|
2038
|
-
//
|
|
2039
|
+
// @nimbus-sh/platform/diag-counters.js:4). The deterministic
|
|
2039
2040
|
// supervisor-heap estimator is the C'.1 replacement; see
|
|
2040
2041
|
// NpmInstaller._estimateSupervisorHeapMiB.
|
|
2041
2042
|
function parentOf(path) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* pre-bundle-facet.ts —
|
|
2
|
+
* pre-bundle-facet.ts — IsolatePool entry for esbuild pre-bundling.
|
|
3
3
|
*
|
|
4
4
|
* Why this exists
|
|
5
5
|
* ───────────────
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* the previous isolate.
|
|
15
15
|
*
|
|
16
16
|
* The fix is to dispatch each per-specifier `esbuild.build` to a
|
|
17
|
-
*
|
|
17
|
+
* IsolatePool isolate. Each facet has its own 128 MB budget and
|
|
18
18
|
* stable-slot reuse keeps the warm-up cost amortized across the 8
|
|
19
19
|
* concurrent specs of a typical install. The install-batch facet uses
|
|
20
20
|
* the same isolate boundary for tarball extraction.
|