@poe-platform/safe-fs 0.1.807 → 0.1.809
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -1
- package/dist/safe-fs/contracts/filesystem.d.ts +20 -1
- package/dist/safe-fs/core.d.ts +3 -1
- package/dist/safe-fs/core.js +3 -1
- package/dist/safe-fs/fs/capabilities.d.ts +1 -1
- package/dist/safe-fs/fs/capabilities.js +9 -5
- package/dist/safe-fs/fs/devices/index.d.ts +3 -2
- package/dist/safe-fs/fs/devices/index.js +53 -9
- package/dist/safe-fs/fs/memory/atomic-view.d.ts +1 -0
- package/dist/safe-fs/fs/memory/index.d.ts +6 -3
- package/dist/safe-fs/fs/memory/index.js +101 -13
- package/dist/safe-fs/fs/mount/index.d.ts +4 -2
- package/dist/safe-fs/fs/mount/index.js +169 -10
- package/dist/safe-fs/fs/overlay/index.d.ts +3 -0
- package/dist/safe-fs/fs/overlay/index.js +177 -59
- package/dist/safe-fs/fs/overlay/memory-publication.d.ts +1 -0
- package/dist/safe-fs/fs/overlay/memory-publication.js +25 -0
- package/dist/safe-fs/fs/quota/index.js +9 -4
- package/dist/safe-fs/fs/read-file-stream.d.ts +7 -0
- package/dist/safe-fs/fs/read-file-stream.js +103 -0
- package/dist/safe-fs/fs/readonly/index.d.ts +1 -0
- package/dist/safe-fs/fs/readonly/index.js +10 -0
- package/dist/safe-fs/fs/real/index.d.ts +1 -0
- package/dist/safe-fs/fs/real/index.js +29 -0
- package/dist/safe-fs/fs/s3/filesystem.d.ts +2 -1
- package/dist/safe-fs/fs/s3/filesystem.js +55 -16
- package/dist/safe-fs/fs/scoped.js +13 -5
- package/dist/safe-fs/index.d.ts +1 -0
- package/dist/safe-fs/index.js +1 -0
- package/dist/safe-fs/runtime-core.d.ts +1 -0
- package/dist/safe-fs/runtime-core.js +1 -0
- package/dist/safe-fs/storage.d.ts +57 -0
- package/dist/safe-fs/storage.js +364 -0
- package/dist/safe-fs/testing/object-io-metrics.js +1 -0
- package/dist/xml-ast/errors.d.ts +4 -0
- package/dist/xml-ast/errors.js +7 -0
- package/dist/xml-ast/index.d.ts +2 -4
- package/dist/xml-ast/index.js +3 -7
- package/dist/xml-ast/stream.d.ts +31 -0
- package/dist/xml-ast/stream.js +179 -0
- package/package.json +10 -6
|
@@ -53,7 +53,7 @@ export declare class S3FileSystem implements FileSystem {
|
|
|
53
53
|
truncate: boolean;
|
|
54
54
|
atomicRenameNoReplace: false;
|
|
55
55
|
rename: boolean;
|
|
56
|
-
streamingAppend:
|
|
56
|
+
streamingAppend: false;
|
|
57
57
|
randomAccessWrite: false;
|
|
58
58
|
readOnly: boolean;
|
|
59
59
|
symlinks: false;
|
|
@@ -109,6 +109,7 @@ export declare class S3FileSystem implements FileSystem {
|
|
|
109
109
|
appendFile(path: string, data: Uint8Array, options?: AppendFileOptions): Promise<void>;
|
|
110
110
|
mkdir(input: string, options?: MkdirOptions): Promise<void>;
|
|
111
111
|
readdir(input: string, options?: ReadDirectoryOptions): Promise<DirectoryEntry[]>;
|
|
112
|
+
iterateDirectory(input: string, options?: FsOptions): AsyncIterable<DirectoryEntry>;
|
|
112
113
|
private tree;
|
|
113
114
|
rmdir(input: string, options?: FsOptions): Promise<void>;
|
|
114
115
|
rm(input: string, options?: RemoveOptions): Promise<void>;
|
|
@@ -152,8 +152,8 @@ export class S3FileSystem {
|
|
|
152
152
|
atomicRenameNoReplace: false,
|
|
153
153
|
rename: this.allowRename && options.transport.capabilities?.conditionalDelete === true
|
|
154
154
|
&& (options.transport.capabilities?.conditionalCopy === true || options.transport.capabilities?.conditionalPut === true),
|
|
155
|
-
|
|
156
|
-
|
|
155
|
+
// Conditional append currently collects the old and incoming bodies.
|
|
156
|
+
streamingAppend: false,
|
|
157
157
|
randomAccessWrite: false,
|
|
158
158
|
readOnly: options.readOnly ?? false,
|
|
159
159
|
symlinks: false, hardlinks: false, permissions: false,
|
|
@@ -341,25 +341,33 @@ export class S3FileSystem {
|
|
|
341
341
|
if (key.endsWith("/") && object.Size !== 0)
|
|
342
342
|
fail("ENOTSUP", "listObjectsV2", path, "nonempty slash-suffixed objects cannot be treated as directories");
|
|
343
343
|
}
|
|
344
|
-
async *pages(prefix, path, options, delimiter, maxKeys = this.pageSize) {
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
344
|
+
async *pages(prefix, path, options, delimiter, maxKeys = this.pageSize, rememberTokens = true) {
|
|
345
|
+
// Buffered callers preserve exact repeated-token rejection. Lazy traversal
|
|
346
|
+
// detects continuation cycles without retaining the whole token history.
|
|
347
|
+
const tokens = rememberTokens ? new Set() : undefined;
|
|
348
|
+
let token, checkpoint;
|
|
349
|
+
let entries = 0, pageCount = 0, span = 0, power = 1;
|
|
348
350
|
do {
|
|
349
351
|
const page = await this.page(prefix, path, options, delimiter, token, maxKeys);
|
|
350
352
|
entries += (page.Contents?.length ?? 0) + (page.CommonPrefixes?.length ?? 0);
|
|
351
|
-
if (entries > this.maxListEntries ||
|
|
353
|
+
if (entries > this.maxListEntries || ++pageCount > this.maxListEntries)
|
|
352
354
|
fail("EFBIG", "listObjectsV2", path, "listing exceeds maxListEntries");
|
|
355
|
+
const truncated = page.IsTruncated, next = page.NextContinuationToken;
|
|
353
356
|
yield page;
|
|
354
|
-
if (!
|
|
357
|
+
if (!truncated)
|
|
355
358
|
return;
|
|
356
|
-
token
|
|
357
|
-
if (typeof token !== "string" || !token || tokens.has(token))
|
|
359
|
+
if (typeof next !== "string" || !next || next === token || next === checkpoint || tokens?.has(next))
|
|
358
360
|
fail("EIO", "listObjectsV2", path, "missing or repeated continuation token");
|
|
359
|
-
|
|
361
|
+
if (++span === power) {
|
|
362
|
+
checkpoint = next;
|
|
363
|
+
span = 0;
|
|
364
|
+
power = Math.min(Number.MAX_SAFE_INTEGER, power * 2);
|
|
365
|
+
}
|
|
366
|
+
tokens?.add(next);
|
|
367
|
+
token = next;
|
|
360
368
|
} while (true);
|
|
361
369
|
}
|
|
362
|
-
async inspect(path, options) {
|
|
370
|
+
async inspect(path, options, rememberTokens = true) {
|
|
363
371
|
if (path === "/") {
|
|
364
372
|
const listing = await this.page(this.prefix, path, options, undefined, undefined, 1);
|
|
365
373
|
const marker = this.prefix && listing.Contents?.some(object => object.Key === this.prefix)
|
|
@@ -375,7 +383,7 @@ export class S3FileSystem {
|
|
|
375
383
|
const directoryKey = this.directoryKey(path);
|
|
376
384
|
let directory = false;
|
|
377
385
|
let hasMarker = false;
|
|
378
|
-
for await (const children of this.pages(directoryKey, path, options, undefined, 1)) {
|
|
386
|
+
for await (const children of this.pages(directoryKey, path, options, undefined, 1, rememberTokens)) {
|
|
379
387
|
if ((children.Contents?.length ?? 0) > 0) {
|
|
380
388
|
directory = true;
|
|
381
389
|
hasMarker = children.Contents.some(object => object.Key === directoryKey);
|
|
@@ -393,7 +401,7 @@ export class S3FileSystem {
|
|
|
393
401
|
return { stat: this.makeStat("directory", marker), ...(marker ? { metadata: marker } : {}) };
|
|
394
402
|
return undefined;
|
|
395
403
|
}
|
|
396
|
-
async lookup(path, options) {
|
|
404
|
+
async lookup(path, options, rememberTokens = true) {
|
|
397
405
|
const ancestors = [];
|
|
398
406
|
for (let parent = posix.dirname(path); path !== "/"; parent = posix.dirname(parent)) {
|
|
399
407
|
ancestors.unshift(parent);
|
|
@@ -403,13 +411,13 @@ export class S3FileSystem {
|
|
|
403
411
|
for (const parent of ancestors) {
|
|
404
412
|
if (parent === "/" && this.prefix === "")
|
|
405
413
|
continue;
|
|
406
|
-
const ancestor = await this.inspect(parent, options);
|
|
414
|
+
const ancestor = await this.inspect(parent, options, rememberTokens);
|
|
407
415
|
if (!ancestor)
|
|
408
416
|
fail("ENOENT", "stat", parent);
|
|
409
417
|
if (ancestor.stat.type !== "directory")
|
|
410
418
|
fail("ENOTDIR", "stat", parent);
|
|
411
419
|
}
|
|
412
|
-
return this.inspect(path, options);
|
|
420
|
+
return this.inspect(path, options, rememberTokens);
|
|
413
421
|
}
|
|
414
422
|
requireDirectorySuffix(input, info) {
|
|
415
423
|
if (input.endsWith("/") && info?.stat.type === "file")
|
|
@@ -596,6 +604,37 @@ export class S3FileSystem {
|
|
|
596
604
|
}
|
|
597
605
|
return [...entries.values()].sort((left, right) => compareKeys(left.name, right.name));
|
|
598
606
|
}
|
|
607
|
+
async *iterateDirectory(input, options = {}) {
|
|
608
|
+
const path = this.path(input);
|
|
609
|
+
const directory = await this.lookup(path, options, false);
|
|
610
|
+
if (!directory)
|
|
611
|
+
fail("ENOENT", "iterateDirectory", path);
|
|
612
|
+
if (directory.stat.type !== "directory")
|
|
613
|
+
fail("ENOTDIR", "iterateDirectory", path);
|
|
614
|
+
const prefix = this.directoryKey(path);
|
|
615
|
+
for await (const page of this.pages(prefix, path, options, "/", this.pageSize, false)) {
|
|
616
|
+
// Snapshot one bounded service page before yielding to caller code.
|
|
617
|
+
if ((page.Contents?.length ?? 0) + (page.CommonPrefixes?.length ?? 0) > this.pageSize)
|
|
618
|
+
fail("EIO", "iterateDirectory", path, "transport exceeded requested page size");
|
|
619
|
+
const entries = [];
|
|
620
|
+
for (const item of page.Contents ?? [])
|
|
621
|
+
if (item.Key !== prefix)
|
|
622
|
+
entries.push({ name: item.Key.slice(prefix.length), type: "file" });
|
|
623
|
+
for (const item of page.CommonPrefixes ?? [])
|
|
624
|
+
entries.push({ name: item.Prefix.slice(prefix.length, -1), type: "directory" });
|
|
625
|
+
for (const entry of entries) {
|
|
626
|
+
this.checkAbort(options, "iterateDirectory", path);
|
|
627
|
+
if (!entry.name || entry.name.includes("/"))
|
|
628
|
+
fail("EIO", "iterateDirectory", path, "invalid direct child in delimited listing");
|
|
629
|
+
// Revalidation preserves file/prefix collision detection across pages
|
|
630
|
+
// and follows the backend's live namespace without retaining all names.
|
|
631
|
+
const child = await this.lookup(path === "/" ? `/${entry.name}` : `${path}/${entry.name}`, options, false);
|
|
632
|
+
this.checkAbort(options, "iterateDirectory", path);
|
|
633
|
+
if (child)
|
|
634
|
+
yield { name: entry.name, type: child.stat.type };
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
}
|
|
599
638
|
async tree(path, options) {
|
|
600
639
|
const objects = new Map();
|
|
601
640
|
for await (const page of this.pages(this.directoryKey(path), path, options)) {
|
|
@@ -21,7 +21,7 @@ const operations = new Set([
|
|
|
21
21
|
"prepareDirectoryAncestry", "prepareStagingResolution", "publishStagedFileSet",
|
|
22
22
|
"publishFileConditional", "removeEntryConditional", "removeTreeConditional", "writeFileConditional", "removeFileConditional", "createStagedFile", "publishStagedFile", "removeStagedFile", "prepareDirectory",
|
|
23
23
|
"confineExtraction", "access", "appendFile", "canonicalizeMissingTarget", "capabilitiesFor", "chmod", "compareEntry",
|
|
24
|
-
"copyFile", "link", "lstat", "mkdir", "openReadFile", "openResizeFile", "readFile", "readStream", "readdir",
|
|
24
|
+
"copyFile", "link", "lstat", "mkdir", "openReadFile", "openResizeFile", "readFile", "readStream", "readdir", "iterateDirectory",
|
|
25
25
|
"readlink", "realpath", "rename", "resizeFile", "rm", "rmdir", "unlink", "stat", "symlink", "truncate", "utimes",
|
|
26
26
|
"writeFile", "writeStream",
|
|
27
27
|
]);
|
|
@@ -188,7 +188,7 @@ export function scopeFileSystem(filesystem, charge, signal, cleanupCharge = char
|
|
|
188
188
|
close: (options = {}) => closing ??= Promise.resolve().then(() => descriptor.close(options)),
|
|
189
189
|
};
|
|
190
190
|
};
|
|
191
|
-
const wrapStream = (source, options) => {
|
|
191
|
+
const wrapStream = (source, options, chargeNext = false) => {
|
|
192
192
|
const budgetFrames = getScopedTransportBudget();
|
|
193
193
|
return {
|
|
194
194
|
[Symbol.asyncIterator]() {
|
|
@@ -203,6 +203,8 @@ export function scopeFileSystem(filesystem, charge, signal, cleanupCharge = char
|
|
|
203
203
|
await closing;
|
|
204
204
|
return { done: true, value: undefined };
|
|
205
205
|
}
|
|
206
|
+
if (chargeNext)
|
|
207
|
+
admit(options);
|
|
206
208
|
const result = await withScopedTransportBudget(budgetFrames, operation);
|
|
207
209
|
assertOpen(options);
|
|
208
210
|
if (closing) {
|
|
@@ -341,6 +343,8 @@ export function scopeFileSystem(filesystem, charge, signal, cleanupCharge = char
|
|
|
341
343
|
if (property === "publishStagedFile") {
|
|
342
344
|
const publishOptions = (options ?? {});
|
|
343
345
|
await requireOwnedMutation(original, args[1], "atomicFileStaging", publishOptions, publishOptions.destination === null);
|
|
346
|
+
if (publishOptions.preserveIdentity)
|
|
347
|
+
await requireOwnedMutation(original, args[1], "atomicStagedFileMutation", publishOptions);
|
|
344
348
|
const query = options.destination === null ? { ...options, create: true } : options;
|
|
345
349
|
const declared = ownedMutationCapabilities(original, await original.capabilitiesFor?.(args[1], query) ?? original.capabilities);
|
|
346
350
|
assertReady();
|
|
@@ -430,7 +434,11 @@ export function scopeFileSystem(filesystem, charge, signal, cleanupCharge = char
|
|
|
430
434
|
});
|
|
431
435
|
return Reflect.apply(method, original, args);
|
|
432
436
|
})();
|
|
437
|
+
if (property === "iterateDirectory")
|
|
438
|
+
args[1] = { ...args[1], signal: combineSignal(args[1]?.signal) };
|
|
433
439
|
const result = Reflect.apply(method, original, args);
|
|
440
|
+
if (property === "iterateDirectory")
|
|
441
|
+
return wrapStream(result, args[1], true);
|
|
434
442
|
return property === "readStream" ? wrapStream(result, args[1]) : result;
|
|
435
443
|
};
|
|
436
444
|
const dispatch = (...args) => {
|
|
@@ -438,7 +446,7 @@ export function scopeFileSystem(filesystem, charge, signal, cleanupCharge = char
|
|
|
438
446
|
return executeDispatch(args);
|
|
439
447
|
if (operations.has(property)) {
|
|
440
448
|
const optionIndex = ["compareEntry", "createStagedFile", "utimes"].includes(String(property)) ? 3
|
|
441
|
-
: ["writeFile", "appendFile", "writeStream", "rename", "copyFile", "link", "symlink", "access", "chmod", "truncate", "resizeFile", "publishFileConditional", "writeFileConditional", "publishStagedFile"].includes(String(property)) ? 2 : 1;
|
|
449
|
+
: ["writeFile", "appendFile", "writeStream", "rename", "copyFile", "link", "symlink", "access", "chmod", "truncate", "resizeFile", "publishFileConditional", "writeFileConditional", "publishStagedFile", "publishStagedFileSet"].includes(String(property)) ? 2 : 1;
|
|
442
450
|
args[optionIndex] = scopeTransportOptions((args[optionIndex] ?? {}), admit, credit);
|
|
443
451
|
}
|
|
444
452
|
return runScopedTransportBudget(admit, () => executeDispatch(args), credit);
|
|
@@ -451,7 +459,7 @@ export function scopeFileSystem(filesystem, charge, signal, cleanupCharge = char
|
|
|
451
459
|
try {
|
|
452
460
|
const declared = ownedMutationCapabilities(original, await original.capabilitiesFor?.(path, { ...controls, stagingResolution: true }) ?? original.capabilities);
|
|
453
461
|
controls.signal?.throwIfAborted();
|
|
454
|
-
if (declared.synchronousStagingResolution !== true)
|
|
462
|
+
if (declared.synchronousStagingResolution !== true || controls.followFinalSymlink === true && declared.synchronousFollowedStagingResolution !== true)
|
|
455
463
|
throw new FsError("ENOTSUP", { path });
|
|
456
464
|
const resolution = snapshotStagingResolution(await Reflect.apply(method, original, [path, controls]));
|
|
457
465
|
controls.signal?.throwIfAborted();
|
|
@@ -528,7 +536,7 @@ export function scopeFileSystem(filesystem, charge, signal, cleanupCharge = char
|
|
|
528
536
|
? async (...args) => ownedMutationCapabilities(original, retainedResizeCapabilities(original, await dispatch(...args)))
|
|
529
537
|
: property === "openReadFile"
|
|
530
538
|
? async (...args) => wrapHandle(await dispatch(...args))
|
|
531
|
-
: operations.has(property) && property !== "canonicalizeMissingTarget" && property !== "readStream"
|
|
539
|
+
: operations.has(property) && property !== "canonicalizeMissingTarget" && property !== "readStream" && property !== "iterateDirectory"
|
|
532
540
|
? (...args) => {
|
|
533
541
|
try {
|
|
534
542
|
const res = dispatch(...args);
|
package/dist/safe-fs/index.d.ts
CHANGED
|
@@ -23,3 +23,4 @@ export { createFileSystem, readConfigRecord, validateFileSystemConfig } from "./
|
|
|
23
23
|
export type { FileSystemConfig, FileSystemAdapterDescriptor, FileSystemAdapterRegistry } from "./config.js";
|
|
24
24
|
export { createNodeFileSystemAdapterRegistry } from "./config.node.js";
|
|
25
25
|
export { createPortableFileSystemAdapterRegistry } from "./config.portable.js";
|
|
26
|
+
export { readFileStream } from "./fs/read-file-stream.js";
|
package/dist/safe-fs/index.js
CHANGED
|
@@ -19,3 +19,4 @@ export * from "./node/index.js";
|
|
|
19
19
|
export { createFileSystem, readConfigRecord, validateFileSystemConfig } from "./config.js";
|
|
20
20
|
export { createNodeFileSystemAdapterRegistry } from "./config.node.js";
|
|
21
21
|
export { createPortableFileSystemAdapterRegistry } from "./config.portable.js";
|
|
22
|
+
export { readFileStream } from "./fs/read-file-stream.js";
|
|
@@ -25,3 +25,4 @@ export declare function tryWriteMemoryFileSync(fs: FileSystem, path: string, dat
|
|
|
25
25
|
export declare function tryMkdirMemorySync(fs: FileSystem, path: string, recursive: boolean, mode: number, signal?: AbortSignal, dirPrefixHint?: string): boolean;
|
|
26
26
|
export declare function tryRmRfMemorySync(fs: FileSystem, path: string, signal?: AbortSignal): boolean;
|
|
27
27
|
export declare function tryWriteMemoryFileInDirSync(fs: FileSystem, dirPrefix: string, name: string, data: Uint8Array, append: boolean, mode: number, signal?: AbortSignal): boolean;
|
|
28
|
+
export { readFileStream } from "./fs/read-file-stream.js";
|
|
@@ -55,3 +55,4 @@ export function tryRmRfMemorySync(fs, path, signal) {
|
|
|
55
55
|
export function tryWriteMemoryFileInDirSync(fs, dirPrefix, name, data, append, mode, signal) {
|
|
56
56
|
return Boolean((fs && _getMemHooks()?.byProto?.get(Object.getPrototypeOf(fs))?.tryWriteMemoryFileInDirSync)?.(fs, dirPrefix, name, data, append, mode, signal));
|
|
57
57
|
}
|
|
58
|
+
export { readFileStream } from "./fs/read-file-stream.js";
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { FileSystem } from "./contracts/filesystem.js";
|
|
2
|
+
export interface PagedStorageContext {
|
|
3
|
+
readonly fs: FileSystem;
|
|
4
|
+
readonly cwd: string;
|
|
5
|
+
readonly env: Readonly<Record<string, string | undefined>>;
|
|
6
|
+
readonly signal: AbortSignal;
|
|
7
|
+
}
|
|
8
|
+
/** A bounded page cache backed only by the caller filesystem. The default is
|
|
9
|
+
* one MiB; callers own close() on every outcome. Memory filesystems still retain
|
|
10
|
+
* spilled data in RAM, so large workloads require an external backing provider. */
|
|
11
|
+
export declare class PagedStorage {
|
|
12
|
+
private readonly context;
|
|
13
|
+
private readonly maxPages;
|
|
14
|
+
private readonly pages;
|
|
15
|
+
private readonly controller;
|
|
16
|
+
private readonly signal;
|
|
17
|
+
private descriptor;
|
|
18
|
+
private parent;
|
|
19
|
+
private initial;
|
|
20
|
+
private path;
|
|
21
|
+
private diskLength;
|
|
22
|
+
private unlinked;
|
|
23
|
+
private end;
|
|
24
|
+
private active;
|
|
25
|
+
private closing;
|
|
26
|
+
constructor(context: PagedStorageContext, maxPages?: number);
|
|
27
|
+
allocate(length: number): number;
|
|
28
|
+
private operation;
|
|
29
|
+
private spill;
|
|
30
|
+
private flush;
|
|
31
|
+
private page;
|
|
32
|
+
write(position: number, bytes: Uint8Array): Promise<void>;
|
|
33
|
+
append(bytes: Uint8Array): Promise<number>;
|
|
34
|
+
read(position: number, length: number): Promise<Uint8Array>;
|
|
35
|
+
close(): Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
/** Fixed-depth radix index: both dense and sparse 64-bit keys use bounded RAM. */
|
|
38
|
+
export declare class IntegerTable {
|
|
39
|
+
private readonly storage;
|
|
40
|
+
private readonly maxEntries;
|
|
41
|
+
private root;
|
|
42
|
+
private revision;
|
|
43
|
+
private readonly branches;
|
|
44
|
+
private readonly cache;
|
|
45
|
+
constructor(storage: Pick<PagedStorage, "allocate" | "read" | "write">, maxEntries?: number);
|
|
46
|
+
private retain;
|
|
47
|
+
get(key: bigint): Promise<bigint | undefined>;
|
|
48
|
+
set(key: bigint, value: bigint): Promise<void>;
|
|
49
|
+
/** Ordered replay with a fixed 16-level radix stack (at most 2 KiB of
|
|
50
|
+
* owned node bytes) and 128 output records. Batches avoid alternating sparse
|
|
51
|
+
* index reads with consumer writes on every record. Mutation is not permitted. */
|
|
52
|
+
entries(start?: bigint, end?: bigint): AsyncGenerator<readonly [bigint, bigint]>;
|
|
53
|
+
/** Hot radix paths must not evict dirty leaf pages in a one-page backend. */
|
|
54
|
+
private branch;
|
|
55
|
+
private lookup;
|
|
56
|
+
private persist;
|
|
57
|
+
}
|
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
import { FsError, isFsError } from "./contracts/errors.js";
|
|
2
|
+
const pageBytes = 16 * 1024;
|
|
3
|
+
let serial = 0;
|
|
4
|
+
/** A bounded page cache backed only by the caller filesystem. The default is
|
|
5
|
+
* one MiB; callers own close() on every outcome. Memory filesystems still retain
|
|
6
|
+
* spilled data in RAM, so large workloads require an external backing provider. */
|
|
7
|
+
export class PagedStorage {
|
|
8
|
+
context;
|
|
9
|
+
maxPages;
|
|
10
|
+
pages = new Map();
|
|
11
|
+
controller = new AbortController();
|
|
12
|
+
signal;
|
|
13
|
+
descriptor;
|
|
14
|
+
parent;
|
|
15
|
+
initial;
|
|
16
|
+
path;
|
|
17
|
+
diskLength = 0;
|
|
18
|
+
unlinked = false;
|
|
19
|
+
end = 8; // Zero is the absent-pointer sentinel.
|
|
20
|
+
active = Promise.resolve();
|
|
21
|
+
closing;
|
|
22
|
+
constructor(context, maxPages = 64) {
|
|
23
|
+
this.context = context;
|
|
24
|
+
this.maxPages = maxPages;
|
|
25
|
+
if (!Number.isSafeInteger(maxPages) || maxPages < 1)
|
|
26
|
+
throw new RangeError("Invalid storage page cache size");
|
|
27
|
+
this.signal = AbortSignal.any([context.signal, this.controller.signal]);
|
|
28
|
+
}
|
|
29
|
+
allocate(length) {
|
|
30
|
+
this.signal.throwIfAborted();
|
|
31
|
+
if (!Number.isSafeInteger(length) || length < 0 || length > Number.MAX_SAFE_INTEGER - this.end)
|
|
32
|
+
throw new FsError("EFBIG");
|
|
33
|
+
const offset = this.end;
|
|
34
|
+
this.end += length;
|
|
35
|
+
return offset;
|
|
36
|
+
}
|
|
37
|
+
operation(action) {
|
|
38
|
+
this.signal.throwIfAborted();
|
|
39
|
+
const work = this.active.then(() => {
|
|
40
|
+
this.signal.throwIfAborted();
|
|
41
|
+
return action();
|
|
42
|
+
});
|
|
43
|
+
this.active = work.then(() => undefined, () => undefined);
|
|
44
|
+
return work;
|
|
45
|
+
}
|
|
46
|
+
async spill() {
|
|
47
|
+
const { fs, cwd, env } = this.context;
|
|
48
|
+
if (!fs.open || !fs.removeFileConditional)
|
|
49
|
+
throw new FsError("ENOTSUP", { message: "spill storage requires retained read/write handles and conditional removal" });
|
|
50
|
+
const selected = env.TMPDIR || cwd;
|
|
51
|
+
const directory = selected.startsWith("/") ? selected : `${cwd}/${selected}`;
|
|
52
|
+
this.parent = await fs.stat(directory, { signal: this.signal });
|
|
53
|
+
this.signal.throwIfAborted();
|
|
54
|
+
if (this.parent.type !== "directory")
|
|
55
|
+
throw new FsError("ENOTDIR", { path: directory });
|
|
56
|
+
for (let attempt = 0; attempt < 128; attempt++) {
|
|
57
|
+
const path = `${directory.endsWith("/") ? directory : directory + "/"}.storage-${++serial}`;
|
|
58
|
+
try {
|
|
59
|
+
this.descriptor = await fs.open(path, { access: "readwrite", creation: "exclusive", mode: 0o600, signal: this.signal });
|
|
60
|
+
this.path = path;
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
this.signal.throwIfAborted();
|
|
65
|
+
if (!isFsError(error) || error.code !== "EEXIST")
|
|
66
|
+
throw error;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (!this.descriptor)
|
|
70
|
+
throw new FsError("EEXIST", { path: directory });
|
|
71
|
+
// Retain the acquisition receipt even if cancellation wins immediately after open.
|
|
72
|
+
this.initial = await this.descriptor.stat();
|
|
73
|
+
this.signal.throwIfAborted();
|
|
74
|
+
if (!this.descriptor.capabilities.positionedRead || !this.descriptor.capabilities.positionedWrite)
|
|
75
|
+
throw new FsError("ENOTSUP");
|
|
76
|
+
// Detach the name before storing input. A later output or random-source path
|
|
77
|
+
// cannot alias our scratch file; the retained descriptor owns its lifetime.
|
|
78
|
+
await fs.removeFileConditional(this.path, { parent: this.parent, expected: this.initial, signal: this.signal });
|
|
79
|
+
this.unlinked = true;
|
|
80
|
+
for (const [number, page] of this.pages)
|
|
81
|
+
await this.flush(number, page);
|
|
82
|
+
}
|
|
83
|
+
async flush(number, page) {
|
|
84
|
+
if (!page.dirty)
|
|
85
|
+
return;
|
|
86
|
+
let offset = 0;
|
|
87
|
+
while (offset < pageBytes) {
|
|
88
|
+
this.signal.throwIfAborted();
|
|
89
|
+
const count = await this.descriptor.write(page.bytes.subarray(offset), number * pageBytes + offset, { signal: this.signal });
|
|
90
|
+
this.signal.throwIfAborted();
|
|
91
|
+
if (!Number.isSafeInteger(count) || count <= 0 || count > pageBytes - offset)
|
|
92
|
+
throw new FsError("EIO");
|
|
93
|
+
offset += count;
|
|
94
|
+
}
|
|
95
|
+
this.diskLength = Math.max(this.diskLength, (number + 1) * pageBytes);
|
|
96
|
+
page.dirty = false;
|
|
97
|
+
}
|
|
98
|
+
async page(number) {
|
|
99
|
+
this.signal.throwIfAborted();
|
|
100
|
+
const cached = this.pages.get(number);
|
|
101
|
+
if (cached) {
|
|
102
|
+
this.pages.delete(number);
|
|
103
|
+
this.pages.set(number, cached);
|
|
104
|
+
return cached;
|
|
105
|
+
}
|
|
106
|
+
let reusable;
|
|
107
|
+
if (this.pages.size === this.maxPages) {
|
|
108
|
+
if (!this.descriptor)
|
|
109
|
+
await this.spill();
|
|
110
|
+
const [oldNumber, oldPage] = this.pages.entries().next().value;
|
|
111
|
+
await this.flush(oldNumber, oldPage);
|
|
112
|
+
this.pages.delete(oldNumber);
|
|
113
|
+
reusable = oldPage;
|
|
114
|
+
}
|
|
115
|
+
// Flush owns the bytes until its awaited write completes. Reuse only then;
|
|
116
|
+
// zero new/unwritten ranges so the previous page cannot leak into them.
|
|
117
|
+
const page = reusable ?? { bytes: new Uint8Array(pageBytes), dirty: false };
|
|
118
|
+
if (reusable)
|
|
119
|
+
page.bytes.fill(0);
|
|
120
|
+
if (number * pageBytes < this.diskLength) {
|
|
121
|
+
let offset = 0;
|
|
122
|
+
while (offset < pageBytes) {
|
|
123
|
+
const count = await this.descriptor.read(page.bytes.subarray(offset), number * pageBytes + offset, { signal: this.signal });
|
|
124
|
+
this.signal.throwIfAborted();
|
|
125
|
+
if (!Number.isSafeInteger(count) || count <= 0 || count > pageBytes - offset)
|
|
126
|
+
throw new FsError("EIO");
|
|
127
|
+
offset += count;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
this.pages.set(number, page);
|
|
131
|
+
return page;
|
|
132
|
+
}
|
|
133
|
+
write(position, bytes) {
|
|
134
|
+
if (!Number.isSafeInteger(position) || position < 8 || position + bytes.length > this.end)
|
|
135
|
+
throw new RangeError("Invalid paged storage range");
|
|
136
|
+
return this.operation(async () => {
|
|
137
|
+
for (let offset = 0; offset < bytes.length;) {
|
|
138
|
+
const target = position + offset;
|
|
139
|
+
const page = await this.page(Math.floor(target / pageBytes));
|
|
140
|
+
const count = Math.min(bytes.length - offset, pageBytes - target % pageBytes);
|
|
141
|
+
page.bytes.set(bytes.subarray(offset, offset + count), target % pageBytes);
|
|
142
|
+
page.dirty = true;
|
|
143
|
+
offset += count;
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
async append(bytes) {
|
|
148
|
+
const position = this.allocate(bytes.length);
|
|
149
|
+
await this.write(position, bytes);
|
|
150
|
+
return position;
|
|
151
|
+
}
|
|
152
|
+
read(position, length) {
|
|
153
|
+
if (!Number.isSafeInteger(position) || position < 8 || !Number.isSafeInteger(length) || length < 0 || length > pageBytes || position + length > this.end)
|
|
154
|
+
throw new RangeError("Invalid paged storage range");
|
|
155
|
+
return this.operation(async () => {
|
|
156
|
+
const bytes = new Uint8Array(length);
|
|
157
|
+
for (let offset = 0; offset < length;) {
|
|
158
|
+
const source = position + offset;
|
|
159
|
+
const page = await this.page(Math.floor(source / pageBytes));
|
|
160
|
+
const count = Math.min(length - offset, pageBytes - source % pageBytes);
|
|
161
|
+
bytes.set(page.bytes.subarray(source % pageBytes, source % pageBytes + count), offset);
|
|
162
|
+
offset += count;
|
|
163
|
+
}
|
|
164
|
+
return bytes;
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
close() {
|
|
168
|
+
this.controller.abort(new FsError("ECANCELED"));
|
|
169
|
+
return this.closing ??= (async () => {
|
|
170
|
+
await this.active.catch(() => { });
|
|
171
|
+
this.pages.clear();
|
|
172
|
+
const descriptor = this.descriptor;
|
|
173
|
+
if (!descriptor)
|
|
174
|
+
return;
|
|
175
|
+
const cancellation = new FsError("ECANCELED");
|
|
176
|
+
try {
|
|
177
|
+
if (!this.unlinked) {
|
|
178
|
+
const expected = this.initial ?? await descriptor.stat();
|
|
179
|
+
await this.context.fs.removeFileConditional(this.path, { parent: this.parent, expected }).catch(error => {
|
|
180
|
+
if (!isFsError(error) || error.code !== "ENOENT")
|
|
181
|
+
throw error;
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
// Conditional descriptors must discard their private staged writes, not
|
|
187
|
+
// publish the scratch data when the command retires its handle.
|
|
188
|
+
await descriptor.close({ signal: AbortSignal.abort(cancellation) }).catch(error => {
|
|
189
|
+
if (error !== cancellation)
|
|
190
|
+
throw error;
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
})();
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
/** Fixed-depth radix index: both dense and sparse 64-bit keys use bounded RAM. */
|
|
197
|
+
export class IntegerTable {
|
|
198
|
+
storage;
|
|
199
|
+
maxEntries;
|
|
200
|
+
root = 0;
|
|
201
|
+
revision = 0;
|
|
202
|
+
branches = new Map();
|
|
203
|
+
cache = new Map();
|
|
204
|
+
constructor(storage, maxEntries = 4096) {
|
|
205
|
+
this.storage = storage;
|
|
206
|
+
this.maxEntries = maxEntries;
|
|
207
|
+
if (!Number.isSafeInteger(maxEntries) || maxEntries < 1)
|
|
208
|
+
throw new RangeError("Invalid integer table cache size");
|
|
209
|
+
}
|
|
210
|
+
async retain(key, value, dirty) {
|
|
211
|
+
if (this.cache.size === this.maxEntries && !this.cache.has(key)) {
|
|
212
|
+
const [oldKey, old] = this.cache.entries().next().value;
|
|
213
|
+
// Flush the bounded dirty window together, before the caller touches other
|
|
214
|
+
// stored data. One dirty leaf per insertion otherwise thrashes a page cache.
|
|
215
|
+
if (old.dirty)
|
|
216
|
+
for (const [key, entry] of this.cache)
|
|
217
|
+
if (entry.dirty) {
|
|
218
|
+
await this.persist(key, entry.value);
|
|
219
|
+
entry.dirty = false;
|
|
220
|
+
}
|
|
221
|
+
this.cache.delete(oldKey);
|
|
222
|
+
}
|
|
223
|
+
this.cache.delete(key);
|
|
224
|
+
this.cache.set(key, { value, dirty });
|
|
225
|
+
}
|
|
226
|
+
async get(key) {
|
|
227
|
+
const cached = this.cache.get(key);
|
|
228
|
+
if (cached) {
|
|
229
|
+
this.cache.delete(key);
|
|
230
|
+
this.cache.set(key, cached);
|
|
231
|
+
return cached.value;
|
|
232
|
+
}
|
|
233
|
+
const value = await this.lookup(key);
|
|
234
|
+
if (value !== undefined)
|
|
235
|
+
await this.retain(key, value, false);
|
|
236
|
+
return value;
|
|
237
|
+
}
|
|
238
|
+
async set(key, value) {
|
|
239
|
+
if (key < 0n || key > 0xffffffffffffffffn || value < 0n || value >= 0xffffffffffffffffn)
|
|
240
|
+
throw new RangeError("Invalid storage integer table value");
|
|
241
|
+
this.revision++;
|
|
242
|
+
await this.retain(key, value, true);
|
|
243
|
+
}
|
|
244
|
+
/** Ordered replay with a fixed 16-level radix stack (at most 2 KiB of
|
|
245
|
+
* owned node bytes) and 128 output records. Batches avoid alternating sparse
|
|
246
|
+
* index reads with consumer writes on every record. Mutation is not permitted. */
|
|
247
|
+
async *entries(start = 0n, end = 0x10000000000000000n) {
|
|
248
|
+
if (start < 0n || start > end || end > 0x10000000000000000n)
|
|
249
|
+
throw new RangeError("Invalid integer table range");
|
|
250
|
+
if (start === end)
|
|
251
|
+
return;
|
|
252
|
+
const revision = this.revision;
|
|
253
|
+
const check = () => { if (revision !== this.revision)
|
|
254
|
+
throw new Error("Integer table changed during iteration"); };
|
|
255
|
+
for (const [key, entry] of this.cache)
|
|
256
|
+
if (entry.dirty) {
|
|
257
|
+
check();
|
|
258
|
+
await this.persist(key, entry.value);
|
|
259
|
+
check();
|
|
260
|
+
entry.dirty = false;
|
|
261
|
+
}
|
|
262
|
+
const storage = this.storage;
|
|
263
|
+
async function* visit(position, shift, prefix) {
|
|
264
|
+
check();
|
|
265
|
+
if (!position)
|
|
266
|
+
return;
|
|
267
|
+
const bytes = new Uint8Array(await storage.read(position, 128));
|
|
268
|
+
check();
|
|
269
|
+
if (bytes.length !== 128)
|
|
270
|
+
throw new RangeError("Truncated integer table node");
|
|
271
|
+
const node = new DataView(bytes.buffer);
|
|
272
|
+
for (let slot = 0; slot < 16; slot++) {
|
|
273
|
+
check();
|
|
274
|
+
const value = node.getBigUint64(slot * 8, true);
|
|
275
|
+
if (!value)
|
|
276
|
+
continue;
|
|
277
|
+
const key = prefix | BigInt(slot) << shift;
|
|
278
|
+
if (key >= end || key + (1n << shift) <= start)
|
|
279
|
+
continue;
|
|
280
|
+
if (shift === 0n)
|
|
281
|
+
yield [key, value - 1n];
|
|
282
|
+
else
|
|
283
|
+
yield* visit(Number(value), shift - 4n, key);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
const batch = [];
|
|
287
|
+
for await (const entry of visit(this.root, 60n, 0n)) {
|
|
288
|
+
batch.push(entry);
|
|
289
|
+
if (batch.length === 128) {
|
|
290
|
+
for (const entry of batch) {
|
|
291
|
+
check();
|
|
292
|
+
yield entry;
|
|
293
|
+
}
|
|
294
|
+
batch.length = 0;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
for (const entry of batch) {
|
|
298
|
+
check();
|
|
299
|
+
yield entry;
|
|
300
|
+
}
|
|
301
|
+
check();
|
|
302
|
+
}
|
|
303
|
+
/** Hot radix paths must not evict dirty leaf pages in a one-page backend. */
|
|
304
|
+
branch(key, shift, position) {
|
|
305
|
+
const prefix = key >> shift | 1n << (64n - shift);
|
|
306
|
+
const value = position ?? this.branches.get(prefix);
|
|
307
|
+
if (value !== undefined) {
|
|
308
|
+
this.branches.delete(prefix);
|
|
309
|
+
this.branches.set(prefix, value);
|
|
310
|
+
if (this.branches.size > 128)
|
|
311
|
+
this.branches.delete(this.branches.keys().next().value);
|
|
312
|
+
}
|
|
313
|
+
return value;
|
|
314
|
+
}
|
|
315
|
+
async lookup(key) {
|
|
316
|
+
let node = this.root;
|
|
317
|
+
for (let shift = 60n; shift >= 0n; shift -= 4n) {
|
|
318
|
+
if (!node)
|
|
319
|
+
return undefined;
|
|
320
|
+
const cached = shift === 0n ? undefined : this.branch(key, shift);
|
|
321
|
+
if (cached !== undefined) {
|
|
322
|
+
node = cached;
|
|
323
|
+
continue;
|
|
324
|
+
}
|
|
325
|
+
const slot = node + Number((key >> shift) & 15n) * 8;
|
|
326
|
+
const bytes = await this.storage.read(slot, 8);
|
|
327
|
+
const value = new DataView(bytes.buffer, bytes.byteOffset, 8).getBigUint64(0, true);
|
|
328
|
+
if (shift === 0n)
|
|
329
|
+
return value === 0n ? undefined : value - 1n;
|
|
330
|
+
node = Number(value);
|
|
331
|
+
if (node)
|
|
332
|
+
this.branch(key, shift, node);
|
|
333
|
+
}
|
|
334
|
+
return undefined;
|
|
335
|
+
}
|
|
336
|
+
async persist(key, value) {
|
|
337
|
+
if (!this.root)
|
|
338
|
+
this.root = this.storage.allocate(128);
|
|
339
|
+
let node = this.root;
|
|
340
|
+
for (let shift = 60n; shift >= 0n; shift -= 4n) {
|
|
341
|
+
const cached = shift === 0n ? undefined : this.branch(key, shift);
|
|
342
|
+
if (cached !== undefined) {
|
|
343
|
+
node = cached;
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
const slot = node + Number((key >> shift) & 15n) * 8;
|
|
347
|
+
const bytes = await this.storage.read(slot, 8);
|
|
348
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, 8);
|
|
349
|
+
if (shift === 0n) {
|
|
350
|
+
view.setBigUint64(0, value + 1n, true);
|
|
351
|
+
await this.storage.write(slot, bytes);
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
let child = Number(view.getBigUint64(0, true));
|
|
355
|
+
if (!child) {
|
|
356
|
+
child = this.storage.allocate(128);
|
|
357
|
+
view.setBigUint64(0, BigInt(child), true);
|
|
358
|
+
await this.storage.write(slot, bytes);
|
|
359
|
+
}
|
|
360
|
+
node = child;
|
|
361
|
+
this.branch(key, shift, child);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
}
|