@poe-platform/safe-fs 0.1.826 → 0.1.827
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/safe-fs/fs/memory/ledger.d.ts +1 -0
- package/dist/safe-fs/fs/memory/ledger.js +12 -18
- package/dist/safe-fs/fs/object-publication/index.d.ts +1 -0
- package/dist/safe-fs/fs/object-publication/index.js +24 -19
- package/dist/safe-fs/fs/scoped.d.ts +1 -0
- package/dist/safe-fs/fs/scoped.js +16 -19
- package/dist/safe-fs/python/directory.d.ts +9 -0
- package/dist/safe-fs/python/directory.js +38 -0
- package/dist/safe-fs/python/emscripten.js +14 -1
- package/dist/safe-fs/python/filesystem.d.ts +2 -2
- package/dist/safe-fs/python/filesystem.js +45 -30
- package/dist/safe-fs/python/native.js +1 -1
- package/dist/safe-fs/python/request.js +1 -0
- package/dist/safe-fs/storage.d.ts +3 -1
- package/dist/safe-fs/storage.js +26 -20
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -14,7 +14,10 @@ To bound aggregate pages across multiple live stores, pass the same
|
|
|
14
14
|
`new PagedStorageCache(16)` as the third argument to each `PagedStorage` constructor.
|
|
15
15
|
This shares a 256 KiB resident-page budget; `cache.residentBytes` reports current
|
|
16
16
|
page retention. Eviction uses each store's filesystem, and each store still owns
|
|
17
|
-
its `close()` cleanup.
|
|
17
|
+
its `close()` cleanup. Use `new PagedStorageCache(16, 65536)` to coalesce scratch
|
|
18
|
+
I/O into 64 KiB pages within a 1 MiB budget, for example with object-backed files.
|
|
19
|
+
Page sizes must be multiples of 16 KiB, up to 1 MiB; individual `read()` calls
|
|
20
|
+
remain limited to 16 KiB. The budget excludes caller buffers and backend storage.
|
|
18
21
|
Call `await storage.prepare()` before opening a directory iterator when its
|
|
19
22
|
snapshot includes the scratch directory. It acquires and unlinks the backing
|
|
20
23
|
handle up front, so later spills do not mutate that directory's namespace.
|
|
@@ -119,6 +122,8 @@ identities without a default device-count budget. Their optional `maxDevices` ac
|
|
|
119
122
|
`Infinity` or a positive safe integer. Explicit budgets remain enforced; device numbers
|
|
120
123
|
must still fit the disjoint application/runtime ranges of the uint32 Python stat ABI.
|
|
121
124
|
|
|
125
|
+
Python filesystem directory cursors use `iterateDirectory` when available, share open-handle admission with files, and drain pending reads before cleanup. Backends without lazy enumeration retain the buffered `readdir` fallback.
|
|
126
|
+
|
|
122
127
|
`withObjectFileDescriptors(fs, store)` supports large shell and Python descriptor
|
|
123
128
|
writes when the host supplies `store.createStaging`: private externally backed
|
|
124
129
|
pages keep working memory bounded without publishing the growing file after
|
|
@@ -12,6 +12,7 @@ export declare class MemoryLedger {
|
|
|
12
12
|
get availableBytes(): number;
|
|
13
13
|
canPreallocate64(nameBytes: number): boolean;
|
|
14
14
|
fileSize(length: number, syscall: string, path: string): void;
|
|
15
|
+
private canReserve;
|
|
15
16
|
check(bytes: number, units: number, syscall: string, path: string): void;
|
|
16
17
|
reserve(bytes: number, units: number, syscall: string, path: string): void;
|
|
17
18
|
release(bytes: number, units: number): void;
|
|
@@ -10,9 +10,9 @@ export class MemoryLedger {
|
|
|
10
10
|
hasInfiniteRetained;
|
|
11
11
|
constructor(limits) {
|
|
12
12
|
this.limits = limits;
|
|
13
|
-
this.maxFileBytesSmi = limits.maxFileBytes
|
|
14
|
-
this.maxRetainedBytesSmi = limits.maxRetainedBytes
|
|
15
|
-
this.maxMetadataUnitsSmi = limits.maxMetadataUnits
|
|
13
|
+
this.maxFileBytesSmi = Math.min(limits.maxFileBytes, MAX_SMI) | 0;
|
|
14
|
+
this.maxRetainedBytesSmi = Math.min(limits.maxRetainedBytes, MAX_SMI) | 0;
|
|
15
|
+
this.maxMetadataUnitsSmi = Math.min(limits.maxMetadataUnits, MAX_SMI) | 0;
|
|
16
16
|
this.hasInfiniteRetained = limits.maxRetainedBytes === Infinity;
|
|
17
17
|
if (limits.maxFileBytes !== Infinity)
|
|
18
18
|
this.hasInfiniteFileBytes = false;
|
|
@@ -31,27 +31,24 @@ export class MemoryLedger {
|
|
|
31
31
|
throw new FsError("EFBIG", { syscall, path });
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
canReserve(bytes, units) {
|
|
35
|
+
return ((bytes | 0) === bytes && bytes >= 0 &&
|
|
36
36
|
(units | 0) === units && units >= 0 &&
|
|
37
37
|
this.retainedBytes <= this.maxRetainedBytesSmi - bytes &&
|
|
38
|
-
this.metadataUnits <= this.maxMetadataUnitsSmi - units)
|
|
38
|
+
this.metadataUnits <= this.maxMetadataUnitsSmi - units);
|
|
39
|
+
}
|
|
40
|
+
check(bytes, units, syscall, path) {
|
|
41
|
+
if (this.canReserve(bytes, units))
|
|
39
42
|
return;
|
|
40
|
-
}
|
|
41
|
-
const maxRetained = this.limits.maxRetainedBytes;
|
|
42
|
-
const maxUnits = this.limits.maxMetadataUnits;
|
|
43
43
|
if (!Number.isSafeInteger(bytes) || bytes < 0 ||
|
|
44
|
-
|
|
44
|
+
bytes > this.limits.maxRetainedBytes - this.retainedBytes ||
|
|
45
45
|
!Number.isSafeInteger(units) || units < 0 ||
|
|
46
|
-
|
|
46
|
+
units > this.limits.maxMetadataUnits - this.metadataUnits) {
|
|
47
47
|
throw new FsError("ENOSPC", { syscall, path });
|
|
48
48
|
}
|
|
49
49
|
}
|
|
50
50
|
reserve(bytes, units, syscall, path) {
|
|
51
|
-
if ((bytes
|
|
52
|
-
(units | 0) === units && units >= 0 &&
|
|
53
|
-
this.retainedBytes <= this.maxRetainedBytesSmi - bytes &&
|
|
54
|
-
this.metadataUnits <= this.maxMetadataUnitsSmi - units) {
|
|
51
|
+
if (this.canReserve(bytes, units)) {
|
|
55
52
|
this.retainedBytes = (this.retainedBytes + bytes) | 0;
|
|
56
53
|
this.metadataUnits = (this.metadataUnits + units) | 0;
|
|
57
54
|
return;
|
|
@@ -90,7 +87,4 @@ export class MemoryAllocation {
|
|
|
90
87
|
}
|
|
91
88
|
}
|
|
92
89
|
}
|
|
93
|
-
Object.assign(MemoryAllocation.prototype, {
|
|
94
|
-
references: 1,
|
|
95
|
-
});
|
|
96
90
|
Object.assign(MemoryLedger.prototype, { hasInfiniteFileBytes: true });
|
|
@@ -20,6 +20,7 @@ export interface ObjectFileAcquireOptions extends FsOptions {
|
|
|
20
20
|
}
|
|
21
21
|
export interface ObjectFileStaging {
|
|
22
22
|
readPage(index: number, options?: FsOptions): Promise<Uint8Array | undefined>;
|
|
23
|
+
/** Own acknowledged bytes; the caller may reuse its buffer after settlement. */
|
|
23
24
|
writePage(index: number, bytes: Uint8Array, options?: FsOptions): Promise<void>;
|
|
24
25
|
truncate(size: number, options?: FsOptions): Promise<void>;
|
|
25
26
|
close(): Promise<void>;
|
|
@@ -28,16 +28,12 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
|
|
|
28
28
|
while (stagedBytes + chunkBytes > maxStagedBytes || stagedBytes / chunkBytes >= maxStagedPages) {
|
|
29
29
|
forwarded.signal?.throwIfAborted();
|
|
30
30
|
await new Promise((resolve, reject) => {
|
|
31
|
-
const
|
|
31
|
+
const cleanup = () => {
|
|
32
32
|
stagingWaiters.delete(wake);
|
|
33
33
|
forwarded.signal?.removeEventListener("abort", abort);
|
|
34
|
-
resolve();
|
|
35
|
-
};
|
|
36
|
-
const abort = () => {
|
|
37
|
-
stagingWaiters.delete(wake);
|
|
38
|
-
forwarded.signal?.removeEventListener("abort", abort);
|
|
39
|
-
reject(forwarded.signal?.reason);
|
|
40
34
|
};
|
|
35
|
+
const wake = () => { cleanup(); resolve(); };
|
|
36
|
+
const abort = () => { cleanup(); reject(forwarded.signal?.reason); };
|
|
41
37
|
stagingWaiters.add(wake);
|
|
42
38
|
forwarded.signal?.addEventListener("abort", abort, { once: true });
|
|
43
39
|
if (forwarded.signal?.aborted)
|
|
@@ -87,9 +83,13 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
|
|
|
87
83
|
if (state.failure)
|
|
88
84
|
throw state.failure.reason;
|
|
89
85
|
};
|
|
86
|
+
const requirePublication = (writable = true) => {
|
|
87
|
+
if (!store.publish || !writable)
|
|
88
|
+
throw new FsError("ENOTSUP", { path, message: "Object writes require authoritative conditional publication" });
|
|
89
|
+
};
|
|
90
90
|
const perform = async (forwarded, action) => {
|
|
91
91
|
check(forwarded);
|
|
92
|
-
const scope = composeAbortSignals([
|
|
92
|
+
const scope = composeAbortSignals([acquiring && admitted.signal, forwarded.signal].filter((signal) => !!signal));
|
|
93
93
|
try {
|
|
94
94
|
const result = await action({ signal: scope.signal });
|
|
95
95
|
scope.signal.throwIfAborted();
|
|
@@ -163,8 +163,7 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
|
|
|
163
163
|
check(forwarded);
|
|
164
164
|
if (!state.dirty)
|
|
165
165
|
return;
|
|
166
|
-
|
|
167
|
-
throw new FsError("ENOTSUP", { path, message: "Object writes require authoritative conditional publication" });
|
|
166
|
+
requirePublication();
|
|
168
167
|
const expected = state.head?.revision ?? null;
|
|
169
168
|
let emitted = 0;
|
|
170
169
|
let active = true;
|
|
@@ -174,7 +173,8 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
|
|
|
174
173
|
let streamScope;
|
|
175
174
|
try {
|
|
176
175
|
await perform(forwarded, async (selected) => {
|
|
177
|
-
|
|
176
|
+
// perform always supplies its composed signal.
|
|
177
|
+
streamScope = composeAbortSignals([selected.signal, streamAbort.signal]);
|
|
178
178
|
const bodyOptions = { signal: streamScope.signal };
|
|
179
179
|
source = (async function* () {
|
|
180
180
|
for (let position = 0; position < state.size; position += chunkBytes) {
|
|
@@ -230,8 +230,8 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
|
|
|
230
230
|
const mutating = admitted.access !== "read" || admitted.creation !== "never" || admitted.truncate;
|
|
231
231
|
if (mutating && capabilities.readOnly === true)
|
|
232
232
|
throw new FsError("EROFS", { path });
|
|
233
|
-
if (mutating
|
|
234
|
-
|
|
233
|
+
if (mutating)
|
|
234
|
+
requirePublication(capabilities.write !== false);
|
|
235
235
|
if (admitted.access !== "write" && capabilities.read === false)
|
|
236
236
|
throw new FsError("EACCES", { path });
|
|
237
237
|
const acquired = await store.acquire(path, { access: admitted.access, ...(admitted.noFollow ? { noFollow: true } : {}), ...(admitted.signal ? { signal: admitted.signal } : {}) });
|
|
@@ -296,13 +296,18 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
|
|
|
296
296
|
const page = Math.floor(offset / chunkBytes);
|
|
297
297
|
const within = offset % chunkBytes;
|
|
298
298
|
const length = Math.min(buffer.byteLength - copied, chunkBytes - within);
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
299
|
+
// Staging owns acknowledged bytes. Borrow complete pages until
|
|
300
|
+
// settlement; only partial writes need read/modify/write storage.
|
|
301
|
+
let bytes = buffer.subarray(copied, copied + length);
|
|
302
|
+
if (within || length !== chunkBytes) {
|
|
303
|
+
let previous = await readStagedPage(page, forwarded);
|
|
304
|
+
if (!previous) {
|
|
305
|
+
previous = new Uint8Array(chunkBytes);
|
|
306
|
+
previous.set(await readBase(page * chunkBytes, chunkBytes, forwarded));
|
|
307
|
+
}
|
|
308
|
+
previous.set(bytes, within);
|
|
309
|
+
bytes = previous;
|
|
304
310
|
}
|
|
305
|
-
bytes.set(buffer.subarray(copied, copied + length), within);
|
|
306
311
|
await perform(forwarded, selected => state.staging.writePage(page, bytes, selected));
|
|
307
312
|
copied += length;
|
|
308
313
|
}
|
|
@@ -7,6 +7,7 @@ export declare function scopeFileSystem(filesystem: FileSystem, charge: () => vo
|
|
|
7
7
|
}): FileSystem;
|
|
8
8
|
export interface RetainedFileSystemCleanupView {
|
|
9
9
|
readonly removeFileConditional?: NonNullable<FileSystem["removeFileConditional"]>;
|
|
10
|
+
readonly removeTreeConditional?: NonNullable<FileSystem["removeTreeConditional"]>;
|
|
10
11
|
readonly removeStagedFile?: NonNullable<FileSystem["removeStagedFile"]>;
|
|
11
12
|
readonly lstat: FileSystem["lstat"];
|
|
12
13
|
readonly realpath: FileSystem["realpath"];
|
|
@@ -82,16 +82,16 @@ export function scopeFileSystem(filesystem, charge, signal, cleanupCharge = char
|
|
|
82
82
|
close() { closed = true; return close(); },
|
|
83
83
|
};
|
|
84
84
|
};
|
|
85
|
-
let
|
|
86
|
-
let cachedCombinedSignal;
|
|
85
|
+
let combinedSignals = new WeakMap();
|
|
87
86
|
const combineSignal = (callerSignal) => {
|
|
88
87
|
if (!callerSignal || callerSignal === signal)
|
|
89
88
|
return signal;
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
89
|
+
let combined = combinedSignals.get(callerSignal);
|
|
90
|
+
if (!combined) {
|
|
91
|
+
combined = AbortSignal.any([signal, callerSignal]);
|
|
92
|
+
combinedSignals.set(callerSignal, combined);
|
|
93
|
+
}
|
|
94
|
+
return combined;
|
|
95
95
|
};
|
|
96
96
|
const resizeOptions = (options) => {
|
|
97
97
|
const scopedOptions = {
|
|
@@ -557,8 +557,7 @@ export function scopeFileSystem(filesystem, charge, signal, cleanupCharge = char
|
|
|
557
557
|
retargeters.set(view, (nextCharge, nextSignal, nextCleanupCharge, nextMaxPathComponents) => {
|
|
558
558
|
charge = nextCharge;
|
|
559
559
|
if (signal !== nextSignal) {
|
|
560
|
-
|
|
561
|
-
cachedCombinedSignal = undefined;
|
|
560
|
+
combinedSignals = new WeakMap();
|
|
562
561
|
}
|
|
563
562
|
signal = nextSignal;
|
|
564
563
|
cleanupCharge = nextCleanupCharge;
|
|
@@ -612,8 +611,8 @@ export function retainFileSystemCleanup(filesystem, cleanupCallback, options) {
|
|
|
612
611
|
if (typeof operation !== "function")
|
|
613
612
|
throw new FsError("ENOTSUP", { syscall: method, path });
|
|
614
613
|
const parameters = method === "rm" ? { ...settings, recursive: false, force: false } : settings;
|
|
615
|
-
result = method === "removeStagedFile" || method === "removeFileConditional"
|
|
616
|
-
? requireOwnedMutation(backing, path, method === "removeFileConditional" ? "atomicFileMutation" : "atomicFileStaging", settings ?? {}).then(() => Reflect.apply(operation, backing, [input, parameters]))
|
|
614
|
+
result = method === "removeStagedFile" || method === "removeFileConditional" || method === "removeTreeConditional"
|
|
615
|
+
? requireOwnedMutation(backing, path, method === "removeFileConditional" ? "atomicFileMutation" : method === "removeTreeConditional" ? "atomicTreeRemoval" : "atomicFileStaging", settings ?? {}).then(() => Reflect.apply(operation, backing, [input, parameters]))
|
|
617
616
|
: Promise.resolve(Reflect.apply(operation, backing, [input, parameters]));
|
|
618
617
|
}
|
|
619
618
|
catch (error) {
|
|
@@ -625,14 +624,12 @@ export function retainFileSystemCleanup(filesystem, cleanupCallback, options) {
|
|
|
625
624
|
pending.add(completion);
|
|
626
625
|
return result;
|
|
627
626
|
};
|
|
628
|
-
const view = Object.
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
...(typeof backing.rmdir === "function" ? { rmdir: invoke.bind(undefined, "rmdir") } : {}),
|
|
635
|
-
}));
|
|
627
|
+
const view = Object.create(null);
|
|
628
|
+
for (const method of ["removeFileConditional", "removeStagedFile", "removeTreeConditional", "lstat", "realpath", "rm", "rmdir"]) {
|
|
629
|
+
if (method === "lstat" || method === "realpath" || method === "rm" || typeof backing[method] === "function")
|
|
630
|
+
Object.defineProperty(view, method, { value: invoke.bind(undefined, method), enumerable: true });
|
|
631
|
+
}
|
|
632
|
+
Object.freeze(view);
|
|
636
633
|
scope?.signal.throwIfAborted();
|
|
637
634
|
return () => {
|
|
638
635
|
if (!closing) {
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { FileSystem, FsOptions } from '../contracts/filesystem.js';
|
|
2
|
+
/** One prefetched entry makes acquisition errors eager without buffering a listing. */
|
|
3
|
+
export declare class PythonDirectory {
|
|
4
|
+
#private;
|
|
5
|
+
constructor(fs: FileSystem, path: string, options: FsOptions, limit: number);
|
|
6
|
+
prepare(): Promise<void>;
|
|
7
|
+
next(): Promise<string | null>;
|
|
8
|
+
close(): Promise<void>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { FsError } from '../contracts/errors.js';
|
|
2
|
+
/** One prefetched entry makes acquisition errors eager without buffering a listing. */
|
|
3
|
+
export class PythonDirectory {
|
|
4
|
+
#first;
|
|
5
|
+
#tail = Promise.resolve();
|
|
6
|
+
#closing;
|
|
7
|
+
#count = 0;
|
|
8
|
+
#iterator;
|
|
9
|
+
#options;
|
|
10
|
+
#limit;
|
|
11
|
+
constructor(fs, path, options, limit) {
|
|
12
|
+
this.#options = options;
|
|
13
|
+
this.#limit = limit;
|
|
14
|
+
const source = fs.iterateDirectory ? fs.iterateDirectory(path, options) : (async function* () { yield* await fs.readdir(path, { ...options, ...limit === Infinity ? {} : { maxEntries: limit } }); })();
|
|
15
|
+
this.#iterator = source[Symbol.asyncIterator]();
|
|
16
|
+
}
|
|
17
|
+
async prepare() { this.#first = await this.#iterator.next(); }
|
|
18
|
+
next() {
|
|
19
|
+
if (this.#closing)
|
|
20
|
+
return Promise.reject(new FsError('EBADF'));
|
|
21
|
+
const work = this.#tail.then(async () => {
|
|
22
|
+
this.#options.signal?.throwIfAborted();
|
|
23
|
+
const value = this.#first ?? await this.#iterator.next();
|
|
24
|
+
this.#first = undefined;
|
|
25
|
+
this.#options.signal?.throwIfAborted();
|
|
26
|
+
if (value.done)
|
|
27
|
+
return null;
|
|
28
|
+
if (++this.#count > this.#limit)
|
|
29
|
+
throw new FsError('EFBIG', { syscall: 'readdir' });
|
|
30
|
+
return value.value.name;
|
|
31
|
+
});
|
|
32
|
+
this.#tail = work.catch(() => { });
|
|
33
|
+
return work;
|
|
34
|
+
}
|
|
35
|
+
close() {
|
|
36
|
+
return this.#closing ??= this.#tail.then(async () => { this.#first = undefined; await this.#iterator.return?.(); });
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -92,7 +92,20 @@ export function mountPythonFileSystem(FS, options) {
|
|
|
92
92
|
close: (stream) => { if (stream.handle && --stream.retained.references === 0)
|
|
93
93
|
rpc('close', stream.handle); },
|
|
94
94
|
fsync: (stream) => { rpc('sync', stream.handle, false); return 0; },
|
|
95
|
-
read(stream, buffer, offset, length, position) {
|
|
95
|
+
read(stream, buffer, offset, length, position) {
|
|
96
|
+
let received = 0;
|
|
97
|
+
while (received < length) {
|
|
98
|
+
const requested = Math.min(length - received, maxTransferBytes);
|
|
99
|
+
const data = rpc('read', stream.handle, requested, stream.seekable ? position + received : null);
|
|
100
|
+
if (data.length > requested)
|
|
101
|
+
throw new FS.ErrnoError(errno.EIO);
|
|
102
|
+
buffer.set(data, offset + received);
|
|
103
|
+
received += data.length;
|
|
104
|
+
if (data.length < requested || !stream.seekable)
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
107
|
+
return received;
|
|
108
|
+
},
|
|
96
109
|
write: (stream, buffer, offset, length, position) => rpc('write', stream.handle, Uint8Array.from(buffer.subarray(offset, offset + Math.min(length, maxTransferBytes))), stream.pythonExplicitPosition ?? (stream.flags & 1024 || !stream.seekable ? null : position)),
|
|
97
110
|
llseek(stream, offset, whence) { if (![0, 1, 2].includes(whence) || !Number.isSafeInteger(offset))
|
|
98
111
|
throw new FS.ErrnoError(errno.EINVAL); const position = whence === 0 ? offset : whence === 1 ? stream.position + offset : rpc('fstat', stream.handle).size + offset; if (!Number.isSafeInteger(position) || position < 0)
|
|
@@ -9,7 +9,7 @@ export type PythonFsRequest = {
|
|
|
9
9
|
readonly op: "write";
|
|
10
10
|
readonly args: readonly [number, Uint8Array, number | null];
|
|
11
11
|
} | {
|
|
12
|
-
readonly op: "fstat" | "close" | "position" | "descriptorCapabilities";
|
|
12
|
+
readonly op: "directoryNext" | "fstat" | "close" | "position" | "descriptorCapabilities";
|
|
13
13
|
readonly args: readonly [number];
|
|
14
14
|
} | {
|
|
15
15
|
readonly op: "ftruncate";
|
|
@@ -18,7 +18,7 @@ export type PythonFsRequest = {
|
|
|
18
18
|
readonly op: "sync";
|
|
19
19
|
readonly args: readonly [number, boolean];
|
|
20
20
|
} | {
|
|
21
|
-
readonly op: "stat" | "lstat" | "readdir" | "realpath" | "readlink" | "rm" | "rmdir" | "rmtree" | "rmtreeSupported";
|
|
21
|
+
readonly op: "directoryOpen" | "stat" | "lstat" | "readdir" | "realpath" | "readlink" | "rm" | "rmdir" | "rmtree" | "rmtreeSupported";
|
|
22
22
|
readonly args: readonly [string];
|
|
23
23
|
} | {
|
|
24
24
|
readonly op: "rename" | "symlink" | "link";
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { PythonDirectory } from "./directory.js";
|
|
1
2
|
import { parsePythonFsRequest } from "./request.js";
|
|
2
3
|
import { composeAbortSignals } from "../contracts/abort.js";
|
|
3
4
|
import { FsError } from "../contracts/errors.js";
|
|
@@ -57,10 +58,37 @@ export class PythonFileSystem {
|
|
|
57
58
|
}
|
|
58
59
|
#handle(id) {
|
|
59
60
|
const handle = this.#handles.get(id);
|
|
60
|
-
if (!handle)
|
|
61
|
+
if (!handle || handle instanceof PythonDirectory)
|
|
61
62
|
throw new FsError("EBADF");
|
|
62
63
|
return handle;
|
|
63
64
|
}
|
|
65
|
+
async #acquire(open) {
|
|
66
|
+
if (this.#handles.size + this.#acquiring >= this.#limit || !Number.isSafeInteger(this.#next))
|
|
67
|
+
throw new FsError("EMFILE", { syscall: "open" });
|
|
68
|
+
this.#acquiring++;
|
|
69
|
+
let handle;
|
|
70
|
+
try {
|
|
71
|
+
handle = await open();
|
|
72
|
+
if (handle instanceof PythonDirectory)
|
|
73
|
+
await handle.prepare();
|
|
74
|
+
this.#scope.signal.throwIfAborted();
|
|
75
|
+
const id = this.#next++;
|
|
76
|
+
this.#handles.set(id, handle);
|
|
77
|
+
return id;
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
if (handle)
|
|
81
|
+
try {
|
|
82
|
+
await handle.close();
|
|
83
|
+
}
|
|
84
|
+
catch { /* Preserve acquisition failure. */ }
|
|
85
|
+
this.#scope.signal.throwIfAborted();
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
this.#acquiring--;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
64
92
|
#size(size) {
|
|
65
93
|
if (!Number.isSafeInteger(size) || size < 0)
|
|
66
94
|
throw new FsError("EINVAL");
|
|
@@ -70,7 +98,9 @@ export class PythonFileSystem {
|
|
|
70
98
|
async #execute(request) {
|
|
71
99
|
if (request.op === "close") {
|
|
72
100
|
const [id] = request.args;
|
|
73
|
-
const handle = this.#
|
|
101
|
+
const handle = this.#handles.get(id);
|
|
102
|
+
if (!handle)
|
|
103
|
+
throw new FsError("EBADF");
|
|
74
104
|
this.#handles.delete(id);
|
|
75
105
|
await handle.close({ signal: this.#scope.signal });
|
|
76
106
|
return;
|
|
@@ -82,40 +112,25 @@ export class PythonFileSystem {
|
|
|
82
112
|
switch (request.op) {
|
|
83
113
|
case "open": {
|
|
84
114
|
const [path, supplied] = request.args;
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
this.#acquiring++;
|
|
90
|
-
let handle;
|
|
91
|
-
try {
|
|
92
|
-
// A following capability query cannot preflight exclusive final-entry acquisition.
|
|
93
|
-
// The canonical open remains authoritative for selected-path policy and atomic refusal.
|
|
115
|
+
return this.#acquire(async () => {
|
|
116
|
+
if (!this.#open && !fs.open)
|
|
117
|
+
throw new FsError("ENOTSUP", { syscall: "open", path });
|
|
118
|
+
// Exclusive creation must reach the canonical atomic operation directly.
|
|
94
119
|
const capabilities = supplied.creation !== "exclusive" && fs.capabilitiesFor ? await fs.capabilitiesFor(this.#path(path), { signal, ...(supplied.creation === "ifMissing" ? { create: true } : {}) }) : fs.capabilities;
|
|
95
120
|
if (capabilities.readOnly === true && (supplied.access === "write" || supplied.access === "readwrite" || supplied.creation === "ifMissing" || supplied.creation === "exclusive" || supplied.truncate === true || supplied.append === true))
|
|
96
121
|
throw new FsError("EROFS", { syscall: "open", path });
|
|
97
122
|
if (capabilities.open === false)
|
|
98
123
|
throw new FsError("ENOTSUP", { syscall: "open", path });
|
|
99
124
|
signal.throwIfAborted();
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
try {
|
|
110
|
-
await handle.close();
|
|
111
|
-
}
|
|
112
|
-
catch { /* Preserve the acquisition failure. */ }
|
|
113
|
-
signal.throwIfAborted();
|
|
114
|
-
throw error;
|
|
115
|
-
}
|
|
116
|
-
finally {
|
|
117
|
-
this.#acquiring--;
|
|
118
|
-
}
|
|
125
|
+
return (this.#open ?? fs.open.bind(fs))(this.#path(path), { ...supplied, signal });
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
case "directoryOpen": return this.#acquire(() => new PythonDirectory(fs, this.#path(request.args[0]), options, this.#directoryLimit));
|
|
129
|
+
case "directoryNext": {
|
|
130
|
+
const handle = this.#handles.get(request.args[0]);
|
|
131
|
+
if (!(handle instanceof PythonDirectory))
|
|
132
|
+
throw new FsError("EBADF");
|
|
133
|
+
return handle.next();
|
|
119
134
|
}
|
|
120
135
|
case "descriptorCapabilities": return { ...this.#handle(request.args[0]).capabilities };
|
|
121
136
|
case "fstat": return this.#handle(request.args[0]).stat(options);
|
|
@@ -165,7 +165,7 @@ export function createPythonNativeSyscalls(options) {
|
|
|
165
165
|
if (entry && !positioned)
|
|
166
166
|
entry.position = writing && entry.flags & 1024
|
|
167
167
|
? await request('position', entry.handle) : entry.position + transferred;
|
|
168
|
-
if (transferred < size || !writing) {
|
|
168
|
+
if (transferred < size || !writing && !entry?.seekable) {
|
|
169
169
|
view().setUint32(output, total, true);
|
|
170
170
|
return 0;
|
|
171
171
|
}
|
|
@@ -2,6 +2,7 @@ import { FsError } from "../contracts/errors.js";
|
|
|
2
2
|
const argumentKinds = {
|
|
3
3
|
open: ["string", "object"], read: ["number", "number", "position"], write: ["number", "bytes", "position"],
|
|
4
4
|
descriptorCapabilities: ["number"], fstat: ["number"], close: ["number"], position: ["number"], ftruncate: ["number", "number"], sync: ["number", "boolean"],
|
|
5
|
+
directoryOpen: ["string"], directoryNext: ["number"],
|
|
5
6
|
stat: ["string"], lstat: ["string"], readdir: ["string"], realpath: ["string"], readlink: ["string"], rm: ["string"], rmdir: ["string"], rmtree: ["string"], rmtreeSupported: ["string"],
|
|
6
7
|
rename: ["string", "string"], symlink: ["string", "string"], link: ["string", "string"], mkdir: ["string", "optionalObject"],
|
|
7
8
|
chmod: ["string", "number"], truncate: ["string", "number"], access: ["string", "number"], utimes: ["string", "number", "number"],
|
|
@@ -13,9 +13,10 @@ type Page = {
|
|
|
13
13
|
* filesystem and cleanup ownership. Serializes cache users through eviction IO. */
|
|
14
14
|
export declare class PagedStorageCache {
|
|
15
15
|
private readonly maxPages;
|
|
16
|
+
readonly pageBytes: number;
|
|
16
17
|
private readonly pages;
|
|
17
18
|
private active;
|
|
18
|
-
constructor(maxPages: number);
|
|
19
|
+
constructor(maxPages: number, pageBytes?: number);
|
|
19
20
|
get residentBytes(): number;
|
|
20
21
|
run<T>(action: () => Promise<T>): Promise<T>;
|
|
21
22
|
acquire(): Promise<Page>;
|
|
@@ -31,6 +32,7 @@ export declare class PagedStorage {
|
|
|
31
32
|
private readonly maxPages;
|
|
32
33
|
private readonly cache?;
|
|
33
34
|
private readonly pages;
|
|
35
|
+
private readonly pageBytes;
|
|
34
36
|
private readonly controller;
|
|
35
37
|
private readonly signal;
|
|
36
38
|
private descriptor;
|
package/dist/safe-fs/storage.js
CHANGED
|
@@ -1,18 +1,22 @@
|
|
|
1
1
|
import { FsError, isFsError } from "./contracts/errors.js";
|
|
2
|
-
const
|
|
2
|
+
const defaultPageBytes = 16 * 1024;
|
|
3
3
|
let serial = 0;
|
|
4
4
|
/** Shares one resident-page limit across stores, retaining each store's caller
|
|
5
5
|
* filesystem and cleanup ownership. Serializes cache users through eviction IO. */
|
|
6
6
|
export class PagedStorageCache {
|
|
7
7
|
maxPages;
|
|
8
|
+
pageBytes;
|
|
8
9
|
pages = new Map();
|
|
9
10
|
active = Promise.resolve();
|
|
10
|
-
constructor(maxPages) {
|
|
11
|
+
constructor(maxPages, pageBytes = defaultPageBytes) {
|
|
11
12
|
this.maxPages = maxPages;
|
|
13
|
+
this.pageBytes = pageBytes;
|
|
14
|
+
if (!Number.isSafeInteger(pageBytes) || pageBytes < defaultPageBytes || pageBytes > 1024 * 1024 || pageBytes % defaultPageBytes !== 0)
|
|
15
|
+
throw new RangeError("Invalid storage page size");
|
|
12
16
|
if (!Number.isSafeInteger(maxPages) || maxPages < 1)
|
|
13
17
|
throw new RangeError("Invalid storage page cache size");
|
|
14
18
|
}
|
|
15
|
-
get residentBytes() { return this.pages.size * pageBytes; }
|
|
19
|
+
get residentBytes() { return this.pages.size * this.pageBytes; }
|
|
16
20
|
run(action) {
|
|
17
21
|
const work = this.active.then(action);
|
|
18
22
|
this.active = work.then(() => undefined, () => undefined);
|
|
@@ -20,7 +24,7 @@ export class PagedStorageCache {
|
|
|
20
24
|
}
|
|
21
25
|
async acquire() {
|
|
22
26
|
if (this.pages.size < this.maxPages)
|
|
23
|
-
return { bytes: new Uint8Array(pageBytes), dirty: false };
|
|
27
|
+
return { bytes: new Uint8Array(this.pageBytes), dirty: false };
|
|
24
28
|
const [page, release] = this.pages.entries().next().value;
|
|
25
29
|
await release();
|
|
26
30
|
this.pages.delete(page);
|
|
@@ -46,6 +50,7 @@ export class PagedStorage {
|
|
|
46
50
|
maxPages;
|
|
47
51
|
cache;
|
|
48
52
|
pages = new Map();
|
|
53
|
+
pageBytes;
|
|
49
54
|
controller = new AbortController();
|
|
50
55
|
signal;
|
|
51
56
|
descriptor;
|
|
@@ -63,6 +68,7 @@ export class PagedStorage {
|
|
|
63
68
|
this.cache = cache;
|
|
64
69
|
if (!Number.isSafeInteger(maxPages) || maxPages < 1)
|
|
65
70
|
throw new RangeError("Invalid storage page cache size");
|
|
71
|
+
this.pageBytes = cache?.pageBytes ?? defaultPageBytes;
|
|
66
72
|
this.signal = AbortSignal.any([context.signal, this.controller.signal]);
|
|
67
73
|
}
|
|
68
74
|
allocate(length) {
|
|
@@ -129,15 +135,15 @@ export class PagedStorage {
|
|
|
129
135
|
if (!page.dirty)
|
|
130
136
|
return;
|
|
131
137
|
let offset = 0;
|
|
132
|
-
while (offset < pageBytes) {
|
|
138
|
+
while (offset < this.pageBytes) {
|
|
133
139
|
this.signal.throwIfAborted();
|
|
134
|
-
const count = await this.descriptor.write(page.bytes.subarray(offset), number * pageBytes + offset, { signal: this.signal });
|
|
140
|
+
const count = await this.descriptor.write(page.bytes.subarray(offset), number * this.pageBytes + offset, { signal: this.signal });
|
|
135
141
|
this.signal.throwIfAborted();
|
|
136
|
-
if (!Number.isSafeInteger(count) || count <= 0 || count > pageBytes - offset)
|
|
142
|
+
if (!Number.isSafeInteger(count) || count <= 0 || count > this.pageBytes - offset)
|
|
137
143
|
throw new FsError("EIO");
|
|
138
144
|
offset += count;
|
|
139
145
|
}
|
|
140
|
-
this.diskLength = Math.max(this.diskLength, (number + 1) * pageBytes);
|
|
146
|
+
this.diskLength = Math.max(this.diskLength, (number + 1) * this.pageBytes);
|
|
141
147
|
page.dirty = false;
|
|
142
148
|
}
|
|
143
149
|
async page(number) {
|
|
@@ -161,17 +167,17 @@ export class PagedStorage {
|
|
|
161
167
|
}
|
|
162
168
|
// Flush owns the bytes until its awaited write completes. Reuse only then;
|
|
163
169
|
// zero new/unwritten ranges so the previous page cannot leak into them.
|
|
164
|
-
const page = reusable ?? (this.cache ? await this.cache.acquire() : { bytes: new Uint8Array(pageBytes), dirty: false });
|
|
170
|
+
const page = reusable ?? (this.cache ? await this.cache.acquire() : { bytes: new Uint8Array(this.pageBytes), dirty: false });
|
|
165
171
|
// Shared eviction may await IO owned by a different, still-live signal.
|
|
166
172
|
this.signal.throwIfAborted();
|
|
167
173
|
if (reusable)
|
|
168
174
|
page.bytes.fill(0);
|
|
169
|
-
if (number * pageBytes < this.diskLength) {
|
|
175
|
+
if (number * this.pageBytes < this.diskLength) {
|
|
170
176
|
let offset = 0;
|
|
171
|
-
while (offset < pageBytes) {
|
|
172
|
-
const count = await this.descriptor.read(page.bytes.subarray(offset), number * pageBytes + offset, { signal: this.signal });
|
|
177
|
+
while (offset < this.pageBytes) {
|
|
178
|
+
const count = await this.descriptor.read(page.bytes.subarray(offset), number * this.pageBytes + offset, { signal: this.signal });
|
|
173
179
|
this.signal.throwIfAborted();
|
|
174
|
-
if (!Number.isSafeInteger(count) || count <= 0 || count > pageBytes - offset)
|
|
180
|
+
if (!Number.isSafeInteger(count) || count <= 0 || count > this.pageBytes - offset)
|
|
175
181
|
throw new FsError("EIO");
|
|
176
182
|
offset += count;
|
|
177
183
|
}
|
|
@@ -191,9 +197,9 @@ export class PagedStorage {
|
|
|
191
197
|
return this.operation(async () => {
|
|
192
198
|
for (let offset = 0; offset < bytes.length;) {
|
|
193
199
|
const target = position + offset;
|
|
194
|
-
const page = await this.page(Math.floor(target / pageBytes));
|
|
195
|
-
const count = Math.min(bytes.length - offset, pageBytes - target % pageBytes);
|
|
196
|
-
page.bytes.set(bytes.subarray(offset, offset + count), target % pageBytes);
|
|
200
|
+
const page = await this.page(Math.floor(target / this.pageBytes));
|
|
201
|
+
const count = Math.min(bytes.length - offset, this.pageBytes - target % this.pageBytes);
|
|
202
|
+
page.bytes.set(bytes.subarray(offset, offset + count), target % this.pageBytes);
|
|
197
203
|
page.dirty = true;
|
|
198
204
|
offset += count;
|
|
199
205
|
}
|
|
@@ -205,15 +211,15 @@ export class PagedStorage {
|
|
|
205
211
|
return position;
|
|
206
212
|
}
|
|
207
213
|
read(position, length) {
|
|
208
|
-
if (!Number.isSafeInteger(position) || position < 8 || !Number.isSafeInteger(length) || length < 0 || length >
|
|
214
|
+
if (!Number.isSafeInteger(position) || position < 8 || !Number.isSafeInteger(length) || length < 0 || length > defaultPageBytes || position + length > this.end)
|
|
209
215
|
throw new RangeError("Invalid paged storage range");
|
|
210
216
|
return this.operation(async () => {
|
|
211
217
|
const bytes = new Uint8Array(length);
|
|
212
218
|
for (let offset = 0; offset < length;) {
|
|
213
219
|
const source = position + offset;
|
|
214
|
-
const page = await this.page(Math.floor(source / pageBytes));
|
|
215
|
-
const count = Math.min(length - offset, pageBytes - source % pageBytes);
|
|
216
|
-
bytes.set(page.bytes.subarray(source % pageBytes, source % pageBytes + count), offset);
|
|
220
|
+
const page = await this.page(Math.floor(source / this.pageBytes));
|
|
221
|
+
const count = Math.min(length - offset, this.pageBytes - source % this.pageBytes);
|
|
222
|
+
bytes.set(page.bytes.subarray(source % this.pageBytes, source % this.pageBytes + count), offset);
|
|
217
223
|
offset += count;
|
|
218
224
|
}
|
|
219
225
|
return bytes;
|