@poe-platform/safe-fs 0.1.677 → 0.1.679

package/README.md

@@ -71,6 +71,16 @@ All required methods must exist, but a backend may reject an operation with `FsE
 
 `createDeviceFileSystem(fs)` adds portable `/dev/null` whole-file, stream, and descriptor I/O. Descriptor reads return EOF and writes discard bytes; stat remains a zero-size character device and descriptor position remains zero. Truncating opens are accepted, exclusive creation fails with `EEXIST`, and descriptor resizing and synchronization are unsupported. Access modes, cancellation, and closed handles use the normal descriptor checks. With an authoritative object store, use `createDeviceFileSystem(withObjectFileDescriptors(fs, store))` so null-device I/O never acquires or publishes an object version; ordinary files retain conditional publication. The device wrapper must be outermost for this composition.
 
+`withObjectFileDescriptors(fs, store)` supports large shell and Python descriptor
+writes when the host supplies `store.createStaging`: private externally backed
+pages keep working memory bounded without publishing the growing file after
+every write. Conditional publication still occurs at sync/close, and retained
+readers keep their old versions. Without that optional backend primitive, the
+default 8 MiB dirty-page budget still limits unflushed output. See the
+[object descriptor and spill contract](src/contracts/object-publication.md) for
+backend methods, failure semantics and qualification; no provider storage is
+configured automatically.
+
 Immutable flat stores can supply `identityScope`, `opaqueIdentity` and an ABA-safe
 `opaqueVersion` on file stats, then expose `atomicFilePublication: true` with
 `publishFileConditional(path, source, { expected, parent, maxBytes, signal })`.

package/dist/safe-fs/fs/object-publication/index.d.ts

@@ -13,9 +13,20 @@ export interface ObjectFilePublicationOptions extends FsOptions {
 export interface ObjectFileAcquireOptions extends FsOptions {
     readonly access: OpenFileOptions["access"];
 }
+export interface ObjectFileStaging {
+    readPage(index: number, options?: FsOptions): Promise<Uint8Array | undefined>;
+    writePage(index: number, bytes: Uint8Array, options?: FsOptions): Promise<void>;
+    truncate(size: number, options?: FsOptions): Promise<void>;
+    close(): Promise<void>;
+}
+export interface ObjectFileStagingOptions extends FsOptions {
+    readonly chunkBytes: number;
+    readonly maxFileBytes: number;
+}
 export interface ObjectFilePublicationStore {
     acquire(path: string, options: ObjectFileAcquireOptions): Promise<ObjectFileVersion | undefined>;
     publish?(path: string, expectedRevision: string | null, source: ByteSource, options: ObjectFilePublicationOptions): Promise<ObjectFileVersion>;
+    createStaging?(path: string, options: ObjectFileStagingOptions): Promise<ObjectFileStaging>;
 }
 export interface ObjectFileDescriptorOptions {
     readonly chunkBytes?: number;

package/dist/safe-fs/fs/object-publication/index.js

@@ -11,10 +11,43 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
     const maxOpenFiles = options.maxOpenFiles ?? 64;
     if (![chunkBytes, maxStagedBytes, maxStagedPages, maxFileBytes, maxOpenFiles].every(value => Number.isSafeInteger(value) && value > 0)
         || chunkBytes > 1048576 || typeof store.acquire !== "function"
-        || store.publish !== undefined && typeof store.publish !== "function")
+        || store.publish !== undefined && typeof store.publish !== "function"
+        || store.createStaging !== undefined && typeof store.createStaging !== "function")
         throw new TypeError("Invalid object descriptor configuration");
+    const createStaging = store.createStaging?.bind(store);
     let stagedBytes = 0;
     let openFiles = 0;
+    const stagingWaiters = new Set();
+    const reservePage = async (path, forwarded) => {
+        if (maxStagedBytes < chunkBytes)
+            throw new FsError("ENOSPC", { path, message: "Object staging requires at least one page of working memory" });
+        while (stagedBytes + chunkBytes > maxStagedBytes || stagedBytes / chunkBytes >= maxStagedPages) {
+            forwarded.signal?.throwIfAborted();
+            await new Promise((resolve, reject) => {
+                const wake = () => {
+                    stagingWaiters.delete(wake);
+                    forwarded.signal?.removeEventListener("abort", abort);
+                    resolve();
+                };
+                const abort = () => {
+                    stagingWaiters.delete(wake);
+                    forwarded.signal?.removeEventListener("abort", abort);
+                    reject(forwarded.signal?.reason);
+                };
+                stagingWaiters.add(wake);
+                forwarded.signal?.addEventListener("abort", abort, { once: true });
+                if (forwarded.signal?.aborted)
+                    abort();
+            });
+        }
+        forwarded.signal?.throwIfAborted();
+        stagedBytes += chunkBytes;
+    };
+    const releasePage = () => {
+        stagedBytes -= chunkBytes;
+        for (const wake of stagingWaiters)
+            wake();
+    };
     const version = (value) => {
         if (!value || typeof value.revision !== "string" || value.revision.length === 0 || value.revision.length > 4096
             || !value.stat || value.stat.type !== "file" || !Number.isSafeInteger(value.stat.size) || value.stat.size < 0
@@ -41,7 +74,7 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
             throw new FsError("EMFILE", { path });
         openFiles++;
         let acquiring = true;
-        const state = { head: undefined, position: 0, size: 0, inheritedSize: 0, modifiedAt: Date.now(), dirty: false, pages: new Map(), failure: undefined };
+        const state = { head: undefined, position: 0, size: 0, inheritedSize: 0, modifiedAt: Date.now(), dirty: false, pages: new Map(), staging: undefined, failure: undefined };
         const check = (forwarded) => {
             if (acquiring)
                 admitted.signal?.throwIfAborted();
@@ -74,6 +107,18 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
                 throw new FsError("EIO", { path, message: "Immutable range read returned an invalid byte count" });
             return Uint8Array.from(data);
         };
+        const readStagedPage = async (page, forwarded) => {
+            const bytes = await perform(forwarded, selected => state.staging.readPage(page, selected));
+            if (bytes !== undefined && (!(bytes instanceof Uint8Array) || bytes.byteLength !== chunkBytes))
+                throw new FsError("EIO", { path, message: "Invalid object staging page" });
+            return bytes;
+        };
+        const retireStaging = async () => {
+            const staging = state.staging;
+            state.staging = undefined;
+            if (typeof staging?.close === "function")
+                await staging.close();
+        };
         const read = async (buffer, position, forwarded) => {
             check(forwarded);
             const count = Math.min(buffer.byteLength, Math.max(0, state.size - position));
@@ -86,6 +131,21 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
                 const staged = state.pages.get(page);
                 if (staged)
                     buffer.set(staged.subarray(within, within + length), copied);
+                else if (state.staging) {
+                    await reservePage(path, forwarded);
+                    try {
+                        const spilled = await readStagedPage(page, forwarded);
+                        if (spilled)
+                            buffer.set(spilled.subarray(within, within + length), copied);
+                        else {
+                            buffer.fill(0, copied, copied + length);
+                            buffer.set(await readBase(offset, length, forwarded), copied);
+                        }
+                    }
+                    finally {
+                        releasePage();
+                    }
+                }
                 else {
                     buffer.fill(0, copied, copied + length);
                     buffer.set(await readBase(offset, length, forwarded), copied);
@@ -133,7 +193,14 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
                 state.modifiedAt = published.stat.mtimeMs;
                 state.dirty = false;
                 clearPages(state);
-                await previous?.close();
+                let retirementFailed = true;
+                try {
+                    await retireStaging();
+                    retirementFailed = false;
+                }
+                finally {
+                    await finishCleanup(() => previous?.close(), retirementFailed);
+                }
             }
             catch (reason) {
                 state.failure = { reason };
@@ -207,6 +274,51 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
                     const end = start + buffer.byteLength;
                     if (!Number.isSafeInteger(end) || end > maxFileBytes)
                         throw new FsError("EFBIG", { path, message: "Object descriptor file limit exceeded" });
+                    if (createStaging) {
+                        try {
+                            if (!state.staging) {
+                                await perform(forwarded, async (selected) => {
+                                    state.staging = await createStaging(path, { ...selected, chunkBytes, maxFileBytes });
+                                    if (!state.staging || ![state.staging.readPage, state.staging.writePage, state.staging.truncate, state.staging.close].every(method => typeof method === "function"))
+                                        throw new FsError("EIO", { path, message: "Invalid object staging handle" });
+                                });
+                            }
+                            let copied = 0;
+                            while (copied < buffer.byteLength) {
+                                await reservePage(path, forwarded);
+                                try {
+                                    const offset = start + copied;
+                                    const page = Math.floor(offset / chunkBytes);
+                                    const within = offset % chunkBytes;
+                                    const length = Math.min(buffer.byteLength - copied, chunkBytes - within);
+                                    let bytes = within === 0 && length === chunkBytes ? undefined : await readStagedPage(page, forwarded);
+                                    if (!bytes) {
+                                        bytes = new Uint8Array(chunkBytes);
+                                        if (within !== 0 || length !== chunkBytes)
+                                            bytes.set(await readBase(page * chunkBytes, chunkBytes, forwarded));
+                                    }
+                                    bytes.set(buffer.subarray(copied, copied + length), within);
+                                    await perform(forwarded, selected => state.staging.writePage(page, bytes, selected));
+                                    copied += length;
+                                }
+                                finally {
+                                    releasePage();
+                                }
+                            }
+                        }
+                        catch (reason) {
+                            state.failure = { reason };
+                            throw reason;
+                        }
+                        state.size = Math.max(state.size, end);
+                        state.modifiedAt = Date.now();
+                        state.dirty = true;
+                        if (position === null)
+                            state.position = end;
+                        if (admitted.synchronization !== undefined)
+                            await flush(forwarded);
+                        return buffer.byteLength;
+                    }
                     const first = Math.floor(start / chunkBytes);
                     const last = Math.floor((end - 1) / chunkBytes);
                     const missing = [];
@@ -254,6 +366,15 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
                     check(forwarded);
                     if (length > maxFileBytes)
                         throw new FsError("EFBIG", { path });
+                    if (state.staging) {
+                        try {
+                            await perform(forwarded, selected => state.staging.truncate(length, selected));
+                        }
+                        catch (reason) {
+                            state.failure = { reason };
+                            throw reason;
+                        }
+                    }
                     for (const [page, bytes] of state.pages) {
                         if (page * chunkBytes >= length) {
                             state.pages.delete(page);
@@ -284,11 +405,18 @@ export function withObjectFileDescriptors(filesystem, store, options = {}) {
                         clearPages(state);
                         const retained = state.head;
                         state.head = undefined;
+                        let retirementFailed = true;
                         try {
-                            await finishCleanup(() => retained?.close(), failed);
+                            await finishCleanup(retireStaging, failed);
+                            retirementFailed = false;
                         }
                         finally {
-                            openFiles--;
+                            try {
+                                await finishCleanup(() => retained?.close(), failed || retirementFailed);
+                            }
+                            finally {
+                                openFiles--;
+                            }
                         }
                     }
                 },

package/dist/safe-fs/testing/object-publication.d.ts

@@ -8,6 +8,7 @@ export interface ObjectFilePublicationConformanceFixture {
 }
 export interface ObjectFilePublicationConformanceOptions {
     readonly createFixture: () => ObjectFilePublicationConformanceFixture | Promise<ObjectFilePublicationConformanceFixture>;
+    readonly requireStaging?: boolean;
 }
 export interface ObjectFilePublicationConformanceCase {
     readonly name: string;

package/dist/safe-fs/testing/object-publication.js

@@ -106,6 +106,68 @@ export function createObjectFilePublicationConformanceCases(options) {
             },
         },
     ];
+    if (options.requireStaging)
+        cases.push({
+            name: "object staging: private pages have owned reads and truncation semantics",
+            async run({ fixture, path, track }) {
+                const name = path("staged");
+                const staging = track(await fixture.store.createStaging(name, { chunkBytes: 4, maxFileBytes: 16 }));
+                const sibling = track(await fixture.store.createStaging(name, { chunkBytes: 4, maxFileBytes: 16 }));
+                check(await staging.readPage(0) === undefined, "new staging contains an unowned page");
+                const input = new Uint8Array([1, 2, 3, 4]);
+                await staging.writePage(0, input);
+                input.fill(0);
+                const retained = await staging.readPage(0);
+                check(retained !== undefined, "staging lost an acknowledged page");
+                bytes(retained, new Uint8Array([1, 2, 3, 4]));
+                retained.fill(0);
+                bytes((await staging.readPage(0)), new Uint8Array([1, 2, 3, 4]));
+                check(await sibling.readPage(0) === undefined, "staging leaked across descriptors");
+                await staging.writePage(2, new Uint8Array([5, 6, 7, 8]));
+                await staging.truncate(2);
+                bytes((await staging.readPage(0)), new Uint8Array([1, 2, 0, 0]));
+                check(await staging.readPage(2) === undefined, "truncate retained a removed page");
+                const visible = await fixture.store.acquire(name, { access: "read" });
+                if (visible)
+                    track(visible);
+                check(visible === undefined, "private staging published a namespace entry");
+            },
+        }, {
+            name: "object staging: cancelled writes preserve acknowledged pages",
+            async run({ fixture, path, track }) {
+                const staging = track(await fixture.store.createStaging(path("stage-cancelled"), { chunkBytes: 4, maxFileBytes: 16 }));
+                await staging.writePage(0, new Uint8Array([1, 2, 3, 4]));
+                const controller = new AbortController();
+                const reason = new Error("staging cancellation");
+                controller.abort(reason);
+                let rejected = false;
+                try {
+                    await staging.writePage(0, new Uint8Array(4), { signal: controller.signal });
+                }
+                catch (error) {
+                    check(error === reason, "staging must preserve cancellation identity");
+                    rejected = true;
+                }
+                check(rejected, "cancelled staging write succeeded");
+                bytes((await staging.readPage(0)), new Uint8Array([1, 2, 3, 4]));
+            },
+        }, {
+            name: "object staging: writes larger than memory stay private until conditional sync",
+            async run({ fixture, path, track, publish }) {
+                const name = path("stage-descriptor");
+                await publish(name, null, new Uint8Array([1, 2, 3, 4]));
+                const fs = withObjectFileDescriptors(fixture.fs, fixture.store, { chunkBytes: 4, maxStagedBytes: 4, maxStagedPages: 1, maxFileBytes: 16 });
+                const descriptor = track(await fs.open(name, { access: "readwrite" }));
+                const content = new Uint8Array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]);
+                await descriptor.write(content, 0);
+                bytes(await fixture.fs.readFile(name), new Uint8Array([1, 2, 3, 4]));
+                const buffer = new Uint8Array(content.length);
+                check(await descriptor.read(buffer, 0) === buffer.length, "staging read was incomplete");
+                bytes(buffer, content);
+                await descriptor.sync(false);
+                bytes(await fixture.fs.readFile(name), content);
+            },
+        });
     return cases.map(entry => ({
         name: entry.name,
         async run() {
@@ -116,6 +178,8 @@ export function createObjectFilePublicationConformanceCases(options) {
                 validatePath(fixture.root);
                 check(fixture.root.startsWith("/"), "conformance fixture root must be absolute");
                 check(typeof fixture.store.publish === "function", "conformance requires authoritative conditional publication");
+                if (options.requireStaging)
+                    check(typeof fixture.store.createStaging === "function", "conformance requires private object staging");
                 const root = fixture.root.endsWith("/") ? fixture.root.slice(0, -1) : fixture.root;
                 const track = (value) => {
                     resources.push(value);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@poe-platform/safe-fs",
-  "version": "0.1.677",
+  "version": "0.1.679",
   "description": "Composable filesystem with a portable core and explicit Node adapters",
   "type": "module",
   "license": "MIT",