@poe-platform/safe-fs 0.1.660 → 0.1.662
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/safe-fs/contracts/descriptor.d.ts +1 -0
- package/dist/safe-fs/contracts/filesystem.d.ts +1 -0
- package/dist/safe-fs/core.d.ts +1 -0
- package/dist/safe-fs/core.js +1 -0
- package/dist/safe-fs/fs/capabilities.js +1 -1
- package/dist/safe-fs/fs/descriptor.js +2 -0
- package/dist/safe-fs/fs/mount/index.js +2 -2
- package/dist/safe-fs/fs/object-publication/index.d.ts +27 -0
- package/dist/safe-fs/fs/object-publication/index.js +324 -0
- package/dist/safe-fs/fs/quota/index.js +46 -7
- package/dist/safe-fs/index.d.ts +1 -0
- package/dist/safe-fs/index.js +1 -0
- package/dist/safe-fs/testing/object-publication.d.ts +16 -0
- package/dist/safe-fs/testing/object-publication.js +145 -0
- package/package.json +7 -1
|
@@ -8,6 +8,7 @@ export interface OpenFileOptions extends FsOptions {
|
|
|
8
8
|
readonly synchronization?: "data" | "all";
|
|
9
9
|
}
|
|
10
10
|
export interface FileDescriptorCapabilities {
|
|
11
|
+
readonly publication?: "conditional";
|
|
11
12
|
readonly position?: boolean;
|
|
12
13
|
readonly readObservation?: boolean;
|
|
13
14
|
readonly openTruncate?: boolean;
|
package/dist/safe-fs/core.d.ts
CHANGED
|
@@ -11,6 +11,7 @@ export * from "./fs/readonly/index.js";
|
|
|
11
11
|
export * from "./fs/mount/index.js";
|
|
12
12
|
export * from "./fs/overlay/index.js";
|
|
13
13
|
export * from "./fs/quota/index.js";
|
|
14
|
+
export * from "./fs/object-publication/index.js";
|
|
14
15
|
export { scopeFileSystem, retainFileSystemCleanup } from "./fs/scoped.js";
|
|
15
16
|
export type { RetainedFileSystemCleanupView, RetainedFileSystemCleanupOptions } from "./fs/scoped.js";
|
|
16
17
|
export * from "./fs/webdav/index.js";
|
package/dist/safe-fs/core.js
CHANGED
|
@@ -10,6 +10,7 @@ export * from "./fs/readonly/index.js";
|
|
|
10
10
|
export * from "./fs/mount/index.js";
|
|
11
11
|
export * from "./fs/overlay/index.js";
|
|
12
12
|
export * from "./fs/quota/index.js";
|
|
13
|
+
export * from "./fs/object-publication/index.js";
|
|
13
14
|
export { scopeFileSystem, retainFileSystemCleanup } from "./fs/scoped.js";
|
|
14
15
|
export * from "./fs/webdav/index.js";
|
|
15
16
|
export * from "./bridge/index.js";
|
|
@@ -101,7 +101,7 @@ export async function openRetainedResizeFile(filesystem, path, options) {
|
|
|
101
101
|
export function readOnlyCapabilities(capabilities) {
|
|
102
102
|
const inspection = Object.fromEntries([
|
|
103
103
|
"read", "stat", "readdir", "realpath", "access", "readlink", "explicitDirectories", "implicitDirectories",
|
|
104
|
-
"symlinks", "streamingRead", "open", "retainedRead",
|
|
104
|
+
"symlinks", "streamingRead", "open", "retainedRead", "versionedDescriptors",
|
|
105
105
|
].filter(name => capabilities[name] !== undefined).map(name => [name, capabilities[name]]));
|
|
106
106
|
return Object.freeze({
|
|
107
107
|
...inspection, readOnly: true, write: false, append: false, exclusiveCreate: false,
|
|
@@ -2,6 +2,7 @@ import { FsError } from "../contracts/errors.js";
|
|
|
2
2
|
import { finishCleanup } from "../contracts/cleanup.js";
|
|
3
3
|
function admitCapabilities(path, options, capabilities) {
|
|
4
4
|
if (![capabilities.positionedRead, capabilities.positionedWrite, capabilities.truncate].every(value => typeof value === "boolean")
|
|
5
|
+
|| capabilities.publication !== undefined && capabilities.publication !== "conditional"
|
|
5
6
|
|| capabilities.position !== undefined && typeof capabilities.position !== "boolean"
|
|
6
7
|
|| capabilities.readObservation !== undefined && typeof capabilities.readObservation !== "boolean"
|
|
7
8
|
|| capabilities.openTruncate !== undefined && typeof capabilities.openTruncate !== "boolean"
|
|
@@ -67,6 +68,7 @@ class ManagedFileDescriptor {
|
|
|
67
68
|
this.#backend = backend;
|
|
68
69
|
const positionedAppendWrite = capabilities.positionedAppendWrite === true && capabilities.positionedWrite && options.access !== "read";
|
|
69
70
|
this.capabilities = Object.freeze({
|
|
71
|
+
...(capabilities.publication === undefined ? {} : { publication: capabilities.publication }),
|
|
70
72
|
...(capabilities.position === undefined ? {} : { position: capabilities.position }),
|
|
71
73
|
...(capabilities.readObservation === undefined ? {} : { readObservation: capabilities.readObservation }),
|
|
72
74
|
...(capabilities.openTruncate === undefined ? {} : { openTruncate: capabilities.openTruncate }),
|
|
@@ -102,7 +102,7 @@ export class MountFileSystem {
|
|
|
102
102
|
};
|
|
103
103
|
const values = mounts.map(({ backend }) => {
|
|
104
104
|
if (backend.capabilities.readOnly === true
|
|
105
|
-
&& !["open", "read", "stat", "readdir", "realpath", "access", "readlink", "explicitDirectories", "implicitDirectories"].includes(capability))
|
|
105
|
+
&& !["open", "versionedDescriptors", "read", "stat", "readdir", "realpath", "access", "readlink", "explicitDirectories", "implicitDirectories"].includes(capability))
|
|
106
106
|
return false;
|
|
107
107
|
const declared = backend.capabilities[capability];
|
|
108
108
|
if (capability === "descriptorWriteStream" && backend.capabilities.streamingWrite === false)
|
|
@@ -114,7 +114,7 @@ export class MountFileSystem {
|
|
|
114
114
|
return values.every(value => value === true) ? true : values.every(value => value === false) ? false : undefined;
|
|
115
115
|
};
|
|
116
116
|
const semantics = Object.fromEntries([
|
|
117
|
-
"atomicEntryRemoval", "atomicTreeRemoval", "atomicFileMutation", "atomicFileStaging", "atomicDirectoryMetadata", "read", "stat", "readdir", "realpath", "access", "open",
|
|
117
|
+
"atomicEntryRemoval", "atomicTreeRemoval", "atomicFileMutation", "atomicFileStaging", "atomicDirectoryMetadata", "read", "stat", "readdir", "realpath", "access", "open", "versionedDescriptors",
|
|
118
118
|
"write", "append", "exclusiveCreate", "explicitDirectories", "implicitDirectories", "mkdir", "recursiveMkdir",
|
|
119
119
|
"remove", "removeDirectory", "recursiveRemove", "rename", "atomicRenameNoReplace", "copy", "exclusiveCopy", "readlink", "truncate",
|
|
120
120
|
"streamingAppend", "randomAccessWrite", "descriptorWriteStream", "retainedResize", "atomicResize", "symlinks", "hardlinks", "permissions", "timestamps",
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { FileStat, FileSystem, FsOptions, OpenFileOptions } from "../../contracts/filesystem.js";
|
|
2
|
+
import type { ByteSource } from "../../contracts/io.js";
|
|
3
|
+
export interface ObjectFileVersion {
|
|
4
|
+
readonly revision: string;
|
|
5
|
+
readonly stat: FileStat;
|
|
6
|
+
read(position: number, maxBytes: number, options?: FsOptions): Promise<Uint8Array>;
|
|
7
|
+
close(): Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
export interface ObjectFilePublicationOptions extends FsOptions {
|
|
10
|
+
readonly size: number;
|
|
11
|
+
readonly mode: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ObjectFileAcquireOptions extends FsOptions {
|
|
14
|
+
readonly access: OpenFileOptions["access"];
|
|
15
|
+
}
|
|
16
|
+
export interface ObjectFilePublicationStore {
|
|
17
|
+
acquire(path: string, options: ObjectFileAcquireOptions): Promise<ObjectFileVersion | undefined>;
|
|
18
|
+
publish?(path: string, expectedRevision: string | null, source: ByteSource, options: ObjectFilePublicationOptions): Promise<ObjectFileVersion>;
|
|
19
|
+
}
|
|
20
|
+
export interface ObjectFileDescriptorOptions {
|
|
21
|
+
readonly chunkBytes?: number;
|
|
22
|
+
readonly maxStagedBytes?: number;
|
|
23
|
+
readonly maxStagedPages?: number;
|
|
24
|
+
readonly maxFileBytes?: number;
|
|
25
|
+
readonly maxOpenFiles?: number;
|
|
26
|
+
}
|
|
27
|
+
export declare function withObjectFileDescriptors(filesystem: FileSystem, store: ObjectFilePublicationStore, options?: ObjectFileDescriptorOptions): FileSystem;
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
import { composeAbortSignals } from "../../contracts/abort.js";
|
|
2
|
+
import { finishCleanup } from "../../contracts/cleanup.js";
|
|
3
|
+
import { FsError } from "../../contracts/errors.js";
|
|
4
|
+
import { validatePath } from "../../contracts/virtual-path.js";
|
|
5
|
+
import { openFileDescriptor } from "../descriptor.js";
|
|
6
|
+
export function withObjectFileDescriptors(filesystem, store, options = {}) {
|
|
7
|
+
const chunkBytes = options.chunkBytes ?? 65536;
|
|
8
|
+
const maxStagedBytes = options.maxStagedBytes ?? 8 * 1024 * 1024;
|
|
9
|
+
const maxStagedPages = options.maxStagedPages ?? 4096;
|
|
10
|
+
const maxFileBytes = options.maxFileBytes ?? 256 * 1024 * 1024;
|
|
11
|
+
const maxOpenFiles = options.maxOpenFiles ?? 64;
|
|
12
|
+
if (![chunkBytes, maxStagedBytes, maxStagedPages, maxFileBytes, maxOpenFiles].every(value => Number.isSafeInteger(value) && value > 0)
|
|
13
|
+
|| chunkBytes > 1048576 || typeof store.acquire !== "function"
|
|
14
|
+
|| store.publish !== undefined && typeof store.publish !== "function")
|
|
15
|
+
throw new TypeError("Invalid object descriptor configuration");
|
|
16
|
+
let stagedBytes = 0;
|
|
17
|
+
let openFiles = 0;
|
|
18
|
+
const version = (value) => {
|
|
19
|
+
if (!value || typeof value.revision !== "string" || value.revision.length === 0 || value.revision.length > 4096
|
|
20
|
+
|| !value.stat || value.stat.type !== "file" || !Number.isSafeInteger(value.stat.size) || value.stat.size < 0
|
|
21
|
+
|| !Number.isSafeInteger(value.stat.mode) || value.stat.mode < 0 || value.stat.mode > 0o177777
|
|
22
|
+
|| ![value.stat.mtimeMs, value.stat.atimeMs, value.stat.ctimeMs].every(Number.isFinite)
|
|
23
|
+
|| typeof value.read !== "function" || typeof value.close !== "function")
|
|
24
|
+
throw new FsError("EIO", { message: "Invalid immutable object version" });
|
|
25
|
+
if (value.stat.size > maxFileBytes)
|
|
26
|
+
throw new FsError("EFBIG", { message: "Object descriptor file limit exceeded" });
|
|
27
|
+
return Object.freeze({ revision: value.revision, stat: Object.freeze({ ...value.stat }), read: value.read.bind(value), close: value.close.bind(value) });
|
|
28
|
+
};
|
|
29
|
+
const clearPages = (state) => {
|
|
30
|
+
stagedBytes -= state.pages.size * chunkBytes;
|
|
31
|
+
state.pages.clear();
|
|
32
|
+
};
|
|
33
|
+
const open = (path, openOptions) => openFileDescriptor(path, openOptions, {
|
|
34
|
+
publication: "conditional", position: true, positionedRead: true, positionedWrite: true,
|
|
35
|
+
truncate: true, synchronization: "storage",
|
|
36
|
+
}, async (admitted) => {
|
|
37
|
+
validatePath(path);
|
|
38
|
+
if (path.length === 0)
|
|
39
|
+
throw new FsError("ENOENT", { path });
|
|
40
|
+
if (openFiles >= maxOpenFiles)
|
|
41
|
+
throw new FsError("EMFILE", { path });
|
|
42
|
+
openFiles++;
|
|
43
|
+
let acquiring = true;
|
|
44
|
+
const state = { head: undefined, position: 0, size: 0, inheritedSize: 0, modifiedAt: Date.now(), dirty: false, pages: new Map(), failure: undefined };
|
|
45
|
+
const check = (forwarded) => {
|
|
46
|
+
if (acquiring)
|
|
47
|
+
admitted.signal?.throwIfAborted();
|
|
48
|
+
forwarded.signal?.throwIfAborted();
|
|
49
|
+
if (state.failure)
|
|
50
|
+
throw state.failure.reason;
|
|
51
|
+
};
|
|
52
|
+
const perform = async (forwarded, action) => {
|
|
53
|
+
check(forwarded);
|
|
54
|
+
const scope = composeAbortSignals([...(acquiring && admitted.signal ? [admitted.signal] : []), ...(forwarded.signal ? [forwarded.signal] : [])]);
|
|
55
|
+
try {
|
|
56
|
+
const result = await action({ signal: scope.signal });
|
|
57
|
+
scope.signal.throwIfAborted();
|
|
58
|
+
return result;
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
scope.signal.throwIfAborted();
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
scope.dispose();
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
const readBase = async (position, length, forwarded) => {
|
|
69
|
+
const count = Math.min(length, Math.max(0, state.inheritedSize - position));
|
|
70
|
+
if (count === 0 || !state.head)
|
|
71
|
+
return new Uint8Array();
|
|
72
|
+
const data = await perform(forwarded, selected => state.head.read(position, count, selected));
|
|
73
|
+
if (!(data instanceof Uint8Array) || data.byteLength !== count)
|
|
74
|
+
throw new FsError("EIO", { path, message: "Immutable range read returned an invalid byte count" });
|
|
75
|
+
return Uint8Array.from(data);
|
|
76
|
+
};
|
|
77
|
+
const read = async (buffer, position, forwarded) => {
|
|
78
|
+
check(forwarded);
|
|
79
|
+
const count = Math.min(buffer.byteLength, Math.max(0, state.size - position));
|
|
80
|
+
let copied = 0;
|
|
81
|
+
while (copied < count) {
|
|
82
|
+
const offset = position + copied;
|
|
83
|
+
const page = Math.floor(offset / chunkBytes);
|
|
84
|
+
const within = offset % chunkBytes;
|
|
85
|
+
const length = Math.min(count - copied, chunkBytes - within);
|
|
86
|
+
const staged = state.pages.get(page);
|
|
87
|
+
if (staged)
|
|
88
|
+
buffer.set(staged.subarray(within, within + length), copied);
|
|
89
|
+
else {
|
|
90
|
+
buffer.fill(0, copied, copied + length);
|
|
91
|
+
buffer.set(await readBase(offset, length, forwarded), copied);
|
|
92
|
+
}
|
|
93
|
+
copied += length;
|
|
94
|
+
}
|
|
95
|
+
return count;
|
|
96
|
+
};
|
|
97
|
+
const flush = async (forwarded) => {
|
|
98
|
+
check(forwarded);
|
|
99
|
+
if (!state.dirty)
|
|
100
|
+
return;
|
|
101
|
+
if (!store.publish)
|
|
102
|
+
throw new FsError("ENOTSUP", { path, message: "Object writes require authoritative conditional publication" });
|
|
103
|
+
const expected = state.head?.revision ?? null;
|
|
104
|
+
let emitted = 0;
|
|
105
|
+
let active = true;
|
|
106
|
+
let received;
|
|
107
|
+
let source;
|
|
108
|
+
const streamAbort = new AbortController();
|
|
109
|
+
let streamScope;
|
|
110
|
+
try {
|
|
111
|
+
await perform(forwarded, async (selected) => {
|
|
112
|
+
streamScope = composeAbortSignals([...(selected.signal ? [selected.signal] : []), streamAbort.signal]);
|
|
113
|
+
const bodyOptions = { signal: streamScope.signal };
|
|
114
|
+
source = (async function* () {
|
|
115
|
+
for (let position = 0; position < state.size; position += chunkBytes) {
|
|
116
|
+
if (!active)
|
|
117
|
+
throw new FsError("EBADF", { path });
|
|
118
|
+
const bytes = new Uint8Array(Math.min(chunkBytes, state.size - position));
|
|
119
|
+
await read(bytes, position, bodyOptions);
|
|
120
|
+
emitted += bytes.length;
|
|
121
|
+
yield bytes;
|
|
122
|
+
}
|
|
123
|
+
})();
|
|
124
|
+
received = await store.publish(path, expected, source, { ...selected, size: state.size, mode: (state.head?.stat.mode ?? admitted.mode) & 0o7777 });
|
|
125
|
+
});
|
|
126
|
+
const published = version(received);
|
|
127
|
+
if (emitted !== state.size || published.stat.size !== state.size || published.revision === expected)
|
|
128
|
+
throw new FsError("EIO", { path, message: "Invalid conditional publication acknowledgement" });
|
|
129
|
+
const previous = state.head;
|
|
130
|
+
state.head = published;
|
|
131
|
+
received = undefined;
|
|
132
|
+
state.inheritedSize = state.size;
|
|
133
|
+
state.modifiedAt = published.stat.mtimeMs;
|
|
134
|
+
state.dirty = false;
|
|
135
|
+
clearPages(state);
|
|
136
|
+
await previous?.close();
|
|
137
|
+
}
|
|
138
|
+
catch (reason) {
|
|
139
|
+
state.failure = { reason };
|
|
140
|
+
if (received)
|
|
141
|
+
await finishCleanup(() => received.close(), true);
|
|
142
|
+
throw reason;
|
|
143
|
+
}
|
|
144
|
+
finally {
|
|
145
|
+
active = false;
|
|
146
|
+
streamAbort.abort(new FsError("ECANCELED", { path, syscall: "object publication stream" }));
|
|
147
|
+
try {
|
|
148
|
+
await finishCleanup(() => source?.return(undefined), state.failure !== undefined);
|
|
149
|
+
}
|
|
150
|
+
finally {
|
|
151
|
+
streamScope?.dispose();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
try {
|
|
156
|
+
const capabilities = await filesystem.capabilitiesFor?.(path, admitted) ?? filesystem.capabilities;
|
|
157
|
+
check(admitted);
|
|
158
|
+
const mutating = admitted.access !== "read" || admitted.creation !== "never" || admitted.truncate;
|
|
159
|
+
if (mutating && capabilities.readOnly === true)
|
|
160
|
+
throw new FsError("EROFS", { path });
|
|
161
|
+
if (mutating && (!store.publish || capabilities.write === false))
|
|
162
|
+
throw new FsError("ENOTSUP", { path, message: "Object writes require authoritative conditional publication" });
|
|
163
|
+
if (admitted.access !== "write" && capabilities.read === false)
|
|
164
|
+
throw new FsError("EACCES", { path });
|
|
165
|
+
const acquired = await store.acquire(path, { access: admitted.access, ...(admitted.signal ? { signal: admitted.signal } : {}) });
|
|
166
|
+
if (acquired) {
|
|
167
|
+
try {
|
|
168
|
+
state.head = version(acquired);
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
await finishCleanup(() => acquired.close(), true);
|
|
172
|
+
throw error;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
check(admitted);
|
|
176
|
+
if (state.head && admitted.creation === "exclusive")
|
|
177
|
+
throw new FsError("EEXIST", { path });
|
|
178
|
+
if (!state.head && admitted.creation === "never")
|
|
179
|
+
throw new FsError("ENOENT", { path });
|
|
180
|
+
state.size = state.head?.stat.size ?? 0;
|
|
181
|
+
state.inheritedSize = state.size;
|
|
182
|
+
state.modifiedAt = state.head?.stat.mtimeMs ?? Date.now();
|
|
183
|
+
if (!state.head || admitted.truncate) {
|
|
184
|
+
state.size = 0;
|
|
185
|
+
state.inheritedSize = 0;
|
|
186
|
+
state.dirty = true;
|
|
187
|
+
await flush(admitted);
|
|
188
|
+
}
|
|
189
|
+
acquiring = false;
|
|
190
|
+
return {
|
|
191
|
+
resource: state,
|
|
192
|
+
getPosition: async () => state.position,
|
|
193
|
+
stat: async (_state, forwarded) => {
|
|
194
|
+
check(forwarded);
|
|
195
|
+
const { allocatedBytes: ignoredAllocated, ...metadata } = state.head.stat;
|
|
196
|
+
return { ...metadata, size: state.size, mtimeMs: state.modifiedAt };
|
|
197
|
+
},
|
|
198
|
+
read: async (_state, buffer, position, forwarded) => {
|
|
199
|
+
const count = await read(buffer, position ?? state.position, forwarded);
|
|
200
|
+
if (position === null)
|
|
201
|
+
state.position += count;
|
|
202
|
+
return count;
|
|
203
|
+
},
|
|
204
|
+
write: async (_state, buffer, position, forwarded) => {
|
|
205
|
+
check(forwarded);
|
|
206
|
+
const start = admitted.append ? state.size : position ?? state.position;
|
|
207
|
+
const end = start + buffer.byteLength;
|
|
208
|
+
if (!Number.isSafeInteger(end) || end > maxFileBytes)
|
|
209
|
+
throw new FsError("EFBIG", { path, message: "Object descriptor file limit exceeded" });
|
|
210
|
+
const first = Math.floor(start / chunkBytes);
|
|
211
|
+
const last = Math.floor((end - 1) / chunkBytes);
|
|
212
|
+
const missing = [];
|
|
213
|
+
const available = Math.min(Math.floor((maxStagedBytes - stagedBytes) / chunkBytes), maxStagedPages - stagedBytes / chunkBytes);
|
|
214
|
+
for (let page = first; page <= last; page++)
|
|
215
|
+
if (!state.pages.has(page)) {
|
|
216
|
+
if (missing.length >= available)
|
|
217
|
+
throw new FsError("ENOSPC", { path, message: "Object descriptor staging budget exceeded; flush staged writes or increase maxStagedBytes/maxStagedPages" });
|
|
218
|
+
missing.push(page);
|
|
219
|
+
}
|
|
220
|
+
const additional = missing.length * chunkBytes;
|
|
221
|
+
stagedBytes += additional;
|
|
222
|
+
const prepared = new Map();
|
|
223
|
+
try {
|
|
224
|
+
for (const page of missing) {
|
|
225
|
+
const bytes = new Uint8Array(chunkBytes);
|
|
226
|
+
bytes.set(await readBase(page * chunkBytes, chunkBytes, forwarded));
|
|
227
|
+
prepared.set(page, bytes);
|
|
228
|
+
}
|
|
229
|
+
check(forwarded);
|
|
230
|
+
}
|
|
231
|
+
catch (error) {
|
|
232
|
+
stagedBytes -= additional;
|
|
233
|
+
throw error;
|
|
234
|
+
}
|
|
235
|
+
for (const [page, bytes] of prepared)
|
|
236
|
+
state.pages.set(page, bytes);
|
|
237
|
+
let copied = 0;
|
|
238
|
+
while (copied < buffer.byteLength) {
|
|
239
|
+
const offset = start + copied;
|
|
240
|
+
const length = Math.min(buffer.byteLength - copied, chunkBytes - offset % chunkBytes);
|
|
241
|
+
state.pages.get(Math.floor(offset / chunkBytes)).set(buffer.subarray(copied, copied + length), offset % chunkBytes);
|
|
242
|
+
copied += length;
|
|
243
|
+
}
|
|
244
|
+
state.size = Math.max(state.size, end);
|
|
245
|
+
state.modifiedAt = Date.now();
|
|
246
|
+
state.dirty = true;
|
|
247
|
+
if (position === null)
|
|
248
|
+
state.position = end;
|
|
249
|
+
if (admitted.synchronization !== undefined)
|
|
250
|
+
await flush(forwarded);
|
|
251
|
+
return buffer.byteLength;
|
|
252
|
+
},
|
|
253
|
+
truncate: async (_state, length, forwarded) => {
|
|
254
|
+
check(forwarded);
|
|
255
|
+
if (length > maxFileBytes)
|
|
256
|
+
throw new FsError("EFBIG", { path });
|
|
257
|
+
for (const [page, bytes] of state.pages) {
|
|
258
|
+
if (page * chunkBytes >= length) {
|
|
259
|
+
state.pages.delete(page);
|
|
260
|
+
stagedBytes -= chunkBytes;
|
|
261
|
+
}
|
|
262
|
+
else if ((page + 1) * chunkBytes > length)
|
|
263
|
+
bytes.fill(0, length % chunkBytes);
|
|
264
|
+
}
|
|
265
|
+
state.inheritedSize = Math.min(state.inheritedSize, length);
|
|
266
|
+
state.size = length;
|
|
267
|
+
state.modifiedAt = Date.now();
|
|
268
|
+
state.dirty = true;
|
|
269
|
+
if (admitted.synchronization !== undefined)
|
|
270
|
+
await flush(forwarded);
|
|
271
|
+
},
|
|
272
|
+
sync: async (_state, _dataOnly, forwarded) => { await flush(forwarded); },
|
|
273
|
+
close: async () => {
|
|
274
|
+
let failed = false;
|
|
275
|
+
try {
|
|
276
|
+
if (state.dirty || state.failure)
|
|
277
|
+
await flush({});
|
|
278
|
+
}
|
|
279
|
+
catch (error) {
|
|
280
|
+
failed = true;
|
|
281
|
+
throw error;
|
|
282
|
+
}
|
|
283
|
+
finally {
|
|
284
|
+
clearPages(state);
|
|
285
|
+
const retained = state.head;
|
|
286
|
+
state.head = undefined;
|
|
287
|
+
try {
|
|
288
|
+
await finishCleanup(() => retained?.close(), failed);
|
|
289
|
+
}
|
|
290
|
+
finally {
|
|
291
|
+
openFiles--;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
},
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
catch (error) {
|
|
298
|
+
clearPages(state);
|
|
299
|
+
try {
|
|
300
|
+
await finishCleanup(() => state.head?.close(), true);
|
|
301
|
+
}
|
|
302
|
+
finally {
|
|
303
|
+
openFiles--;
|
|
304
|
+
}
|
|
305
|
+
throw error;
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
return new Proxy(Object.create(filesystem), {
|
|
309
|
+
get(_target, property) {
|
|
310
|
+
if (property === "open")
|
|
311
|
+
return open;
|
|
312
|
+
if (property === "capabilities")
|
|
313
|
+
return { ...filesystem.capabilities, open: true, versionedDescriptors: true };
|
|
314
|
+
if (property === "capabilitiesFor")
|
|
315
|
+
return async (path, query) => {
|
|
316
|
+
const capabilities = await filesystem.capabilitiesFor?.(path, query) ?? filesystem.capabilities;
|
|
317
|
+
query?.signal?.throwIfAborted();
|
|
318
|
+
return { ...capabilities, open: true, versionedDescriptors: true };
|
|
319
|
+
};
|
|
320
|
+
const value = Reflect.get(filesystem, property, filesystem);
|
|
321
|
+
return typeof value === "function" ? value.bind(filesystem) : value;
|
|
322
|
+
},
|
|
323
|
+
});
|
|
324
|
+
}
|
|
@@ -27,7 +27,7 @@ function namespaceMetadata(metadata, signal) {
|
|
|
27
27
|
abort();
|
|
28
28
|
});
|
|
29
29
|
}
|
|
30
|
-
async function
|
|
30
|
+
async function scanUsedBytes(fs, limits, options, change, retained = change?.retained === true) {
|
|
31
31
|
let total = 0;
|
|
32
32
|
let retainedTotal = 0n;
|
|
33
33
|
let shrinkCredited = false;
|
|
@@ -144,6 +144,8 @@ export function withFileSystemQuota(fs, options) {
|
|
|
144
144
|
throw new RangeError(`${name} must be a nonnegative safe integer`);
|
|
145
145
|
}
|
|
146
146
|
let queue = Promise.resolve();
|
|
147
|
+
let reservedBytes = 0;
|
|
148
|
+
const usedBytes = async (...args) => await scanUsedBytes(...args) + reservedBytes;
|
|
147
149
|
const mutate = (operation) => {
|
|
148
150
|
const result = queue.then(operation);
|
|
149
151
|
queue = result.catch(() => undefined);
|
|
@@ -200,6 +202,14 @@ export function withFileSystemQuota(fs, options) {
|
|
|
200
202
|
try {
|
|
201
203
|
admitted.signal?.throwIfAborted();
|
|
202
204
|
const pinned = admitted.access === "read" ? undefined : { ...await descriptor.stat(admitted) };
|
|
205
|
+
const conditional = descriptor.capabilities.publication === "conditional" && pinned !== undefined;
|
|
206
|
+
let publishedSize = pinned?.size ?? 0;
|
|
207
|
+
let reserved = 0;
|
|
208
|
+
const reserve = (size) => {
|
|
209
|
+
const next = Math.max(0, size - publishedSize);
|
|
210
|
+
reservedBytes += next - reserved;
|
|
211
|
+
reserved = next;
|
|
212
|
+
};
|
|
203
213
|
admitted.signal?.throwIfAborted();
|
|
204
214
|
if (pinned && (pinned.type !== "file" || !completeIdentity(pinned))) {
|
|
205
215
|
throw new FsError("ENOTSUP", { syscall: "open", path, message: "quota requires complete retained file identity" });
|
|
@@ -211,7 +221,7 @@ export function withFileSystemQuota(fs, options) {
|
|
|
211
221
|
const stat = { ...await descriptor.stat(operationOptions) };
|
|
212
222
|
operationOptions.signal?.throwIfAborted();
|
|
213
223
|
if (!pinned || stat.type !== "file" || !completeIdentity(stat)
|
|
214
|
-
|| stat.identityScope !== pinned.identityScope || stat.dev !== pinned.dev || stat.ino !== pinned.ino
|
|
224
|
+
|| !conditional && (stat.identityScope !== pinned.identityScope || stat.dev !== pinned.dev || stat.ino !== pinned.ino)
|
|
215
225
|
|| !Number.isSafeInteger(stat.size) || stat.size < 0) {
|
|
216
226
|
throw new FsError("EIO", { syscall: "fstat", path, message: "invalid retained quota identity or size" });
|
|
217
227
|
}
|
|
@@ -220,14 +230,31 @@ export function withFileSystemQuota(fs, options) {
|
|
|
220
230
|
const admitSize = async (stat, nextBytes, operationOptions) => {
|
|
221
231
|
if (!Number.isSafeInteger(nextBytes) || nextBytes < 0)
|
|
222
232
|
throw new FsError("EFBIG", { syscall: "write", path });
|
|
223
|
-
const projected =
|
|
233
|
+
const projected = conditional
|
|
234
|
+
? await usedBytes(fs, scanLimits, operationOptions) - reserved + Math.max(0, nextBytes - publishedSize)
|
|
235
|
+
: await usedBytes(fs, scanLimits, operationOptions, { path, stat, delta: nextBytes - stat.size, retained: true });
|
|
224
236
|
operationOptions.signal?.throwIfAborted();
|
|
225
237
|
if (projected > options.maxBytes)
|
|
226
238
|
throw new FileSystemQuotaError(options.maxBytes);
|
|
239
|
+
if (conditional)
|
|
240
|
+
reserve(Math.max(nextBytes, publishedSize + reserved));
|
|
227
241
|
};
|
|
228
|
-
const forwarded = forwardFileDescriptor(descriptor, (syscall, operationOptions, action) => syscall === "close" ? action() : mutate(async () => {
|
|
229
|
-
|
|
230
|
-
|
|
242
|
+
const forwarded = forwardFileDescriptor(descriptor, (syscall, operationOptions, action) => syscall === "close" && !conditional ? action() : mutate(async () => {
|
|
243
|
+
try {
|
|
244
|
+
operationOptions.signal?.throwIfAborted();
|
|
245
|
+
const result = await action();
|
|
246
|
+
if (conditional && (syscall === "fsync" || syscall === "fdatasync")) {
|
|
247
|
+
publishedSize = (await current(operationOptions)).size;
|
|
248
|
+
reserve(publishedSize);
|
|
249
|
+
}
|
|
250
|
+
return result;
|
|
251
|
+
}
|
|
252
|
+
finally {
|
|
253
|
+
if (syscall === "close") {
|
|
254
|
+
reservedBytes -= reserved;
|
|
255
|
+
reserved = 0;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
231
258
|
}));
|
|
232
259
|
return {
|
|
233
260
|
...forwarded,
|
|
@@ -251,12 +278,24 @@ export function withFileSystemQuota(fs, options) {
|
|
|
251
278
|
if (!Number.isSafeInteger(offset) || offset < 0)
|
|
252
279
|
throw new FsError("EIO", { syscall: "getPosition", path });
|
|
253
280
|
await admitSize(stat, Math.max(stat.size, offset + buffer.byteLength), operationOptions);
|
|
254
|
-
|
|
281
|
+
const count = await retained.write(buffer, position, operationOptions);
|
|
282
|
+
if (conditional) {
|
|
283
|
+
const size = (await current(operationOptions)).size;
|
|
284
|
+
if (admitted.synchronization !== undefined)
|
|
285
|
+
publishedSize = size;
|
|
286
|
+
reserve(size);
|
|
287
|
+
}
|
|
288
|
+
return count;
|
|
255
289
|
}),
|
|
256
290
|
truncate: (retained, length, operationOptions) => mutate(async () => {
|
|
257
291
|
operationOptions.signal?.throwIfAborted();
|
|
258
292
|
await admitSize(await current(operationOptions), length, operationOptions);
|
|
259
293
|
await retained.truncate(length, operationOptions);
|
|
294
|
+
if (conditional) {
|
|
295
|
+
if (admitted.synchronization !== undefined)
|
|
296
|
+
publishedSize = length;
|
|
297
|
+
reserve(length);
|
|
298
|
+
}
|
|
260
299
|
}),
|
|
261
300
|
};
|
|
262
301
|
}
|
package/dist/safe-fs/index.d.ts
CHANGED
|
@@ -13,6 +13,7 @@ export * from "./fs/readonly/index.js";
|
|
|
13
13
|
export * from "./fs/mount/index.js";
|
|
14
14
|
export * from "./fs/overlay/index.js";
|
|
15
15
|
export * from "./fs/quota/index.js";
|
|
16
|
+
export * from "./fs/object-publication/index.js";
|
|
16
17
|
export { scopeFileSystem, retainFileSystemCleanup } from "./fs/scoped.js";
|
|
17
18
|
export type { RetainedFileSystemCleanupView, RetainedFileSystemCleanupOptions } from "./fs/scoped.js";
|
|
18
19
|
export * from "./node/index.js";
|
package/dist/safe-fs/index.js
CHANGED
|
@@ -11,6 +11,7 @@ export * from "./fs/readonly/index.js";
|
|
|
11
11
|
export * from "./fs/mount/index.js";
|
|
12
12
|
export * from "./fs/overlay/index.js";
|
|
13
13
|
export * from "./fs/quota/index.js";
|
|
14
|
+
export * from "./fs/object-publication/index.js";
|
|
14
15
|
export { scopeFileSystem, retainFileSystemCleanup } from "./fs/scoped.js";
|
|
15
16
|
export * from "./node/index.js";
|
|
16
17
|
export { createFileSystem, readConfigRecord, validateFileSystemConfig } from "./config.js";
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { FileSystem } from "../contracts/filesystem.js";
|
|
2
|
+
import type { ObjectFilePublicationStore } from "../fs/object-publication/index.js";
|
|
3
|
+
export interface ObjectFilePublicationConformanceFixture {
|
|
4
|
+
readonly fs: FileSystem;
|
|
5
|
+
readonly store: ObjectFilePublicationStore;
|
|
6
|
+
readonly root: string;
|
|
7
|
+
dispose(): void | Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
export interface ObjectFilePublicationConformanceOptions {
|
|
10
|
+
readonly createFixture: () => ObjectFilePublicationConformanceFixture | Promise<ObjectFilePublicationConformanceFixture>;
|
|
11
|
+
}
|
|
12
|
+
export interface ObjectFilePublicationConformanceCase {
|
|
13
|
+
readonly name: string;
|
|
14
|
+
run(): Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
export declare function createObjectFilePublicationConformanceCases(options: ObjectFilePublicationConformanceOptions): ObjectFilePublicationConformanceCase[];
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { finishCleanup } from "../contracts/cleanup.js";
|
|
2
|
+
import { toByteSource } from "../contracts/io.js";
|
|
3
|
+
import { validatePath } from "../contracts/virtual-path.js";
|
|
4
|
+
import { withObjectFileDescriptors } from "../fs/object-publication/index.js";
|
|
5
|
+
function check(condition, message) {
|
|
6
|
+
if (!condition)
|
|
7
|
+
throw new Error(message);
|
|
8
|
+
}
|
|
9
|
+
function bytes(actual, expected) {
|
|
10
|
+
check(actual instanceof Uint8Array && actual.length === expected.length
|
|
11
|
+
&& actual.every((value, index) => value === expected[index]), "object publication returned incorrect bytes");
|
|
12
|
+
}
|
|
13
|
+
function conflict(reason) {
|
|
14
|
+
check(typeof reason === "object" && reason !== null && "code" in reason
|
|
15
|
+
&& (reason.code === "EAGAIN" || reason.code === "EEXIST"), "conditional publication must report a version/create conflict");
|
|
16
|
+
}
|
|
17
|
+
export function createObjectFilePublicationConformanceCases(options) {
|
|
18
|
+
const cases = [
|
|
19
|
+
{
|
|
20
|
+
name: "object publication: immutable reads survive replacement",
|
|
21
|
+
async run({ fixture, path, track, publish }) {
|
|
22
|
+
const name = path("pinned");
|
|
23
|
+
const initial = await publish(name, null, new Uint8Array([1, 2, 3, 4]));
|
|
24
|
+
const acquired = await fixture.store.acquire(name, { access: "read" });
|
|
25
|
+
check(acquired, "published file could not be acquired");
|
|
26
|
+
track(acquired);
|
|
27
|
+
check(acquired.revision === initial.revision, "acquisition did not observe the published namespace revision");
|
|
28
|
+
const replacement = await publish(name, initial.revision, new Uint8Array([5, 6, 7]));
|
|
29
|
+
check(replacement.revision !== initial.revision, "namespace revisions must change on publication");
|
|
30
|
+
bytes(await acquired.read(1, 2), new Uint8Array([2, 3]));
|
|
31
|
+
const mutable = await acquired.read(0, 4);
|
|
32
|
+
mutable.fill(0);
|
|
33
|
+
bytes(await acquired.read(0, 4), new Uint8Array([1, 2, 3, 4]));
|
|
34
|
+
bytes(await fixture.fs.readFile(name), new Uint8Array([5, 6, 7]));
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
name: "object publication: exclusive create has exactly one winner",
|
|
39
|
+
async run({ fixture, path, publish }) {
|
|
40
|
+
const name = path("exclusive");
|
|
41
|
+
const results = await Promise.allSettled([
|
|
42
|
+
publish(name, null, new Uint8Array([1])), publish(name, null, new Uint8Array([2])),
|
|
43
|
+
]);
|
|
44
|
+
const winners = results.filter(result => result.status === "fulfilled");
|
|
45
|
+
const failures = results.filter(result => result.status === "rejected");
|
|
46
|
+
check(winners.length === 1 && failures.length === 1, "atomic creation must admit exactly one concurrent writer");
|
|
47
|
+
conflict(failures[0].reason);
|
|
48
|
+
bytes(await fixture.fs.readFile(name), new Uint8Array([results[0].status === "fulfilled" ? 1 : 2]));
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
name: "object publication: stale updates cannot replace an acknowledged generation",
|
|
53
|
+
async run({ fixture, path, publish }) {
|
|
54
|
+
const name = path("conditional");
|
|
55
|
+
const initial = await publish(name, null, new Uint8Array([0]));
|
|
56
|
+
const results = await Promise.allSettled([
|
|
57
|
+
publish(name, initial.revision, new Uint8Array([1])),
|
|
58
|
+
publish(name, initial.revision, new Uint8Array([2])),
|
|
59
|
+
]);
|
|
60
|
+
const failures = results.filter(result => result.status === "rejected");
|
|
61
|
+
check(results.filter(result => result.status === "fulfilled").length === 1 && failures.length === 1, "conditional publication admitted conflicting revisions");
|
|
62
|
+
conflict(failures[0].reason);
|
|
63
|
+
bytes(await fixture.fs.readFile(name), new Uint8Array([results[0].status === "fulfilled" ? 1 : 2]));
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
name: "object publication: cancelled creation does not publish",
|
|
68
|
+
async run({ fixture, path, track }) {
|
|
69
|
+
const name = path("cancelled");
|
|
70
|
+
const controller = new AbortController();
|
|
71
|
+
const reason = new Error("conformance cancellation");
|
|
72
|
+
controller.abort(reason);
|
|
73
|
+
let rejected = false;
|
|
74
|
+
try {
|
|
75
|
+
track(await fixture.store.publish(name, null, toByteSource(new Uint8Array([1])), {
|
|
76
|
+
signal: controller.signal, size: 1, mode: 0o600,
|
|
77
|
+
}));
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
check(error === reason, "publication must preserve cancellation identity");
|
|
81
|
+
rejected = true;
|
|
82
|
+
}
|
|
83
|
+
check(rejected, "cancelled publication succeeded");
|
|
84
|
+
const visible = await fixture.store.acquire(name, { access: "read" });
|
|
85
|
+
if (visible)
|
|
86
|
+
track(visible);
|
|
87
|
+
check(visible === undefined, "cancelled creation left a published key");
|
|
88
|
+
},
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
name: "object publication: descriptor updates flush conditionally",
|
|
92
|
+
async run({ fixture, path, track, publish }) {
|
|
93
|
+
const name = path("descriptor");
|
|
94
|
+
await publish(name, null, new Uint8Array([1, 2, 3, 4]));
|
|
95
|
+
const fs = withObjectFileDescriptors(fixture.fs, fixture.store, { chunkBytes: 4, maxStagedBytes: 16 });
|
|
96
|
+
const descriptor = track(await fs.open(name, { access: "readwrite" }));
|
|
97
|
+
check(descriptor.capabilities.publication === "conditional", "descriptor publication profile was lost");
|
|
98
|
+
await descriptor.write(new Uint8Array([7]), 1);
|
|
99
|
+
bytes(await fixture.fs.readFile(name), new Uint8Array([1, 2, 3, 4]));
|
|
100
|
+
await descriptor.sync(false);
|
|
101
|
+
bytes(await fixture.fs.readFile(name), new Uint8Array([1, 7, 3, 4]));
|
|
102
|
+
await descriptor.truncate(2);
|
|
103
|
+
await descriptor.truncate(4);
|
|
104
|
+
await descriptor.sync(false);
|
|
105
|
+
bytes(await fixture.fs.readFile(name), new Uint8Array([1, 7, 0, 0]));
|
|
106
|
+
},
|
|
107
|
+
},
|
|
108
|
+
];
|
|
109
|
+
return cases.map(entry => ({
|
|
110
|
+
name: entry.name,
|
|
111
|
+
async run() {
|
|
112
|
+
const fixture = await options.createFixture();
|
|
113
|
+
const resources = [];
|
|
114
|
+
let failed = false;
|
|
115
|
+
try {
|
|
116
|
+
validatePath(fixture.root);
|
|
117
|
+
check(fixture.root.startsWith("/"), "conformance fixture root must be absolute");
|
|
118
|
+
check(typeof fixture.store.publish === "function", "conformance requires authoritative conditional publication");
|
|
119
|
+
const root = fixture.root.endsWith("/") ? fixture.root.slice(0, -1) : fixture.root;
|
|
120
|
+
const track = (value) => {
|
|
121
|
+
resources.push(value);
|
|
122
|
+
return value;
|
|
123
|
+
};
|
|
124
|
+
await entry.run({ fixture, path: name => `${root}/${name}`, track,
|
|
125
|
+
async publish(path, revision, data) {
|
|
126
|
+
return track(await fixture.store.publish(path, revision, toByteSource(data), { size: data.length, mode: 0o600 }));
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
failed = true;
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
await finishCleanup(async () => {
|
|
136
|
+
const closed = await Promise.allSettled(resources.map(resource => resource.close()));
|
|
137
|
+
const rejected = closed.find(result => result.status === "rejected");
|
|
138
|
+
await finishCleanup(() => fixture.dispose(), rejected !== undefined);
|
|
139
|
+
if (rejected?.status === "rejected")
|
|
140
|
+
throw rejected.reason;
|
|
141
|
+
}, failed);
|
|
142
|
+
}
|
|
143
|
+
},
|
|
144
|
+
}));
|
|
145
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@poe-platform/safe-fs",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.662",
|
|
4
4
|
"description": "Composable filesystem with a portable core and explicit Node adapters",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -41,6 +41,12 @@
|
|
|
41
41
|
"browser": "./dist/safe-fs/testing/atomic-filesystem.js",
|
|
42
42
|
"import": "./dist/safe-fs/testing/atomic-filesystem.js"
|
|
43
43
|
},
|
|
44
|
+
"./testing/object-publication": {
|
|
45
|
+
"types": "./dist/safe-fs/testing/object-publication.d.ts",
|
|
46
|
+
"workerd": "./dist/safe-fs/testing/object-publication.js",
|
|
47
|
+
"browser": "./dist/safe-fs/testing/object-publication.js",
|
|
48
|
+
"import": "./dist/safe-fs/testing/object-publication.js"
|
|
49
|
+
},
|
|
44
50
|
"./node": {
|
|
45
51
|
"types": {
|
|
46
52
|
"browser": "./dist/safe-fs/node-unavailable.d.ts",
|