@telepath-computer/autofile 0.4.0 → 0.5.1

package/dist/store.js

@@ -12,20 +12,37 @@ import { setTimeout as delay } from "node:timers/promises";
 import { lookup as lookupMimeType } from "mime-types";
 import { Query } from "mingo";
 import lockfile from "proper-lockfile";
-import { MAX_BLOB_BYTES, blobNameFromUri, deleteBlobUris, blobReferenceProblem, blobSizeProblem, isMediaType, writeBlob, } from "./blobs.js";
+import { blobNameFromUri, deleteBlobUris, blobReferenceProblem, isMediaType, writeBlob, writeBlobStream, } from "./blobs.js";
 import { getWriteLockPath } from "./config.js";
 import { jsonObjectProblem } from "./json.js";
 import { prepareSchema, SchemaRefusal } from "./schema.js";
 import { watchFolder } from "./watch.js";
 const typeNamePattern = /^[a-z][a-z0-9-]*$/u;
 const keyPattern = /^[a-z0-9][a-z0-9-]*$/u;
-const recordUriPattern = /^autofile:\/\/\/([a-z][a-z0-9-]*)\/([a-z0-9][a-z0-9-]*)$/u;
+const recordUriPattern = /^autofile:\/\/\/records\/([a-z][a-z0-9-]*)\/([a-z0-9][a-z0-9-]*)$/u;
+const collectionUriPattern = /^autofile:\/\/\/records\/collection\/[a-z0-9][a-z0-9-]*$/u;
 const timestampPattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/u;
 const MAX_FIND_RECORDS_PAGE_BYTES = 64 * 1024;
 const tidAlphabet = "234567abcdefghijklmnopqrstuvwxyz";
 const invalidTypeMessage = "type: must use lowercase letters, digits, and hyphens and start with a letter";
 const invalidKeyMessage = "key: must use lowercase letters, digits, and hyphens and start with a letter or digit";
-const invalidRecordUriMessage = "uri: must be a record URI, such as autofile:///task/book-the-dentist";
+const invalidRecordUriMessage = "uri: must be a record URI, such as autofile:///records/task/book-the-dentist";
+const builtInTypeMessage = "type: collection is built in and cannot be created, changed or deleted";
+const COLLECTION_TYPE = "collection";
+/** The built-in type whose records group other records, as folders do. */
+const collectionDefinition = {
+    description: "Collections group records the way folders do. A record joins a collection through its collections list, not through a field.",
+    title: "Collection",
+    titleField: "title",
+    schema: {
+        type: "object",
+        required: ["title"],
+        properties: {
+            title: { type: "string", description: "The name of the collection, as shown in a folder list." },
+            description: { type: "string", description: "What belongs in the collection." },
+        },
+    },
+};
 const clockIdentifier = BigInt(randomBytes(2).readUInt16BE() & 0x3ff);
 let lastTidTimestamp = 0n;
 /** A refused operation, with every problem the caller can correct. */
@@ -55,8 +72,8 @@ export class Autofile {
             refuse("folder: must be a non-empty string");
         }
         this.#dataFolder = resolve(options.folder);
-        this.#typeFolder = join(this.#dataFolder, "_types");
-        this.#blobFolder = join(this.#dataFolder, "_blobs");
+        this.#typeFolder = join(this.#dataFolder, "types");
+        this.#blobFolder = join(this.#dataFolder, "blobs");
         // Keep the existing process-home lock policy until a separate change moves it.
         this.#writeLockPath = resolve(getWriteLockPath());
     }
@@ -70,13 +87,18 @@ export class Autofile {
             if (problem !== undefined)
                 return refuse(`record: ${problem}`);
             // Snapshot checked JSON before asynchronous storage work begins.
-            return this.#createRecord(type, JSON.parse(JSON.stringify(value)), { key: options.key });
+            return this.#createRecord(type, JSON.parse(JSON.stringify(value)), {
+                key: options.key,
+                collections: snapshotCollections(options.collections),
+            });
         }),
-        update: (uri, patch) => this.#operation(() => {
+        update: (uri, patch, options = {}) => this.#operation(() => {
+            if (!isJsonObject(options))
+                return refuse("options: must be an object", uri);
             const problem = jsonObjectProblem(patch);
             if (problem !== undefined)
                 return refuse(`patch: ${problem}`, uri);
-            return this.#updateRecord(uri, JSON.parse(JSON.stringify(patch)));
+            return this.#updateRecord(uri, JSON.parse(JSON.stringify(patch)), snapshotCollections(options.collections));
         }),
         delete: (uri) => this.#operation(() => this.#deleteRecord(uri)),
     };
@@ -91,16 +113,21 @@ export class Autofile {
         put: (bytes, mediaType) => this.#operation(() => {
             if (!(bytes instanceof Uint8Array))
                 return refuse("bytes: must be a Uint8Array");
-            if (bytes.byteLength > MAX_BLOB_BYTES)
-                return refuse(blobSizeProblem(bytes.byteLength));
             if (!isMediaType(mediaType))
                 return refuse("mediaType: must be a valid media type, such as application/pdf");
             const snapshot = Buffer.from(bytes);
-            if (snapshot.length > MAX_BLOB_BYTES)
-                return refuse(blobSizeProblem(snapshot.length));
             return this.#withWriteLock(() => writeBlob(this.#blobFolder, snapshot, mediaType));
         }),
+        putStream: (chunks, mediaType) => this.#operation(() => {
+            if (typeof chunks?.[Symbol.asyncIterator] !== "function")
+                return refuse("chunks: must be an AsyncIterable of Uint8Array chunks");
+            if (!isMediaType(mediaType))
+                return refuse("mediaType: must be a valid media type, such as application/pdf");
+            return this.#withWriteLock(() => writeBlobStream(this.#blobFolder, snapshotBlobChunks(chunks), mediaType));
+        }),
         get: (uri) => this.#operation(() => this.#getBlob(uri)),
+        stat: (uri) => this.#operation(() => this.#statBlob(uri)),
+        stream: (uri, range) => this.#operation(() => this.#streamBlob(uri, range)),
     };
     /** Checks a schema without accessing storage or checking link targets. */
     static async prepareSchema(schema) {
@@ -202,9 +229,12 @@ export class Autofile {
         const uncertainTypes = new Set();
         const events = [];
         for (const change of changes) {
-            if (!change.uri.startsWith("autofile:///_types/"))
+            if (!change.uri.startsWith("autofile:///types/"))
+                continue;
+            const type = change.uri.slice("autofile:///types/".length);
+            // The built-in declaration never changes; a file at its path is ignored.
+            if (type === COLLECTION_TYPE)
                 continue;
-            const type = change.uri.slice("autofile:///_types/".length);
             try {
                 const declaration = await readValidDeclaration(this.#typeFolder, type);
                 if (declaration === undefined)
@@ -270,6 +300,11 @@ export class Autofile {
         if (input.q !== undefined && typeof input.q !== "string") {
             return refuse("q: must be a string");
         }
+        const collection = input.collection;
+        if (collection !== undefined && collection !== null
+            && (typeof collection !== "string" || !collectionUriPattern.test(collection))) {
+            return refuse("collection: must be a collection URI or null");
+        }
         if (input.where !== undefined && !isJsonObject(input.where)) {
             return refuse("where: must be a JSON object");
         }
@@ -295,6 +330,9 @@ export class Autofile {
         if (input.cursor !== undefined && afterUri === undefined) {
             return refuse("cursor: is invalid");
         }
+        if (typeof collection === "string" && !await recordExists(this.#dataFolder, this.#typeFolder, collection)) {
+            return refuse("collection does not exist", collection);
+        }
         const declarations = requestedTypes === undefined
             ? await this.#listTypes()
             : await Promise.all([...new Set(requestedTypes)].map(async (type) => {
@@ -306,7 +344,7 @@ export class Autofile {
         const entries = [];
         for (const declaration of declarations) {
             const schema = await preparedDeclarationSchema(declaration, declaration.uri);
-            for (const entry of await recognizedRecordEntries(join(this.#dataFolder, declaration.type))) {
+            for (const entry of await recognizedRecordEntries(recordFolder(this.#dataFolder, declaration.type))) {
                 const key = entry.name.slice(0, -".json".length);
                 entries.push({ type: declaration.type, key, name: entry.name, uri: recordUri(declaration.type, key), declaration, schema });
             }
@@ -333,7 +371,7 @@ export class Autofile {
             let record;
             let resultError;
             try {
-                stored = await readStoredRecord(join(this.#dataFolder, type, name));
+                stored = await readStoredRecord(join(recordFolder(this.#dataFolder, type), name));
             }
             catch (error) {
                 resultError = { uri, message: recordReadError(error) };
@@ -345,7 +383,8 @@ export class Autofile {
                         message: stored.status === "invalid" ? stored.message : "record does not exist",
                     };
                 }
-                else if (stringValuesContainWords(stored.value.value, words)
+                else if (inCollection(stored.value, collection)
+                    && stringValuesContainWords(stored.value.value, words)
                     && query.test(stored.value.value)) {
                     const validationErrors = await schema.validate(stored.value.value);
                     if (validationErrors.length > 0) {
@@ -435,7 +474,10 @@ export class Autofile {
             const linkErrors = await missingLinkErrors(this.#dataFolder, this.#typeFolder, schema, value, sourceUri);
             if (linkErrors.length > 0)
                 throw new AutofileError(linkErrors);
-            await mkdir(join(this.#dataFolder, type), { recursive: true });
+            if (options.collections !== undefined) {
+                await this.#checkCollections(options.collections, sourceUri);
+            }
+            await mkdir(recordFolder(this.#dataFolder, type), { recursive: true });
             let key = options.key;
             if (key === undefined) {
                 do
@@ -450,12 +492,12 @@ export class Autofile {
             if (blobErrors.length > 0)
                 throw new AutofileError(blobErrors);
             const now = new Date().toISOString();
-            const stored = { createdAt: now, updatedAt: now, value };
+            const stored = storedRecord(now, now, options.collections, value);
             await writeJsonAtomically(recordPath(this.#dataFolder, type, key), stored);
             return recordResult(type, key, declaration.value, stored);
         });
     }
-    async #updateRecord(uri, patch) {
+    async #updateRecord(uri, patch, collections) {
         return this.#withWriteLock(async () => {
             const parsed = parseRecordUri(uri);
             if (parsed === undefined)
@@ -484,12 +526,10 @@ export class Autofile {
             ];
             if (errors.length > 0)
                 throw new AutofileError(errors);
+            if (collections !== undefined)
+                await this.#checkCollections(collections, uri);
             const previousBlobs = await blobsHeldByRecord(stored.value.value, schema);
-            const updated = {
-                createdAt: stored.value.createdAt,
-                updatedAt: new Date().toISOString(),
-                value,
-            };
+            const updated = storedRecord(stored.value.createdAt, timestampAfter(stored.value.updatedAt), collections ?? stored.value.collections, value);
             await writeJsonAtomically(path, updated);
             await deleteUnheldBlobs(this.#dataFolder, this.#typeFolder, previousBlobs);
             return recordResult(parsed.type, parsed.key, declaration.value, updated);
@@ -524,25 +564,19 @@ export class Autofile {
             catch (error) {
                 errors.push({ uri: targetUri, message: recordReadError(error) });
             }
-            const typeEntries = (await readDirectory(this.#typeFolder))
-                .filter((entry) => (entry.isFile()
-                && entry.name.endsWith(".json")
-                && typeNamePattern.test(entry.name.slice(0, -".json".length))))
-                .sort((left, right) => left.name.localeCompare(right.name));
-            for (const typeEntry of typeEntries) {
-                const type = typeEntry.name.slice(0, -".json".length);
+            for (const type of await storedTypeNames(this.#typeFolder)) {
                 const otherDeclaration = await readValidDeclaration(this.#typeFolder, type);
                 if (otherDeclaration === undefined)
                     continue;
                 const schema = await preparedDeclarationSchema(otherDeclaration, typeUri(type));
-                for (const entry of await recognizedRecordEntries(join(this.#dataFolder, type))) {
+                for (const entry of await recognizedRecordEntries(recordFolder(this.#dataFolder, type))) {
                     const key = entry.name.slice(0, -".json".length);
                     const uri = recordUri(type, key);
                     if (uri === targetUri)
                         continue;
                     let stored;
                     try {
-                        stored = await readStoredRecord(join(this.#dataFolder, type, entry.name));
+                        stored = await readStoredRecord(join(recordFolder(this.#dataFolder, type), entry.name));
                     }
                     catch (error) {
                         errors.push({ uri, message: recordReadError(error) });
@@ -555,34 +589,37 @@ export class Autofile {
                         });
                         continue;
                     }
-                    const validationErrors = await schema.validate(stored.value.value);
+                    const collections = stored.value.collections?.filter((collection) => collection !== targetUri);
+                    const leavesCollection = collections?.length !== stored.value.collections?.length;
+                    let value = stored.value.value;
+                    const validationErrors = await schema.validate(value);
                     if (validationErrors.length > 0) {
-                        if (containsStringValue(stored.value.value, targetUri)) {
+                        if (containsStringValue(value, targetUri)) {
                             errors.push({
                                 uri,
                                 message: `record does not match its type and holds ${targetUri} as a string value`,
                             });
+                            continue;
                         }
-                        continue;
                     }
-                    const pointers = schema.formatFields(stored.value.value, "uri")
-                        .filter((field) => field.value === targetUri)
-                        .map(({ pointer }) => pointer);
-                    if (pointers.length === 0)
-                        continue;
-                    const value = removeFields(stored.value.value, pointers);
-                    const remainingErrors = await schema.validate(value);
-                    if (remainingErrors.length > 0) {
-                        errors.push({ uri, message: remainingErrors.join("; ") });
-                        continue;
+                    else {
+                        const pointers = schema.formatFields(value, "uri")
+                            .filter((field) => field.value === targetUri)
+                            .map(({ pointer }) => pointer);
+                        if (pointers.length > 0) {
+                            value = removeFields(value, pointers);
+                            const remainingErrors = await schema.validate(value);
+                            if (remainingErrors.length > 0) {
+                                errors.push({ uri, message: remainingErrors.join("; ") });
+                                continue;
+                            }
+                        }
                     }
+                    if (value === stored.value.value && !leavesCollection)
+                        continue;
                     changes.push({
-                        path: join(this.#dataFolder, type, entry.name),
-                        record: {
-                            createdAt: stored.value.createdAt,
-                            updatedAt: timestampAfter(stored.value.updatedAt),
-                            value,
-                        },
+                        path: join(recordFolder(this.#dataFolder, type), entry.name),
+                        record: storedRecord(stored.value.createdAt, timestampAfter(stored.value.updatedAt), collections, value),
                     });
                 }
             }
@@ -612,20 +649,41 @@ export class Autofile {
             await deleteUnheldBlobs(this.#dataFolder, this.#typeFolder, previousBlobs);
         });
     }
+    /** Refuses a collection list naming anything but existing collections, or placing a collection inside itself. */
+    async #checkCollections(collections, uri) {
+        for (const collection of collections) {
+            if (!await recordExists(this.#dataFolder, this.#typeFolder, collection)) {
+                refuse(`collections: ${collection} is not a collection that exists`, uri);
+            }
+        }
+        if (uri === undefined || parseRecordUri(uri)?.type !== COLLECTION_TYPE)
+            return;
+        const visited = new Set();
+        const pending = [...collections];
+        while (pending.length > 0) {
+            const current = pending.pop();
+            if (current === uri)
+                refuse(`collections: would put ${uri} inside itself`, uri);
+            if (visited.has(current))
+                continue;
+            visited.add(current);
+            const parsed = parseRecordUri(current);
+            if (parsed === undefined)
+                continue;
+            const stored = await readStoredRecord(recordPath(this.#dataFolder, parsed.type, parsed.key));
+            if (stored.status === "found")
+                pending.push(...stored.value.collections ?? []);
+        }
+    }
     async #listTypes() {
         await this.#ensureReady();
         const types = [];
-        for (const entry of await readDirectory(this.#typeFolder)) {
-            if (!entry.isFile() || !entry.name.endsWith(".json"))
-                continue;
-            const type = entry.name.slice(0, -".json".length);
-            if (!typeNamePattern.test(type))
-                continue;
+        for (const type of await storedTypeNames(this.#typeFolder)) {
             const declaration = await readValidDeclaration(this.#typeFolder, type);
             if (declaration !== undefined)
                 types.push(typeResult(type, declaration));
         }
-        return types.sort((left, right) => left.type.localeCompare(right.type));
+        return types;
     }
     async #getType(type) {
         await this.#ensureReady();
@@ -642,6 +700,8 @@ export class Autofile {
     async #createType(type, definition) {
         if (typeof type !== "string" || !typeNamePattern.test(type))
             return refuse(invalidTypeMessage);
+        if (type === COLLECTION_TYPE)
+            return refuse(builtInTypeMessage, typeUri(type));
         if (!isJsonObject(definition))
             return refuse("declaration: must be an object");
         const { description, titleField } = definition;
@@ -669,7 +729,7 @@ export class Autofile {
             await prepareDeclarationSchema(declaration.schema, typeUri(type));
             await Promise.all([
                 mkdir(this.#typeFolder, { recursive: true }),
-                mkdir(join(this.#dataFolder, type), { recursive: true }),
+                mkdir(recordFolder(this.#dataFolder, type), { recursive: true }),
             ]);
             await writeJsonAtomically(declarationPath, declaration);
             return typeResult(type, declaration);
@@ -678,6 +738,8 @@ export class Autofile {
     async #updateType(type, changes) {
         if (typeof type !== "string" || !typeNamePattern.test(type))
             return refuse(invalidTypeMessage);
+        if (type === COLLECTION_TYPE)
+            return refuse(builtInTypeMessage, typeUri(type));
         if (!isJsonObject(changes))
             return refuse("changes: must be an object");
         const { description, titleField: changedTitleField, schema: changedSchema } = changes;
@@ -723,11 +785,13 @@ export class Autofile {
         return this.#withWriteLock(async () => {
             if (typeof type !== "string" || !typeNamePattern.test(type))
                 return refuse(invalidTypeMessage);
+            if (type === COLLECTION_TYPE)
+                return refuse(builtInTypeMessage, typeUri(type));
             const declaration = await readDeclaration(this.#typeFolder, type);
             if (declaration.status !== "found")
                 return declarationRefusal(type, declaration);
             try {
-                await rmdir(join(this.#dataFolder, type));
+                await rmdir(recordFolder(this.#dataFolder, type));
             }
             catch (error) {
                 // Keep the declaration whenever any stored data remains, including
@@ -746,9 +810,53 @@ export class Autofile {
         const name = blobNameFromUri(uri);
         if (name === undefined || !await isDirectory(this.#blobFolder))
             return undefined;
-        const bytes = await readRegularFile(join(this.#blobFolder, name), MAX_BLOB_BYTES);
+        const bytes = await readRegularFile(join(this.#blobFolder, name));
         return bytes === undefined ? undefined : { bytes, mimeType: mimeTypeForBlob(name) };
     }
+    async #statBlob(uri) {
+        await this.#ensureReady();
+        const name = blobNameFromUri(uri);
+        if (name === undefined || !await isDirectory(this.#blobFolder))
+            return undefined;
+        const path = join(this.#blobFolder, name);
+        try {
+            const details = await lstat(path);
+            return details.isFile() ? { size: details.size, mimeType: mimeTypeForBlob(name), path } : undefined;
+        }
+        catch (error) {
+            if (hasCode(error, "ENOENT") || hasCode(error, "ENOTDIR"))
+                return undefined;
+            throw error;
+        }
+    }
+    async #streamBlob(uri, range) {
+        await this.#ensureReady();
+        const name = blobNameFromUri(uri);
+        if (name === undefined || !await isDirectory(this.#blobFolder))
+            return undefined;
+        // Read caller-controlled offsets before opening a file: an accessor may throw.
+        const start = range === undefined ? 0 : range?.start;
+        const requestedEnd = range?.end;
+        const file = await openRegularFile(join(this.#blobFolder, name));
+        if (file === undefined)
+            return undefined;
+        const end = range === undefined ? file.size : requestedEnd;
+        if (typeof start !== "number" || typeof end !== "number"
+            || !Number.isSafeInteger(start) || !Number.isSafeInteger(end)
+            || start < 0 || end < start || end > file.size) {
+            await file.handle.close();
+            return refuse("range: must contain integer offsets with 0 <= start <= end <= blob size", uri);
+        }
+        return streamFile(file.handle, start, end);
+    }
+}
+/** Copies one chunk before asynchronous writes can observe caller mutations. */
+async function* snapshotBlobChunks(chunks) {
+    for await (const chunk of chunks) {
+        if (!(chunk instanceof Uint8Array))
+            refuse("chunks: each chunk must be a Uint8Array");
+        yield Buffer.from(chunk);
+    }
 }
 function recordResult(type, key, declaration, stored) {
     const title = declaration.titleField === undefined ? undefined : stored.value[declaration.titleField];
@@ -850,23 +958,20 @@ async function deleteUnheldBlobs(dataFolder, typeFolder, candidates) {
     const held = await blobsHeldInStore(dataFolder, typeFolder);
     if (held === undefined)
         return;
-    await deleteBlobUris(join(dataFolder, "_blobs"), [...candidates].filter((uri) => !held.has(uri)));
+    await deleteBlobUris(join(dataFolder, "blobs"), [...candidates].filter((uri) => !held.has(uri)));
 }
 async function blobsHeldInStore(dataFolder, typeFolder) {
     const held = new Set();
-    let typeEntries;
+    let types;
     try {
-        typeEntries = (await readDirectory(typeFolder)).filter((entry) => (entry.isFile() && entry.name.endsWith(".json")));
+        types = await storedTypeNames(typeFolder);
     }
     catch (error) {
         if (hasErrorCode(error))
             return undefined;
         throw error;
     }
-    for (const typeEntry of typeEntries) {
-        const type = typeEntry.name.slice(0, -".json".length);
-        if (!typeNamePattern.test(type))
-            continue;
+    for (const type of types) {
         let declaration;
         try {
             declaration = await readDeclaration(typeFolder, type);
@@ -888,7 +993,7 @@ async function blobsHeldInStore(dataFolder, typeFolder) {
         }
         let recordEntries;
         try {
-            recordEntries = await recognizedRecordEntries(join(dataFolder, type));
+            recordEntries = await recognizedRecordEntries(recordFolder(dataFolder, type));
         }
         catch (error) {
             if (hasErrorCode(error))
@@ -898,7 +1003,7 @@ async function blobsHeldInStore(dataFolder, typeFolder) {
         for (const entry of recordEntries) {
             let record;
             try {
-                record = await readJson(join(dataFolder, type, entry.name), "record");
+                record = await readJson(join(recordFolder(dataFolder, type), entry.name), "record");
             }
             catch (error) {
                 if (hasErrorCode(error))
@@ -965,13 +1070,13 @@ async function readValidDeclaration(typeFolder, type) {
 }
 async function invalidRecords(dataFolder, type, schema) {
     const errors = [];
-    const entries = await recognizedRecordEntries(join(dataFolder, type));
+    const entries = await recognizedRecordEntries(recordFolder(dataFolder, type));
     for (const entry of entries) {
         const key = entry.name.slice(0, -".json".length);
         const uri = recordUri(type, key);
         let record;
         try {
-            record = await readStoredRecord(join(dataFolder, type, entry.name));
+            record = await readStoredRecord(join(recordFolder(dataFolder, type), entry.name));
         }
         catch (error) {
             errors.push({ uri, message: recordReadError(error) });
@@ -988,6 +1093,8 @@ async function invalidRecords(dataFolder, type, schema) {
     return errors;
 }
 async function readDeclaration(folder, type) {
+    if (type === COLLECTION_TYPE)
+        return { status: "found", value: collectionDefinition };
     if (!await isDirectory(folder))
         return { status: "missing" };
     const result = await readJson(join(folder, `${type}.json`), "type declaration");
@@ -999,7 +1106,7 @@ async function readDeclaration(folder, type) {
     return { status: "found", value: result.value };
 }
 async function readStoredRecord(path) {
-    if (!await isDirectory(dirname(path)))
+    if (!await isDirectory(dirname(dirname(path))) || !await isDirectory(dirname(path)))
         return { status: "missing" };
     const result = await readJson(path, "record");
     if (result.status !== "found")
@@ -1084,16 +1191,19 @@ function parseRecordUri(uri) {
     return { type: match[1], key: match[2] };
 }
 function recordUri(type, key) {
-    return `autofile:///${type}/${key}`;
+    return `autofile:///records/${type}/${key}`;
 }
 function typeUri(type) {
-    return `autofile:///_types/${type}`;
+    return `autofile:///types/${type}`;
 }
 function mimeTypeForBlob(name) {
     return lookupMimeType(name) || "application/octet-stream";
 }
 function recordPath(dataFolder, type, key) {
-    return join(dataFolder, type, `${key}.json`);
+    return join(recordFolder(dataFolder, type), `${key}.json`);
+}
+function recordFolder(dataFolder, type) {
+    return join(dataFolder, "records", type);
 }
 function encodeRecordCursor(uri) {
     return Buffer.from(uri, "utf8").toString("base64url");
@@ -1253,6 +1363,8 @@ function recordReadRefusal(uri, result) {
     throw new Error("A found record is not a refusal");
 }
 async function recognizedRecordEntries(folder) {
+    if (!await isDirectory(dirname(folder)))
+        return [];
     return (await readDirectory(folder))
         .filter((entry) => {
         if (!entry.isFile() || !entry.name.endsWith(".json"))
@@ -1321,15 +1433,45 @@ async function openRegularFile(path) {
         throw error;
     }
 }
-async function readRegularFile(path, maximumBytes = Number.POSITIVE_INFINITY) {
+/** Keeps a file open only while a bounded, backpressured read is active. */
+function streamFile(handle, start, end) {
+    let position = start;
+    let closing;
+    const close = () => closing ??= handle.close().catch(() => { throw new Error("Storage operation failed"); });
+    return new ReadableStream({
+        async pull(controller) {
+            try {
+                if (position === end) {
+                    await close();
+                    controller.close();
+                    return;
+                }
+                const bytes = Buffer.allocUnsafe(Math.min(64 * 1024, end - position));
+                const { bytesRead } = await handle.read(bytes, 0, bytes.length, position);
+                if (closing !== undefined)
+                    return;
+                if (bytesRead === 0) {
+                    await close();
+                    controller.close();
+                    return;
+                }
+                position += bytesRead;
+                controller.enqueue(bytes.subarray(0, bytesRead));
+            }
+            catch {
+                await close();
+                controller.error(new Error("Storage operation failed"));
+            }
+        },
+        cancel: () => close(),
+    });
+}
+async function readRegularFile(path) {
     const file = await openRegularFile(path);
     if (file === undefined)
         return undefined;
     try {
-        if (file.size > maximumBytes)
-            return undefined;
-        const bytes = await file.handle.readFile();
-        return bytes.byteLength <= maximumBytes ? bytes : undefined;
+        return await file.handle.readFile();
     }
     finally {
         await file.handle.close();
@@ -1362,12 +1504,60 @@ function isTypeDefinition(value) {
 function isStoredRecord(value) {
     if (!isJsonObject(value))
         return false;
-    if (Object.keys(value).length !== 3)
+    const hasCollections = Object.hasOwn(value, "collections");
+    if (Object.keys(value).length !== (hasCollections ? 4 : 3))
         return false;
     if (!["createdAt", "updatedAt", "value"].every((key) => Object.hasOwn(value, key)))
         return false;
+    if (hasCollections && (collectionsProblem(value.collections) !== undefined || value.collections.length === 0)) {
+        return false;
+    }
     return isTimestamp(value.createdAt) && isTimestamp(value.updatedAt) && isJsonObject(value.value);
 }
+/** Says what is wrong with a list of collection URIs, ignoring whether they exist. */
+function collectionsProblem(value) {
+    if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+        return "collections: must be an array of collection URIs";
+    }
+    const other = value.find((item) => !collectionUriPattern.test(item));
+    if (other !== undefined)
+        return `collections: ${other} is not a collection URI`;
+    if (new Set(value).size !== value.length)
+        return "collections: must not list a collection more than once";
+    return undefined;
+}
+/** Checks and copies a caller's collection list before asynchronous work. */
+function snapshotCollections(value) {
+    if (value === undefined)
+        return undefined;
+    const problem = collectionsProblem(value);
+    if (problem !== undefined)
+        refuse(problem);
+    return [...value];
+}
+/** Builds a stored record, leaving out an empty collection list. */
+function storedRecord(createdAt, updatedAt, collections, value) {
+    return { createdAt, updatedAt, ...(collections?.length ? { collections } : {}), value };
+}
+function inCollection(record, collection) {
+    if (collection === undefined)
+        return true;
+    if (collection === null)
+        return record.collections === undefined;
+    return record.collections?.includes(collection) === true;
+}
+/** Every type with a declaration file, plus the built-in collection type, ordered by name. */
+async function storedTypeNames(typeFolder) {
+    const names = new Set([COLLECTION_TYPE]);
+    for (const entry of await readDirectory(typeFolder)) {
+        if (!entry.isFile() || !entry.name.endsWith(".json"))
+            continue;
+        const type = entry.name.slice(0, -".json".length);
+        if (typeNamePattern.test(type))
+            names.add(type);
+    }
+    return [...names].sort((left, right) => left.localeCompare(right));
+}
 function isTimestamp(value) {
     if (typeof value !== "string" || !timestampPattern.test(value))
         return false;