@incoqnito.io/bajadab 1.2.2 → 1.4.0

package/src/main/fileio.ts

@@ -1,207 +1,305 @@
-import { IBCollectionConfig, IBDatabase, IBDatabaseConfig, IBFunctionRegistry, IBSnapshotOptions, } from "./base.js";
-import { BCollection, BDatabase, DEFAULT_DATABASE_CONFIG, } from "./basics.js";
-import { uuidv7 } from "uuidv7";
-import { access, chmod, constants, cp, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-
-interface IStoredDatabase {
-    config: IBDatabaseConfig;
-    nameIDMap: { [k: string]: string; };
-}
-
-interface IStoredCollection<T> {
-    collectionID: string;
-    collectionName: string;
-    collectionConfig: IBCollectionConfig;
-    path: string;
-}
-
-const DB_META_FILE_NAME = "dbmeta.json";
-const DB_COLLECTION_FOLDER_SUFFIX = "_data";
-const DB_COLLECTION_META_FILE_SUFFIX = ".json";
-const DB_COLLECTION_DATA_FILE = "data.json";
-const RESTRICTED_FILE_MODE = 0o600;
-const RESTRICTED_DIR_MODE = 0o700;
-const SNAPSHOT_VERSION_PATTERN = /^\d{8}T\d{9}Z(_[A-Za-z0-9_-]+)?$/;
-const SNAPSHOT_LABEL_PATTERN = /^[A-Za-z0-9_-]+$/;
-
-function snapshotVersionName(label?: string): string {
-    const timestamp = new Date().toISOString().replace(/[-:.]/g, "");
-    return label ? `${timestamp}_${label}` : timestamp;
-}
-
-// re-applies the restricted mode explicitly rather than relying on fs.cp() to have preserved it
-async function restrictPermissions(path: string): Promise<void> {
-    const info = await stat(path);
-    if (info.isDirectory()) {
-        await chmod(path, RESTRICTED_DIR_MODE);
-        await Promise.all((await readdir(path)).map(entry => restrictPermissions(join(path, entry))));
-    }
-    else {
-        await chmod(path, RESTRICTED_FILE_MODE);
-    }
-}
-
-async function atomicWriteFile(targetPath: string, data: string): Promise<void> {
-    const tempPath = `${targetPath}.${uuidv7()}.tmp`;
-    await writeFile(tempPath, data, { encoding: "utf-8", mode: RESTRICTED_FILE_MODE });
-    await rename(tempPath, targetPath);
-}
-
-export class IODatabase {
-    readonly basePath: string;
-    private readonly nameIDMap: Map<string, string>;
-    private databaseConfig: IBDatabaseConfig;
-
-    public constructor(basePath: string, databaseConfig: IBDatabaseConfig, nameIDMap?: Map<string, string>) {
-        this.basePath = basePath;
-        this.databaseConfig = databaseConfig;
-        this.nameIDMap = nameIDMap ?? new Map();
-    }
-
-    private async loadCollectionMeta(id: string, database: BDatabase): Promise<BCollection> {
-        const meta: IStoredCollection<any> = JSON.parse(await readFile(join(this.basePath, id + DB_COLLECTION_META_FILE_SUFFIX), { encoding: "utf-8" }));
-        return new BCollection(database, id, meta.collectionName, meta.collectionConfig);
-    }
-
-    public static async createOrLoad(basePath: string, functionRegistry?: IBFunctionRegistry): Promise<BDatabase> {
-        const metaPath = join(basePath, DB_META_FILE_NAME);
-        let data: string;
-        try {
-            await access(metaPath, constants.F_OK);
-            data = await readFile(metaPath, { encoding: "utf-8" })
-        }
-        catch (err: any) {
-            if (err.code === 'ENOENT') {
-                const iodb = new IODatabase(basePath, DEFAULT_DATABASE_CONFIG);
-                return new BDatabase(iodb, DEFAULT_DATABASE_CONFIG, functionRegistry);
-            }
-            throw err;
-        }
-        const meta: IStoredDatabase = JSON.parse(data);
-        const nameIDMap = new Map(Object.entries(meta.nameIDMap));
-        const iodb = new IODatabase(basePath, meta.config, nameIDMap);
-        const database = new BDatabase(iodb, meta.config, functionRegistry);
-        const collections = await Promise.all(
-            nameIDMap.values()
-                .map(id => iodb.loadCollectionMeta(id, database))
-                .toArray());
-        collections.forEach(c => database.addCollection(c));
-        return database;
-    }
-
-    public async flush(): Promise<void> {
-        const meta: IStoredDatabase = {
-            config: this.databaseConfig,
-            nameIDMap: Object.fromEntries(this.nameIDMap),
-        };
-        const data = JSON.stringify(meta);
-        await atomicWriteFile(join(this.basePath, DB_META_FILE_NAME), data);
-    }
-
-    public async createCollection<T = any>(collection: BCollection<T>): Promise<BCollection<T>> {
-        const id = collection.collectionID;
-        const collectionPath = join(this.basePath, id + DB_COLLECTION_FOLDER_SUFFIX);
-        await mkdir(collectionPath, { recursive: true, mode: RESTRICTED_DIR_MODE });
-        const meta: IStoredCollection<T> = {
-            collectionConfig: collection.collectionConfig,
-            collectionName: collection.collectionName,
-            collectionID: id,
-            path: collectionPath,
-        };
-        const data = JSON.stringify(meta);
-        await atomicWriteFile(join(this.basePath, id + DB_COLLECTION_META_FILE_SUFFIX), data);
-        this.nameIDMap.set(collection.collectionName, id);
-        await this.flush();
-        return collection;
-    }
-
-    public async deleteCollection(collectionName: string): Promise<void> {
-        const id = this.nameIDMap.get(collectionName);
-        if (!id) {
-            throw new Error(`no such database ${collectionName}`);
-        }
-        const collectionPath = join(this.basePath, id + DB_COLLECTION_FOLDER_SUFFIX);
-        await rm(collectionPath, { recursive: true, force: true });
-        const metaPath = join(this.basePath, id + DB_COLLECTION_META_FILE_SUFFIX);
-        await rm(metaPath, { force: true });
-        this.nameIDMap.delete(collectionName);
-    }
-
-    private async loadData(collectionID: string): Promise<any> {
-        try {
-            const data = await readFile(join(this.basePath, collectionID + DB_COLLECTION_FOLDER_SUFFIX, DB_COLLECTION_DATA_FILE), { encoding: "utf-8" });
-            return JSON.parse(data);
-        }
-        catch (err: any) {
-            if (err.code === 'ENOENT') {
-                return null;
-            }
-            throw err;
-        }
-    }
-
-    public async loadArrayData<T>(collectionID: string): Promise<T[]> {
-        const data = await this.loadData(collectionID);
-        return data || [];
-    }
-
-    public async loadMapData<T>(collectionID: string): Promise<Map<string, T>> {
-        const data = await this.loadData(collectionID);
-        return data ? new Map(Object.entries(data)) : new Map();
-    }
-
-    private async writeData(collectionID: string, data: any): Promise<void> {
-        return atomicWriteFile(join(this.basePath, collectionID + DB_COLLECTION_FOLDER_SUFFIX, DB_COLLECTION_DATA_FILE), JSON.stringify(data));
-    }
-
-    public async writeArrayData<T = any>(collectionID: string, arrayData: T[]): Promise<void> {
-        return this.writeData(collectionID, arrayData);
-    }
-
-    public async writeMapData<T = any>(collectionID: string, mapData: Map<string, T>): Promise<void> {
-        return this.writeData(collectionID, Object.fromEntries(mapData));
-    }
-
-    public async snapshot(targetDir: string, options?: IBSnapshotOptions): Promise<string> {
-        if (options?.keepLast !== undefined && options.keepLast < 0) {
-            throw new Error(`keepLast must not be negative, got ${options.keepLast}`);
-        }
-        if (options?.label !== undefined && !SNAPSHOT_LABEL_PATTERN.test(options.label)) {
-            throw new Error(`snapshot label "${options.label}" must only contain letters, digits, "-" or "_"`);
-        }
-        await mkdir(targetDir, { recursive: true, mode: RESTRICTED_DIR_MODE });
-        const versionPath = join(targetDir, snapshotVersionName(options?.label));
-        try {
-            await cp(this.basePath, versionPath, { recursive: true });
-            await restrictPermissions(versionPath);
-        }
-        catch (err) {
-            await rm(versionPath, { recursive: true, force: true });
-            throw err;
-        }
-        if (options?.keepLast !== undefined) {
-            await this.pruneSnapshots(targetDir, options.keepLast);
-        }
-        return versionPath;
-    }
-
-    // only ever deletes folders matching our own naming scheme, never anything else the caller keeps in targetDir
-    private async pruneSnapshots(targetDir: string, keepLast: number): Promise<void> {
-        const entries = await readdir(targetDir, { withFileTypes: true });
-        const versions = entries
-            .filter(e => e.isDirectory() && SNAPSHOT_VERSION_PATTERN.test(e.name))
-            .map(e => e.name)
-            .sort();
-        const toDelete = versions.slice(0, Math.max(0, versions.length - (keepLast + 1)));
-        await Promise.all(toDelete.map(name => rm(join(targetDir, name), { recursive: true, force: true })));
-    }
-}
-
-/**
- * Opens the database at `basePath`, creating it if it doesn't exist yet.
- * `functionRegistry` resolves any {@link IBCollectionConfig.getID}/`setID` keys used by its collections.
- */
-export async function getDatabase(basePath: string, functionRegistry?: IBFunctionRegistry): Promise<IBDatabase> {
-    return IODatabase.createOrLoad(basePath, functionRegistry);
-}
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import { chmod, cp, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
+import { setTimeout } from "node:timers/promises";
+import { uuidv7 } from "uuidv7";
+
+import { IBCollectionConfig, IBDatabase, IBDatabaseConfig, IBFunctionRegistry, IBSnapshotOptions, } from "./base.js";
+import { BCollection, BDatabase, DEFAULT_DATABASE_CONFIG, } from "./basics.js";
+
+interface IStoredDatabase {
+    config: IBDatabaseConfig;
+    nameIDMap: { [k: string]: string; };
+}
+
+interface IStoredCollection {
+    collectionID: string;
+    collectionName: string;
+    collectionConfig: IBCollectionConfig;
+}
+
+const DB_META_FILE_NAME = "dbmeta.json";
+const DB_COLLECTION_FOLDER_SUFFIX = "_data";
+const DB_COLLECTION_META_FILE_SUFFIX = ".json";
+const DB_COLLECTION_DATA_FILE = "data.json";
+const RESTRICTED_FILE_MODE = 0o600;
+const RESTRICTED_DIR_MODE = 0o700;
+const SNAPSHOT_VERSION_PATTERN = /^\d{8}T\d{9}Z(_[A-Za-z0-9_-]+)?$/;
+const SNAPSHOT_LABEL_PATTERN = /^[A-Za-z0-9_-]+$/;
+const RENAME_RETRY_DELAYS_MS = [10, 25, 50, 100];
+const SNAPSHOT_NAME_ATTEMPTS = 100;
+
+enum FsErrorCode {
+    ENOENT = "ENOENT",
+    EPERM = "EPERM",
+    EBUSY = "EBUSY",
+    EACCES = "EACCES",
+    EEXIST = "EEXIST",
+}
+
+const RETRYABLE_RENAME_ERRORS: ReadonlySet<string> =
+    new Set([FsErrorCode.EPERM, FsErrorCode.EBUSY, FsErrorCode.EACCES]);
+
+export interface IPreparedWrite {
+    commit(): Promise<void>;
+    discard(): Promise<void>;
+}
+
+function isSameOrInside(path: string, dir: string): boolean {
+    const rel = relative(resolve(dir), resolve(path));
+    return !isAbsolute(rel) && rel !== ".." && !rel.startsWith(".." + sep);
+}
+
+let lastSnapshotMs = 0;
+
+// strictly increasing within the process, so consecutive snapshots never share a name
+function nextSnapshotMs(): number {
+    lastSnapshotMs = Math.max(Date.now(), lastSnapshotMs + 1);
+    return lastSnapshotMs;
+}
+
+function snapshotVersionName(ms: number, label?: string): string {
+    const timestamp = new Date(ms).toISOString().replace(/[-:.]/g, "");
+    return label ? `${timestamp}_${label}` : timestamp;
+}
+
+// exclusive mkdir claims the name atomically, also against other processes using the same targetDir
+async function createSnapshotVersionDir(targetDir: string, label?: string): Promise<string> {
+    for (let attempt = 1; ; attempt++) {
+        const versionName = snapshotVersionName(nextSnapshotMs(), label);
+        try {
+            await mkdir(join(targetDir, versionName), { mode: RESTRICTED_DIR_MODE });
+            return versionName;
+        }
+        catch (err: any) {
+            if (err.code !== FsErrorCode.EEXIST || attempt >= SNAPSHOT_NAME_ATTEMPTS) {
+                throw err;
+            }
+        }
+    }
+}
+
+// re-applies the restricted mode explicitly rather than relying on fs.cp() to have preserved it
+async function restrictPermissions(path: string): Promise<void> {
+    const info = await stat(path);
+    if (info.isDirectory()) {
+        await chmod(path, RESTRICTED_DIR_MODE);
+        await Promise.all((await readdir(path)).map(entry => restrictPermissions(join(path, entry))));
+    }
+    else {
+        await chmod(path, RESTRICTED_FILE_MODE);
+    }
+}
+
+async function renameWithRetry(from: string, to: string): Promise<void> {
+    for (let attempt = 0; ; attempt++) {
+        try {
+            return await rename(from, to);
+        }
+        catch (err: any) {
+            if (attempt >= RENAME_RETRY_DELAYS_MS.length || !RETRYABLE_RENAME_ERRORS.has(err.code)) {
+                throw err;
+            }
+            await setTimeout(RENAME_RETRY_DELAYS_MS[attempt]);
+        }
+    }
+}
+
+async function prepareAtomicWrite(targetPath: string, data: string): Promise<IPreparedWrite> {
+    const tempPath = `${targetPath}.${uuidv7()}.tmp`;
+    try {
+        await writeFile(tempPath, data, { encoding: "utf-8", mode: RESTRICTED_FILE_MODE });
+    }
+    catch (err) {
+        await rm(tempPath, { force: true });
+        throw err;
+    }
+    return {
+        commit: async () => {
+            try {
+                await renameWithRetry(tempPath, targetPath);
+            }
+            catch (err) {
+                await rm(tempPath, { force: true });
+                throw err;
+            }
+        },
+        discard: () => rm(tempPath, { force: true }),
+    };
+}
+
+async function atomicWriteFile(targetPath: string, data: string): Promise<void> {
+    return (await prepareAtomicWrite(targetPath, data)).commit();
+}
+
+export class IODatabase {
+    readonly basePath: string;
+    private readonly nameIDMap: Map<string, string>;
+    private databaseConfig: IBDatabaseConfig;
+
+    public constructor(basePath: string, databaseConfig: IBDatabaseConfig, nameIDMap?: Map<string, string>) {
+        this.basePath = basePath;
+        this.databaseConfig = databaseConfig;
+        this.nameIDMap = nameIDMap ?? new Map();
+    }
+
+    private async loadCollectionMeta(id: string, database: BDatabase): Promise<BCollection> {
+        const meta: IStoredCollection = JSON.parse(await readFile(join(this.basePath, id + DB_COLLECTION_META_FILE_SUFFIX), { encoding: "utf-8" }));
+        return new BCollection(database, id, meta.collectionName, meta.collectionConfig);
+    }
+
+    public static async createOrLoad(basePath: string, functionRegistry?: IBFunctionRegistry): Promise<BDatabase> {
+        const metaPath = join(basePath, DB_META_FILE_NAME);
+        let data: string;
+        try {
+            data = await readFile(metaPath, { encoding: "utf-8" })
+        }
+        catch (err: any) {
+            if (err.code === FsErrorCode.ENOENT) {
+                const iodb = new IODatabase(basePath, DEFAULT_DATABASE_CONFIG);
+                return new BDatabase(iodb, DEFAULT_DATABASE_CONFIG, functionRegistry);
+            }
+            throw err;
+        }
+        const meta: IStoredDatabase = JSON.parse(data);
+        const nameIDMap = new Map(Object.entries(meta.nameIDMap));
+        const iodb = new IODatabase(basePath, meta.config, nameIDMap);
+        const database = new BDatabase(iodb, meta.config, functionRegistry);
+        const collections = await Promise.all(
+            nameIDMap.values()
+                .map(id => iodb.loadCollectionMeta(id, database))
+                .toArray());
+        collections.forEach(c => database.addCollection(c));
+        return database;
+    }
+
+    public async flush(): Promise<void> {
+        const meta: IStoredDatabase = {
+            config: this.databaseConfig,
+            nameIDMap: Object.fromEntries(this.nameIDMap),
+        };
+        const data = JSON.stringify(meta);
+        await atomicWriteFile(join(this.basePath, DB_META_FILE_NAME), data);
+    }
+
+    public async createCollection<T = any>(collection: BCollection<T>): Promise<BCollection<T>> {
+        const id = collection.collectionID;
+        const collectionPath = join(this.basePath, id + DB_COLLECTION_FOLDER_SUFFIX);
+        await mkdir(collectionPath, { recursive: true, mode: RESTRICTED_DIR_MODE });
+        const meta: IStoredCollection = {
+            collectionConfig: collection.collectionConfig,
+            collectionName: collection.collectionName,
+            collectionID: id,
+        };
+        const data = JSON.stringify(meta);
+        await atomicWriteFile(join(this.basePath, id + DB_COLLECTION_META_FILE_SUFFIX), data);
+        this.nameIDMap.set(collection.collectionName, id);
+        await this.flush();
+        return collection;
+    }
+
+    public async deleteCollection(collectionName: string): Promise<void> {
+        const id = this.nameIDMap.get(collectionName);
+        if (!id) {
+            throw new Error(`no such collection ${collectionName}`);
+        }
+        const collectionPath = join(this.basePath, id + DB_COLLECTION_FOLDER_SUFFIX);
+        const metaPath = join(this.basePath, id + DB_COLLECTION_META_FILE_SUFFIX);
+        this.nameIDMap.delete(collectionName);
+        await this.flush();
+        await rm(collectionPath, { recursive: true, force: true });
+        await rm(metaPath, { force: true });
+    }
+
+    private async loadData(collectionID: string): Promise<any> {
+        try {
+            const data = await readFile(join(this.basePath, collectionID + DB_COLLECTION_FOLDER_SUFFIX, DB_COLLECTION_DATA_FILE), { encoding: "utf-8" });
+            return JSON.parse(data);
+        }
+        catch (err: any) {
+            if (err.code === FsErrorCode.ENOENT) {
+                return null;
+            }
+            throw err;
+        }
+    }
+
+    public async loadArrayData<T>(collectionID: string): Promise<T[]> {
+        const data = await this.loadData(collectionID);
+        return data || [];
+    }
+
+    public async loadMapData<T>(collectionID: string): Promise<Map<string, T>> {
+        const data = await this.loadData(collectionID);
+        return data ? new Map(Object.entries(data)) : new Map();
+    }
+
+    private async prepareData(collectionID: string, data: any): Promise<IPreparedWrite> {
+        return prepareAtomicWrite(join(this.basePath, collectionID + DB_COLLECTION_FOLDER_SUFFIX, DB_COLLECTION_DATA_FILE), JSON.stringify(data));
+    }
+
+    private async writeData(collectionID: string, data: any): Promise<void> {
+        return (await this.prepareData(collectionID, data)).commit();
+    }
+
+    public async prepareArrayData<T = any>(collectionID: string, arrayData: T[]): Promise<IPreparedWrite> {
+        return this.prepareData(collectionID, arrayData);
+    }
+
+    public async prepareMapData<T = any>(collectionID: string, mapData: Map<string, T>): Promise<IPreparedWrite> {
+        return this.prepareData(collectionID, Object.fromEntries(mapData));
+    }
+
+    public async writeArrayData<T = any>(collectionID: string, arrayData: T[]): Promise<void> {
+        return this.writeData(collectionID, arrayData);
+    }
+
+    public async writeMapData<T = any>(collectionID: string, mapData: Map<string, T>): Promise<void> {
+        return this.writeData(collectionID, Object.fromEntries(mapData));
+    }
+
+    public async snapshot(targetDir: string, options?: IBSnapshotOptions): Promise<string> {
+        if (options?.keepLast !== undefined && options.keepLast < 0) {
+            throw new Error(`keepLast must not be negative, got ${options.keepLast}`);
+        }
+        if (options?.label !== undefined && !SNAPSHOT_LABEL_PATTERN.test(options.label)) {
+            throw new Error(`snapshot label "${options.label}" must only contain letters, digits, "-" or "_"`);
+        }
+        if (isSameOrInside(targetDir, this.basePath)) {
+            throw new Error(`snapshot target "${targetDir}" must not be inside the database directory "${this.basePath}"`);
+        }
+        await mkdir(targetDir, { recursive: true, mode: RESTRICTED_DIR_MODE });
+        const versionName = await createSnapshotVersionDir(targetDir, options?.label);
+        const versionPath = join(targetDir, versionName);
+        try {
+            await cp(this.basePath, versionPath, { recursive: true });
+            await restrictPermissions(versionPath);
+        }
+        catch (err) {
+            await rm(versionPath, { recursive: true, force: true });
+            throw err;
+        }
+        if (options?.keepLast !== undefined) {
+            await this.pruneSnapshots(targetDir, versionName, options.keepLast);
+        }
+        return versionPath;
+    }
+
+    // only ever deletes folders matching our own naming scheme, never anything else the caller keeps in targetDir
+    private async pruneSnapshots(targetDir: string, createdVersionName: string, keepLast: number): Promise<void> {
+        const entries = await readdir(targetDir, { withFileTypes: true });
+        const olderVersions = entries
+            .filter(e => e.isDirectory() && e.name !== createdVersionName && SNAPSHOT_VERSION_PATTERN.test(e.name))
+            .map(e => e.name)
+            .sort();
+        const toDelete = olderVersions.slice(0, Math.max(0, olderVersions.length - keepLast));
+        await Promise.all(toDelete.map(name => rm(join(targetDir, name), { recursive: true, force: true })));
+    }
+}
+
+/**
+ * Opens the database at `basePath`, creating it if it doesn't exist yet.
+ * `functionRegistry` resolves any {@link IBCollectionConfig.getID}/`setID` keys used by its collections.
+ */
+export async function getDatabase(basePath: string, functionRegistry?: IBFunctionRegistry): Promise<IBDatabase> {
+    return IODatabase.createOrLoad(basePath, functionRegistry);
+}