@twin.org/entity-storage-connector-file 0.0.3-next.9 → 0.9.0-next.1

package/dist/es/fileEntityStorageConnector.js

@@ -1,10 +1,11 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
-import { access, mkdir, readFile, writeFile } from "node:fs/promises";
+import { access, mkdir, readFile, rename, rm, statfs, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
-import { BaseError, Coerce, ComponentFactory, Guards, Is, ObjectHelper } from "@twin.org/core";
+import { BaseError, Coerce, ComponentFactory, GeneralError, Guards, HealthStatus, Is, Mutex, ObjectHelper, Validation } from "@twin.org/core";
 import { ComparisonOperator, EntityConditions, EntitySchemaFactory, EntitySchemaHelper, EntitySorter, LogicalOperator } from "@twin.org/entity";
+import { EntityStorageHelper } from "@twin.org/entity-storage-models";
 /**
  * Class for performing entity storage operations in file.
  */
@@ -23,6 +24,21 @@ export class FileEntityStorageConnector {
      * @internal
      */
     static _PARTITION_KEY = "partitionId";
+    /**
+     * Default disk space warning threshold: 500 MB.
+     * @internal
+     */
+    static _DEFAULT_DISK_WARNING_THRESHOLD_BYTES = 500 * 1024 * 1024;
+    /**
+     * Default disk space error threshold: 100 MB.
+     * @internal
+     */
+    static _DEFAULT_DISK_ERROR_THRESHOLD_BYTES = 100 * 1024 * 1024;
+    /**
+     * The name for the schema.
+     * @internal
+     */
+    _entitySchemaName;
     /**
      * The schema for the entity.
      * @internal
@@ -43,6 +59,21 @@ export class FileEntityStorageConnector {
      * @internal
      */
     _directory;
+    /**
+     * Free bytes below which health reports an error.
+     * @internal
+     */
+    _diskErrorThresholdBytes;
+    /**
+     * Free bytes below which health reports a warning.
+     * @internal
+     */
+    _diskWarningThresholdBytes;
+    /**
+     * Milliseconds to wait for the directory lock before throwing.
+     * @internal
+     */
+    _mutexTimeoutMs;
     /**
      * Create a new instance of FileEntityStorageConnector.
      * @param options The options for the connector.
@@ -52,32 +83,18 @@ export class FileEntityStorageConnector {
         Guards.stringValue(FileEntityStorageConnector.CLASS_NAME, "options.entitySchema", options.entitySchema);
         Guards.object(FileEntityStorageConnector.CLASS_NAME, "options.config", options.config);
         Guards.stringValue(FileEntityStorageConnector.CLASS_NAME, "options.config.directory", options.config.directory);
+        this._entitySchemaName = options.entitySchema;
         this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
         this._partitionContextIds = options.partitionContextIds;
         this._primaryKey = EntitySchemaHelper.getPrimaryKey(this._entitySchema);
         this._directory = path.resolve(options.config.directory);
-    }
-    /**
-     * Deep-clone condition tree and map `null` to `undefined` on Equals/NotEquals leaves
-     * so in-memory evaluation matches SQL-style "IS NULL" / "IS NOT NULL" semantics.
-     * @param condition The user-supplied condition (not mutated).
-     * @returns A clone safe to pass to {@link EntityConditions.check}.
-     * @internal
-     */
-    static normalizeNullToUndefined(condition) {
-        if ("conditions" in condition) {
-            return {
-                ...condition,
-                conditions: condition.conditions.map(c => FileEntityStorageConnector.normalizeNullToUndefined(c))
-            };
-        }
-        const leaf = condition;
-        if ((leaf.comparison === ComparisonOperator.Equals ||
-            leaf.comparison === ComparisonOperator.NotEquals) &&
-            leaf.value === null) {
-            return { ...leaf, value: undefined };
-        }
-        return { ...leaf };
+        this._diskErrorThresholdBytes =
+            options.config.diskErrorThresholdBytes ??
+                FileEntityStorageConnector._DEFAULT_DISK_ERROR_THRESHOLD_BYTES;
+        this._diskWarningThresholdBytes =
+            options.config.diskWarningThresholdBytes ??
+                FileEntityStorageConnector._DEFAULT_DISK_WARNING_THRESHOLD_BYTES;
+        this._mutexTimeoutMs = Coerce.integer(options.config.mutexTimeoutMs);
     }
     /**
      * Bootstrap the connector by creating and initializing any resources it needs.
@@ -138,6 +155,65 @@ export class FileEntityStorageConnector {
     className() {
         return FileEntityStorageConnector.CLASS_NAME;
     }
+    /**
+     * Returns the health status of the component.
+     * @returns The health status of the component, can return multiple entries for elements within the component.
+     */
+    async health() {
+        try {
+            const stats = await statfs(this._directory);
+            const freeBytes = stats.bavail * stats.bsize;
+            if (freeBytes < this._diskErrorThresholdBytes) {
+                return [
+                    {
+                        source: FileEntityStorageConnector.CLASS_NAME,
+                        status: HealthStatus.Error,
+                        description: "healthDescription",
+                        message: "diskSpaceError",
+                        data: {
+                            directory: this._directory,
+                            freeBytes,
+                            thresholdBytes: this._diskErrorThresholdBytes
+                        }
+                    }
+                ];
+            }
+            else if (freeBytes < this._diskWarningThresholdBytes) {
+                return [
+                    {
+                        source: FileEntityStorageConnector.CLASS_NAME,
+                        status: HealthStatus.Warning,
+                        description: "healthDescription",
+                        message: "diskSpaceWarning",
+                        data: {
+                            directory: this._directory,
+                            freeBytes,
+                            thresholdBytes: this._diskWarningThresholdBytes
+                        }
+                    }
+                ];
+            }
+            return [
+                {
+                    source: FileEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Ok,
+                    description: "healthDescription",
+                    data: { directory: this._directory, freeBytes }
+                }
+            ];
+        }
+        catch {
+            return [
+                {
+                    source: FileEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Error,
+                    description: "healthDescription",
+                    message: "diskSpaceCheckFailed",
+                    data: { directory: this._directory }
+                }
+            ];
+        }
+    }
     /**
      * Get the schema for the entities.
      * @returns The schema for the entities.
@@ -156,7 +232,7 @@ export class FileEntityStorageConnector {
         Guards.stringValue(FileEntityStorageConnector.CLASS_NAME, "id", id);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        const store = await this.readStore();
+        const store = await this.readStoreWithLock();
         const finalConditions = conditions ?? [];
         if (Is.stringValue(partitionKey)) {
             finalConditions.push({
@@ -165,11 +241,13 @@ export class FileEntityStorageConnector {
             });
         }
         const index = this.findItem(store, id, secondaryIndex, finalConditions);
-        const item = index >= 0 ? store[index] : undefined;
+        const item = store[index];
         if (Is.objectValue(item)) {
-            ObjectHelper.propertyDelete(item, FileEntityStorageConnector._PARTITION_KEY);
+            return EntityStorageHelper.unPrepareEntity(item, [
+                FileEntityStorageConnector._PARTITION_KEY
+            ]);
         }
-        return item;
+        return undefined;
     }
     /**
      * Set an entity.
@@ -181,25 +259,143 @@ export class FileEntityStorageConnector {
         Guards.object(FileEntityStorageConnector.CLASS_NAME, "entity", entity);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        EntitySchemaHelper.validateEntity(entity, this.getSchema());
-        const store = await this.readStore();
-        const finalEntity = ObjectHelper.clone(entity);
-        const finalConditions = conditions ?? [];
-        if (Is.stringValue(partitionKey)) {
-            finalConditions.push({
-                property: FileEntityStorageConnector._PARTITION_KEY,
-                value: partitionKey
+        const prepared = EntityStorageHelper.prepareEntity(entity, this._entitySchema, Is.stringValue(partitionKey)
+            ? [{ property: FileEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : undefined, { nullBehavior: "omit" });
+        await this.withLock(async () => {
+            const store = await this.readStore();
+            const finalConditions = conditions ?? [];
+            if (Is.stringValue(partitionKey)) {
+                finalConditions.push({
+                    property: FileEntityStorageConnector._PARTITION_KEY,
+                    value: partitionKey
+                });
+            }
+            const existingIndex = this.findItem(store, prepared[this._primaryKey.property], undefined, finalConditions);
+            if (existingIndex >= 0) {
+                store[existingIndex] = prepared;
+            }
+            else {
+                store.push(prepared);
+            }
+            await this.writeStore(store);
+        });
+    }
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    async setBatch(entities) {
+        Guards.arrayValue(FileEntityStorageConnector.CLASS_NAME, "entities", entities);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        await this.withLock(async () => {
+            const store = await this.readStore();
+            for (const entity of entities) {
+                Guards.object(FileEntityStorageConnector.CLASS_NAME, "entity", entity);
+                const prepared = EntityStorageHelper.prepareEntity(entity, this._entitySchema, Is.stringValue(partitionKey)
+                    ? [{ property: FileEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+                    : undefined, { nullBehavior: "omit" });
+                const existingIndex = this.findItem(store, prepared[this._primaryKey.property], undefined, Is.stringValue(partitionKey)
+                    ? [
+                        {
+                            property: FileEntityStorageConnector._PARTITION_KEY,
+                            value: partitionKey
+                        }
+                    ]
+                    : []);
+                if (existingIndex >= 0) {
+                    store[existingIndex] = prepared;
+                }
+                else {
+                    store.push(prepared);
+                }
+            }
+            await this.writeStore(store);
+        });
+    }
+    /**
+     * Remove all entities from the storage.
+     * @returns Nothing.
+     */
+    async empty() {
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        try {
+            await this.withLock(async () => {
+                const store = await this.readStore();
+                const remaining = Is.stringValue(partitionKey)
+                    ? store.filter(item => ObjectHelper.propertyGet(item, FileEntityStorageConnector._PARTITION_KEY) !== partitionKey)
+                    : [];
+                await this.writeStore(remaining);
             });
-            ObjectHelper.propertySet(finalEntity, FileEntityStorageConnector._PARTITION_KEY, partitionKey);
         }
-        const existingIndex = this.findItem(store, finalEntity[this._primaryKey.property], undefined, finalConditions);
-        if (existingIndex >= 0) {
-            store[existingIndex] = finalEntity;
+        catch (err) {
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "emptyFailed", undefined, err);
         }
-        else {
-            store.push(finalEntity);
+    }
+    /**
+     * Remove multiple entities by id.
+     * @param ids The ids of the entities to remove.
+     * @returns Nothing.
+     */
+    async removeBatch(ids) {
+        Guards.arrayValue(FileEntityStorageConnector.CLASS_NAME, "ids", ids);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        try {
+            await this.withLock(async () => {
+                const store = await this.readStore();
+                const idSet = new Set(ids);
+                const remaining = store.filter(item => {
+                    if (Is.stringValue(partitionKey) &&
+                        ObjectHelper.propertyGet(item, FileEntityStorageConnector._PARTITION_KEY) !==
+                            partitionKey) {
+                        return true;
+                    }
+                    return !idSet.has(item[this._primaryKey.property]);
+                });
+                await this.writeStore(remaining);
+            });
+        }
+        catch (err) {
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "removeBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Teardown the storage by deleting the underlying store file.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns True if the teardown process was successful.
+     */
+    async teardown(nodeLoggingComponentType) {
+        const nodeLogging = ComponentFactory.getIfExists(nodeLoggingComponentType);
+        await nodeLogging?.log({
+            level: "info",
+            source: FileEntityStorageConnector.CLASS_NAME,
+            ts: Date.now(),
+            message: "storeTearingDown"
+        });
+        try {
+            await rm(this._directory, { recursive: true, force: true });
+            await nodeLogging?.log({
+                level: "info",
+                source: FileEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "storeTornDown"
+            });
+            return true;
+        }
+        catch (err) {
+            await nodeLogging?.log({
+                level: "error",
+                source: FileEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "teardownFailed",
+                error: BaseError.fromError(err)
+            });
+            return false;
         }
-        await this.writeStore(store);
     }
     /**
      * Remove the entity.
@@ -211,19 +407,21 @@ export class FileEntityStorageConnector {
         Guards.stringValue(FileEntityStorageConnector.CLASS_NAME, "id", id);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        const store = await this.readStore();
-        const finalConditions = conditions ?? [];
-        if (Is.stringValue(partitionKey)) {
-            finalConditions.push({
-                property: FileEntityStorageConnector._PARTITION_KEY,
-                value: partitionKey
-            });
-        }
-        const index = this.findItem(store, id, undefined, finalConditions);
-        if (index >= 0) {
-            store.splice(index, 1);
-            await this.writeStore(store);
-        }
+        await this.withLock(async () => {
+            const store = await this.readStore();
+            const finalConditions = conditions ?? [];
+            if (Is.stringValue(partitionKey)) {
+                finalConditions.push({
+                    property: FileEntityStorageConnector._PARTITION_KEY,
+                    value: partitionKey
+                });
+            }
+            const index = this.findItem(store, id, undefined, finalConditions);
+            if (index >= 0) {
+                store.splice(index, 1);
+                await this.writeStore(store);
+            }
+        });
     }
     /**
      * Find all the entities which match the conditions.
@@ -238,7 +436,14 @@ export class FileEntityStorageConnector {
     async query(conditions, sortProperties, properties, cursor, limit) {
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        let allEntities = await this.readStore();
+        EntityStorageHelper.validateSortProperties(this._entitySchema, sortProperties);
+        EntityStorageHelper.validateProperties(this._entitySchema, properties);
+        if (!Is.empty(limit)) {
+            const validationFailures = [];
+            Validation.integer("limit", limit, validationFailures, undefined, { minValue: 1 });
+            Validation.asValidationError(FileEntityStorageConnector.CLASS_NAME, "query", validationFailures);
+        }
+        let allEntities = await this.readStoreWithLock();
         const finalConditions = {
             conditions: [],
             logicalOperator: LogicalOperator.And
@@ -251,7 +456,7 @@ export class FileEntityStorageConnector {
             });
         }
         if (!Is.empty(conditions)) {
-            finalConditions.conditions.push(FileEntityStorageConnector.normalizeNullToUndefined(conditions));
+            finalConditions.conditions.push(EntityStorageHelper.normalizeConditionValues(conditions));
         }
         const entities = [];
         const finalLimit = limit ?? FileEntityStorageConnector._DEFAULT_LIMIT;
@@ -263,9 +468,12 @@ export class FileEntityStorageConnector {
             for (let i = startIndex; i < allEntities.length; i++) {
                 if (EntityConditions.check(allEntities[i], finalConditions) &&
                     entities.length < finalLimit) {
-                    const entity = ObjectHelper.pick(allEntities[i], properties);
-                    ObjectHelper.propertyDelete(entity, FileEntityStorageConnector._PARTITION_KEY);
-                    entities.push(entity);
+                    const entity = Is.arrayValue(properties)
+                        ? ObjectHelper.pick(allEntities[i], properties)
+                        : allEntities[i];
+                    entities.push(EntityStorageHelper.unPrepareEntity(entity, [
+                        FileEntityStorageConnector._PARTITION_KEY
+                    ]));
                     if (entities.length >= finalLimit) {
                         if (i < allEntities.length - 1) {
                             nextCursor = (i + 1).toString();
@@ -281,22 +489,144 @@ export class FileEntityStorageConnector {
         };
     }
     /**
-     * Read the store from file.
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
+     */
+    async count(conditions) {
+        const store = await this.readStoreWithLock();
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        const finalConditions = {
+            conditions: [],
+            logicalOperator: LogicalOperator.And
+        };
+        if (Is.stringValue(partitionKey)) {
+            finalConditions.conditions.push({
+                property: FileEntityStorageConnector._PARTITION_KEY,
+                comparison: ComparisonOperator.Equals,
+                value: partitionKey
+            });
+        }
+        if (!Is.empty(conditions)) {
+            finalConditions.conditions.push(EntityStorageHelper.normalizeConditionValues(conditions));
+        }
+        if (finalConditions.conditions.length === 0) {
+            return store.length;
+        }
+        return store.filter(item => EntityConditions.check(item, finalConditions)).length;
+    }
+    /**
+     * Get a unique list of all the context ids from the storage.
+     * @returns The list of unique context ids.
+     */
+    async getPartitionContextIds() {
+        const contextIds = {};
+        const store = await this.readStoreWithLock();
+        for (const entity of store) {
+            const partitionId = ObjectHelper.propertyGet(entity, FileEntityStorageConnector._PARTITION_KEY);
+            if (Is.stringValue(partitionId)) {
+                contextIds[partitionId] = ContextIdHelper.shortSplit(this._partitionContextIds ?? [], partitionId);
+            }
+        }
+        return Object.values(contextIds);
+    }
+    /**
+     * Create the target connector for performing the migration it will use a temporary storage location.
+     * @param newEntitySchema The name of the new entity schema to create the connector for.
+     * @returns Connector for performing the migration.
+     */
+    async createTargetConnector(newEntitySchema) {
+        const baseName = path.basename(this._directory);
+        const parentDir = path.resolve(this._directory, "..");
+        const migrationDir = path.join(parentDir, `${baseName}_migration_${Date.now()}`);
+        return new FileEntityStorageConnector({
+            entitySchema: newEntitySchema,
+            partitionContextIds: this._partitionContextIds,
+            config: {
+                directory: migrationDir,
+                diskErrorThresholdBytes: this._diskErrorThresholdBytes,
+                diskWarningThresholdBytes: this._diskWarningThresholdBytes
+            }
+        });
+    }
+    /**
+     * Finalize the migration by tearing down the old connector and replacing it with the target connector.
+     * @param targetConnector The target connector to finalize the migration with.
+     * @param options The options to control how the migration is finalized.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is finalized.
+     */
+    async finalizeMigration(targetConnector, options, loggingComponentType) {
+        const originalDir = this._directory;
+        const migrationDir = targetConnector._directory;
+        // Teardown the original connector, removing the entire source directory.
+        await this.teardown(loggingComponentType);
+        // Rename the migration directory into the original location.
+        await rename(migrationDir, originalDir);
+        return new FileEntityStorageConnector({
+            entitySchema: targetConnector._entitySchemaName,
+            partitionContextIds: targetConnector._partitionContextIds,
+            config: {
+                directory: this._directory,
+                diskErrorThresholdBytes: targetConnector._diskErrorThresholdBytes,
+                diskWarningThresholdBytes: targetConnector._diskWarningThresholdBytes
+            }
+        });
+    }
+    /**
+     * Cleanup the migration if a migration fails or needs to be aborted.
+     * @param targetConnector The target connector to cleanup the migration with.
+     * @param options The options to control how the migration is cleaned up.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is cleaned up.
+     */
+    async cleanupMigration(targetConnector, options, loggingComponentType) {
+        await targetConnector?.teardown?.(loggingComponentType);
+    }
+    /**
+     * Read the store from file while holding the directory mutex.
+     * Use this for standalone reads (get, query, count, getPartitionContextIds) where
+     * no outer lock is held. Do not call from inside a withLock callback —
+     * use readStore instead to avoid a re-entrant deadlock.
+     * @returns The store.
+     * @internal
+     */
+    async readStoreWithLock() {
+        return this.withLock(async () => this.readStore());
+    }
+    /**
+     * Read the store from file without acquiring the directory mutex.
+     * Must only be called from inside a withLock callback, where the mutex
+     * is already held. Performing the read under the held lock prevents a race with
+     * the atomic rename(tmp→store.json) window on Windows that would otherwise
+     * return ENOENT and be misinterpreted as an empty store.
      * @returns The store.
      * @internal
      */
     async readStore() {
+        const filename = path.join(this._directory, "store.json");
+        let store;
+        try {
+            store = await readFile(filename, "utf8");
+        }
+        catch (err) {
+            if (ObjectHelper.propertyGet(err, "code") === "ENOENT") {
+                // The store has not been written yet, which is valid for a new store.
+                return [];
+            }
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "readStoreFailed", { directory: this._directory }, err);
+        }
         try {
-            const filename = path.join(this._directory, "store.json");
-            const store = await readFile(filename, "utf8");
             return JSON.parse(store);
         }
-        catch {
-            return [];
+        catch (err) {
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "readStoreCorrupt", { directory: this._directory }, err);
         }
     }
     /**
-     * Write the store to the file.
+     * Write the store to the file, atomically replacing the previous version so
+     * a reader can never observe a partially written store.
      * @param store The store to write.
      * @returns Nothing.
      * @internal
@@ -304,9 +634,34 @@ export class FileEntityStorageConnector {
     async writeStore(store) {
         try {
             const filename = path.join(this._directory, "store.json");
-            await writeFile(filename, JSON.stringify(store, undefined, "\t"), "utf8");
+            const tempFilename = `${filename}.tmp`;
+            await writeFile(tempFilename, JSON.stringify(store, undefined, "\t"), "utf8");
+            await rename(tempFilename, filename);
+        }
+        catch (err) {
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "writeStoreFailed", { directory: this._directory }, err);
+        }
+    }
+    /**
+     * Serialize an update so that concurrent modifications cannot interleave
+     * their read-modify-write cycles, which would lose updates or tear the
+     * store file. The mutex is keyed on the storage directory, so every
+     * connector or worker using the same store serializes with the others.
+     * @param update The update operation to perform.
+     * @returns The result of the update.
+     * @internal
+     */
+    async withLock(update) {
+        await Mutex.lock(this._directory, {
+            throwOnTimeout: true,
+            timeoutMs: this._mutexTimeoutMs
+        });
+        try {
+            return await update();
+        }
+        finally {
+            Mutex.unlock(this._directory);
         }
-        catch { }
     }
     /**
      * Check if the dir exists.