@twin.org/entity-storage-connector-file 0.0.3-next.3 → 0.0.3-next.30

package/README.md

@@ -1,6 +1,6 @@
-# TWIN Entity Storage Connector File
+# Entity Storage Connector File
 
-Entity Storage connector implementation using file storage.
+This package provides a file-backed backend for persisting entities on local or mounted disks in straightforward environments. It is designed to work with the wider storage ecosystem so applications can keep behaviour consistent across connectors and environments.
 
 ## Installation
 

package/dist/es/fileEntityStorageConnector.js

@@ -1,10 +1,11 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
-import { access, mkdir, readFile, writeFile } from "node:fs/promises";
+import { access, mkdir, readFile, rename, rm, statfs, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
-import { BaseError, Coerce, ComponentFactory, Guards, Is, ObjectHelper } from "@twin.org/core";
+import { BaseError, Coerce, ComponentFactory, GeneralError, Guards, HealthStatus, Is, Mutex, ObjectHelper, Validation } from "@twin.org/core";
 import { ComparisonOperator, EntityConditions, EntitySchemaFactory, EntitySchemaHelper, EntitySorter, LogicalOperator } from "@twin.org/entity";
+import { EntityStorageHelper } from "@twin.org/entity-storage-models";
 /**
  * Class for performing entity storage operations in file.
  */
@@ -23,6 +24,26 @@ export class FileEntityStorageConnector {
      * @internal
      */
     static _PARTITION_KEY = "partitionId";
+    /**
+     * Default disk space warning threshold: 500 MB.
+     * @internal
+     */
+    static _DEFAULT_DISK_WARNING_THRESHOLD_BYTES = 500 * 1024 * 1024;
+    /**
+     * Default disk space error threshold: 100 MB.
+     * @internal
+     */
+    static _DEFAULT_DISK_ERROR_THRESHOLD_BYTES = 100 * 1024 * 1024;
+    /**
+     * Maximum time to wait for the store update lock.
+     * @internal
+     */
+    static _UPDATE_LOCK_TIMEOUT_MS = 60000;
+    /**
+     * The name for the schema.
+     * @internal
+     */
+    _entitySchemaName;
     /**
      * The schema for the entity.
      * @internal
@@ -43,6 +64,21 @@ export class FileEntityStorageConnector {
      * @internal
      */
     _directory;
+    /**
+     * Free bytes below which health reports an error.
+     * @internal
+     */
+    _diskErrorThresholdBytes;
+    /**
+     * Free bytes below which health reports a warning.
+     * @internal
+     */
+    _diskWarningThresholdBytes;
+    /**
+     * Milliseconds to wait for the directory lock before throwing.
+     * @internal
+     */
+    _lockTimeoutMs;
     /**
      * Create a new instance of FileEntityStorageConnector.
      * @param options The options for the connector.
@@ -52,10 +88,19 @@ export class FileEntityStorageConnector {
         Guards.stringValue(FileEntityStorageConnector.CLASS_NAME, "options.entitySchema", options.entitySchema);
         Guards.object(FileEntityStorageConnector.CLASS_NAME, "options.config", options.config);
         Guards.stringValue(FileEntityStorageConnector.CLASS_NAME, "options.config.directory", options.config.directory);
+        this._entitySchemaName = options.entitySchema;
         this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
         this._partitionContextIds = options.partitionContextIds;
         this._primaryKey = EntitySchemaHelper.getPrimaryKey(this._entitySchema);
         this._directory = path.resolve(options.config.directory);
+        this._diskErrorThresholdBytes =
+            options.config.diskErrorThresholdBytes ??
+                FileEntityStorageConnector._DEFAULT_DISK_ERROR_THRESHOLD_BYTES;
+        this._diskWarningThresholdBytes =
+            options.config.diskWarningThresholdBytes ??
+                FileEntityStorageConnector._DEFAULT_DISK_WARNING_THRESHOLD_BYTES;
+        this._lockTimeoutMs =
+            options.config.lockTimeoutMs ?? FileEntityStorageConnector._UPDATE_LOCK_TIMEOUT_MS;
     }
     /**
      * Bootstrap the connector by creating and initializing any resources it needs.
@@ -116,6 +161,65 @@ export class FileEntityStorageConnector {
     className() {
         return FileEntityStorageConnector.CLASS_NAME;
     }
+    /**
+     * Returns the health status of the component.
+     * @returns The health status of the component, can return multiple entries for elements within the component.
+     */
+    async health() {
+        try {
+            const stats = await statfs(this._directory);
+            const freeBytes = stats.bavail * stats.bsize;
+            if (freeBytes < this._diskErrorThresholdBytes) {
+                return [
+                    {
+                        source: FileEntityStorageConnector.CLASS_NAME,
+                        status: HealthStatus.Error,
+                        description: "healthDescription",
+                        message: "diskSpaceError",
+                        data: {
+                            directory: this._directory,
+                            freeBytes,
+                            thresholdBytes: this._diskErrorThresholdBytes
+                        }
+                    }
+                ];
+            }
+            else if (freeBytes < this._diskWarningThresholdBytes) {
+                return [
+                    {
+                        source: FileEntityStorageConnector.CLASS_NAME,
+                        status: HealthStatus.Warning,
+                        description: "healthDescription",
+                        message: "diskSpaceWarning",
+                        data: {
+                            directory: this._directory,
+                            freeBytes,
+                            thresholdBytes: this._diskWarningThresholdBytes
+                        }
+                    }
+                ];
+            }
+            return [
+                {
+                    source: FileEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Ok,
+                    description: "healthDescription",
+                    data: { directory: this._directory, freeBytes }
+                }
+            ];
+        }
+        catch {
+            return [
+                {
+                    source: FileEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Error,
+                    description: "healthDescription",
+                    message: "diskSpaceCheckFailed",
+                    data: { directory: this._directory }
+                }
+            ];
+        }
+    }
     /**
      * Get the schema for the entities.
      * @returns The schema for the entities.
@@ -134,7 +238,7 @@ export class FileEntityStorageConnector {
         Guards.stringValue(FileEntityStorageConnector.CLASS_NAME, "id", id);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        const store = await this.readStore();
+        const store = await this.readStoreWithLock();
         const finalConditions = conditions ?? [];
         if (Is.stringValue(partitionKey)) {
             finalConditions.push({
@@ -145,9 +249,11 @@ export class FileEntityStorageConnector {
         const index = this.findItem(store, id, secondaryIndex, finalConditions);
         const item = index >= 0 ? store[index] : undefined;
         if (Is.objectValue(item)) {
-            ObjectHelper.propertyDelete(item, FileEntityStorageConnector._PARTITION_KEY);
+            return EntityStorageHelper.unPrepareEntity(item, [
+                FileEntityStorageConnector._PARTITION_KEY
+            ]);
         }
-        return item;
+        return undefined;
     }
     /**
      * Set an entity.
@@ -159,25 +265,143 @@ export class FileEntityStorageConnector {
         Guards.object(FileEntityStorageConnector.CLASS_NAME, "entity", entity);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        EntitySchemaHelper.validateEntity(entity, this.getSchema());
-        const store = await this.readStore();
-        const finalEntity = ObjectHelper.clone(entity);
-        const finalConditions = conditions ?? [];
-        if (Is.stringValue(partitionKey)) {
-            finalConditions.push({
-                property: FileEntityStorageConnector._PARTITION_KEY,
-                value: partitionKey
+        const prepared = EntityStorageHelper.prepareEntity(entity, this._entitySchema, Is.stringValue(partitionKey)
+            ? [{ property: FileEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : undefined, { nullBehavior: "omit" });
+        await this.withLock(async () => {
+            const store = await this.readStore();
+            const finalConditions = conditions ?? [];
+            if (Is.stringValue(partitionKey)) {
+                finalConditions.push({
+                    property: FileEntityStorageConnector._PARTITION_KEY,
+                    value: partitionKey
+                });
+            }
+            const existingIndex = this.findItem(store, prepared[this._primaryKey.property], undefined, finalConditions);
+            if (existingIndex >= 0) {
+                store[existingIndex] = prepared;
+            }
+            else {
+                store.push(prepared);
+            }
+            await this.writeStore(store);
+        });
+    }
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    async setBatch(entities) {
+        Guards.arrayValue(FileEntityStorageConnector.CLASS_NAME, "entities", entities);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        await this.withLock(async () => {
+            const store = await this.readStore();
+            for (const entity of entities) {
+                Guards.object(FileEntityStorageConnector.CLASS_NAME, "entity", entity);
+                const prepared = EntityStorageHelper.prepareEntity(entity, this._entitySchema, Is.stringValue(partitionKey)
+                    ? [{ property: FileEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+                    : undefined, { nullBehavior: "omit" });
+                const existingIndex = this.findItem(store, prepared[this._primaryKey.property], undefined, Is.stringValue(partitionKey)
+                    ? [
+                        {
+                            property: FileEntityStorageConnector._PARTITION_KEY,
+                            value: partitionKey
+                        }
+                    ]
+                    : []);
+                if (existingIndex >= 0) {
+                    store[existingIndex] = prepared;
+                }
+                else {
+                    store.push(prepared);
+                }
+            }
+            await this.writeStore(store);
+        });
+    }
+    /**
+     * Remove all entities from the storage.
+     * @returns Nothing.
+     */
+    async empty() {
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        try {
+            await this.withLock(async () => {
+                const store = await this.readStore();
+                const remaining = Is.stringValue(partitionKey)
+                    ? store.filter(item => ObjectHelper.propertyGet(item, FileEntityStorageConnector._PARTITION_KEY) !== partitionKey)
+                    : [];
+                await this.writeStore(remaining);
             });
-            ObjectHelper.propertySet(finalEntity, FileEntityStorageConnector._PARTITION_KEY, partitionKey);
         }
-        const existingIndex = this.findItem(store, finalEntity[this._primaryKey.property], undefined, finalConditions);
-        if (existingIndex >= 0) {
-            store[existingIndex] = finalEntity;
+        catch (err) {
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "emptyFailed", undefined, err);
         }
-        else {
-            store.push(finalEntity);
+    }
+    /**
+     * Remove multiple entities by id.
+     * @param ids The ids of the entities to remove.
+     * @returns Nothing.
+     */
+    async removeBatch(ids) {
+        Guards.arrayValue(FileEntityStorageConnector.CLASS_NAME, "ids", ids);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        try {
+            await this.withLock(async () => {
+                const store = await this.readStore();
+                const idSet = new Set(ids);
+                const remaining = store.filter(item => {
+                    if (Is.stringValue(partitionKey) &&
+                        ObjectHelper.propertyGet(item, FileEntityStorageConnector._PARTITION_KEY) !==
+                            partitionKey) {
+                        return true;
+                    }
+                    return !idSet.has(item[this._primaryKey.property]);
+                });
+                await this.writeStore(remaining);
+            });
+        }
+        catch (err) {
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "removeBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Teardown the storage by deleting the underlying store file.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns True if the teardown process was successful.
+     */
+    async teardown(nodeLoggingComponentType) {
+        const nodeLogging = ComponentFactory.getIfExists(nodeLoggingComponentType);
+        await nodeLogging?.log({
+            level: "info",
+            source: FileEntityStorageConnector.CLASS_NAME,
+            ts: Date.now(),
+            message: "storeTearingDown"
+        });
+        try {
+            await rm(this._directory, { recursive: true, force: true });
+            await nodeLogging?.log({
+                level: "info",
+                source: FileEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "storeTornDown"
+            });
+            return true;
+        }
+        catch (err) {
+            await nodeLogging?.log({
+                level: "error",
+                source: FileEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "teardownFailed",
+                error: BaseError.fromError(err)
+            });
+            return false;
         }
-        await this.writeStore(store);
     }
     /**
      * Remove the entity.
@@ -189,19 +413,21 @@ export class FileEntityStorageConnector {
         Guards.stringValue(FileEntityStorageConnector.CLASS_NAME, "id", id);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        const store = await this.readStore();
-        const finalConditions = conditions ?? [];
-        if (Is.stringValue(partitionKey)) {
-            finalConditions.push({
-                property: FileEntityStorageConnector._PARTITION_KEY,
-                value: partitionKey
-            });
-        }
-        const index = this.findItem(store, id, undefined, finalConditions);
-        if (index >= 0) {
-            store.splice(index, 1);
-            await this.writeStore(store);
-        }
+        await this.withLock(async () => {
+            const store = await this.readStore();
+            const finalConditions = conditions ?? [];
+            if (Is.stringValue(partitionKey)) {
+                finalConditions.push({
+                    property: FileEntityStorageConnector._PARTITION_KEY,
+                    value: partitionKey
+                });
+            }
+            const index = this.findItem(store, id, undefined, finalConditions);
+            if (index >= 0) {
+                store.splice(index, 1);
+                await this.writeStore(store);
+            }
+        });
     }
     /**
      * Find all the entities which match the conditions.
@@ -216,7 +442,14 @@ export class FileEntityStorageConnector {
     async query(conditions, sortProperties, properties, cursor, limit) {
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        let allEntities = await this.readStore();
+        EntityStorageHelper.validateSortProperties(this._entitySchema, sortProperties);
+        EntityStorageHelper.validateProperties(this._entitySchema, properties);
+        if (!Is.empty(limit)) {
+            const validationFailures = [];
+            Validation.integer("limit", limit, validationFailures, undefined, { minValue: 1 });
+            Validation.asValidationError(FileEntityStorageConnector.CLASS_NAME, "query", validationFailures);
+        }
+        let allEntities = await this.readStoreWithLock();
         const finalConditions = {
             conditions: [],
             logicalOperator: LogicalOperator.And
@@ -229,7 +462,7 @@ export class FileEntityStorageConnector {
             });
         }
         if (!Is.empty(conditions)) {
-            finalConditions.conditions.push(conditions);
+            finalConditions.conditions.push(EntityStorageHelper.normalizeConditionValues(conditions));
         }
         const entities = [];
         const finalLimit = limit ?? FileEntityStorageConnector._DEFAULT_LIMIT;
@@ -241,9 +474,12 @@ export class FileEntityStorageConnector {
             for (let i = startIndex; i < allEntities.length; i++) {
                 if (EntityConditions.check(allEntities[i], finalConditions) &&
                     entities.length < finalLimit) {
-                    const entity = ObjectHelper.pick(allEntities[i], properties);
-                    ObjectHelper.propertyDelete(entity, FileEntityStorageConnector._PARTITION_KEY);
-                    entities.push(entity);
+                    const entity = Is.arrayValue(properties)
+                        ? ObjectHelper.pick(allEntities[i], properties)
+                        : allEntities[i];
+                    entities.push(EntityStorageHelper.unPrepareEntity(entity, [
+                        FileEntityStorageConnector._PARTITION_KEY
+                    ]));
                     if (entities.length >= finalLimit) {
                         if (i < allEntities.length - 1) {
                             nextCursor = (i + 1).toString();
@@ -259,22 +495,144 @@ export class FileEntityStorageConnector {
         };
     }
     /**
-     * Read the store from file.
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
+     */
+    async count(conditions) {
+        const store = await this.readStoreWithLock();
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        const finalConditions = {
+            conditions: [],
+            logicalOperator: LogicalOperator.And
+        };
+        if (Is.stringValue(partitionKey)) {
+            finalConditions.conditions.push({
+                property: FileEntityStorageConnector._PARTITION_KEY,
+                comparison: ComparisonOperator.Equals,
+                value: partitionKey
+            });
+        }
+        if (!Is.empty(conditions)) {
+            finalConditions.conditions.push(EntityStorageHelper.normalizeConditionValues(conditions));
+        }
+        if (finalConditions.conditions.length === 0) {
+            return store.length;
+        }
+        return store.filter(item => EntityConditions.check(item, finalConditions)).length;
+    }
+    /**
+     * Get a unique list of all the context ids from the storage.
+     * @returns The list of unique context ids.
+     */
+    async getPartitionContextIds() {
+        const contextIds = {};
+        const store = await this.readStoreWithLock();
+        for (const entity of store) {
+            const partitionId = ObjectHelper.propertyGet(entity, FileEntityStorageConnector._PARTITION_KEY);
+            if (Is.stringValue(partitionId)) {
+                contextIds[partitionId] = ContextIdHelper.shortSplit(this._partitionContextIds ?? [], partitionId);
+            }
+        }
+        return Object.values(contextIds);
+    }
+    /**
+     * Create the target connector for performing the migration it will use a temporary storage location.
+     * @param newEntitySchema The name of the new entity schema to create the connector for.
+     * @returns Connector for performing the migration.
+     */
+    async createTargetConnector(newEntitySchema) {
+        const baseName = path.basename(this._directory);
+        const parentDir = path.resolve(this._directory, "..");
+        const migrationDir = path.join(parentDir, `${baseName}_migration_${Date.now()}`);
+        return new FileEntityStorageConnector({
+            entitySchema: newEntitySchema,
+            partitionContextIds: this._partitionContextIds,
+            config: {
+                directory: migrationDir,
+                diskErrorThresholdBytes: this._diskErrorThresholdBytes,
+                diskWarningThresholdBytes: this._diskWarningThresholdBytes
+            }
+        });
+    }
+    /**
+     * Finalize the migration by tearing down the old connector and replacing it with the target connector.
+     * @param targetConnector The target connector to finalize the migration with.
+     * @param options The options to control how the migration is finalized.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is finalized.
+     */
+    async finalizeMigration(targetConnector, options, loggingComponentType) {
+        const originalDir = this._directory;
+        const migrationDir = targetConnector._directory;
+        // Teardown the original connector, removing the entire source directory.
+        await this.teardown(loggingComponentType);
+        // Rename the migration directory into the original location.
+        await rename(migrationDir, originalDir);
+        return new FileEntityStorageConnector({
+            entitySchema: targetConnector._entitySchemaName,
+            partitionContextIds: targetConnector._partitionContextIds,
+            config: {
+                directory: this._directory,
+                diskErrorThresholdBytes: targetConnector._diskErrorThresholdBytes,
+                diskWarningThresholdBytes: targetConnector._diskWarningThresholdBytes
+            }
+        });
+    }
+    /**
+     * Cleanup the migration if a migration fails or needs to be aborted.
+     * @param targetConnector The target connector to cleanup the migration with.
+     * @param options The options to control how the migration is cleaned up.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is cleaned up.
+     */
+    async cleanupMigration(targetConnector, options, loggingComponentType) {
+        await targetConnector?.teardown?.(loggingComponentType);
+    }
+    /**
+     * Read the store from file while holding the directory mutex.
+     * Use this for standalone reads (get, query, count, getPartitionContextIds) where
+     * no outer lock is held. Do not call from inside a withLock callback —
+     * use readStore instead to avoid a re-entrant deadlock.
+     * @returns The store.
+     * @internal
+     */
+    async readStoreWithLock() {
+        return this.withLock(async () => this.readStore());
+    }
+    /**
+     * Read the store from file without acquiring the directory mutex.
+     * Must only be called from inside a withLock callback, where the mutex
+     * is already held. Performing the read under the held lock prevents a race with
+     * the atomic rename(tmp→store.json) window on Windows that would otherwise
+     * return ENOENT and be misinterpreted as an empty store.
      * @returns The store.
      * @internal
      */
     async readStore() {
+        const filename = path.join(this._directory, "store.json");
+        let store;
+        try {
+            store = await readFile(filename, "utf8");
+        }
+        catch (err) {
+            if (ObjectHelper.propertyGet(err, "code") === "ENOENT") {
+                // The store has not been written yet, which is valid for a new store.
+                return [];
+            }
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "readStoreFailed", { directory: this._directory }, err);
+        }
         try {
-            const filename = path.join(this._directory, "store.json");
-            const store = await readFile(filename, "utf8");
             return JSON.parse(store);
         }
-        catch {
-            return [];
+        catch (err) {
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "readStoreCorrupt", { directory: this._directory }, err);
         }
     }
     /**
-     * Write the store to the file.
+     * Write the store to the file, atomically replacing the previous version so
+     * a reader can never observe a partially written store.
      * @param store The store to write.
      * @returns Nothing.
      * @internal
@@ -282,9 +640,34 @@ export class FileEntityStorageConnector {
     async writeStore(store) {
         try {
             const filename = path.join(this._directory, "store.json");
-            await writeFile(filename, JSON.stringify(store, undefined, "\t"), "utf8");
+            const tempFilename = `${filename}.tmp`;
+            await writeFile(tempFilename, JSON.stringify(store, undefined, "\t"), "utf8");
+            await rename(tempFilename, filename);
+        }
+        catch (err) {
+            throw new GeneralError(FileEntityStorageConnector.CLASS_NAME, "writeStoreFailed", { directory: this._directory }, err);
+        }
+    }
+    /**
+     * Serialize an update so that concurrent modifications cannot interleave
+     * their read-modify-write cycles, which would lose updates or tear the
+     * store file. The mutex is keyed on the storage directory, so every
+     * connector or worker using the same store serializes with the others.
+     * @param update The update operation to perform.
+     * @returns The result of the update.
+     * @internal
+     */
+    async withLock(update) {
+        await Mutex.lock(this._directory, {
+            timeoutMs: this._lockTimeoutMs,
+            throwOnTimeout: true
+        });
+        try {
+            return await update();
+        }
+        finally {
+            Mutex.unlock(this._directory);
         }
-        catch { }
     }
     /**
      * Check if the dir exists.