@twin.org/entity-storage-connector-memory 0.0.3-next.3 → 0.0.3-next.30

package/README.md

@@ -1,6 +1,6 @@
-# TWIN Entity Storage Connector Memory
+# Entity Storage Connector Memory
 
-Entity Storage connector implementation using in-memory storage.
+This package provides an in-memory backend suited to local development, automated testing and short-lived workloads. It is designed to work with the wider storage ecosystem so applications can keep behaviour consistent across connectors and environments.
 
 ## Installation
 

package/dist/es/index.js

@@ -1,4 +1,4 @@
-// Copyright 2024 IOTA Stiftung.
+// Copyright 2026 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 export * from "./memoryEntityStorageConnector.js";
 export * from "./models/IMemoryEntityStorageConnectorConstructorOptions.js";

package/dist/es/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,cAAc,mCAAmC,CAAC;AAClD,cAAc,6DAA6D,CAAC","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nexport * from \"./memoryEntityStorageConnector.js\";\nexport * from \"./models/IMemoryEntityStorageConnectorConstructorOptions.js\";\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,cAAc,mCAAmC,CAAC;AAClD,cAAc,6DAA6D,CAAC","sourcesContent":["// Copyright 2026 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nexport * from \"./memoryEntityStorageConnector.js\";\nexport * from \"./models/IMemoryEntityStorageConnectorConstructorOptions.js\";\n"]}

package/dist/es/memoryEntityStorageConnector.js

@@ -1,10 +1,19 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
-import { Coerce, Guards, Is, ObjectHelper } from "@twin.org/core";
+import { Coerce, ComponentFactory, Guards, HealthStatus, Is, Mutex, ObjectHelper, SharedObjectBuffer, Validation } from "@twin.org/core";
 import { ComparisonOperator, EntityConditions, EntitySchemaFactory, EntitySchemaHelper, EntitySorter, LogicalOperator } from "@twin.org/entity";
+import { EntityStorageHelper } from "@twin.org/entity-storage-models";
 /**
- * Class for performing entity storage operations in-memory.
+ * Class for performing entity storage operations in-memory backed by a shared object buffer.
+ *
+ * All reads and writes are serialised with a per-schema lock so that concurrent async
+ * access, including across worker threads, never produces torn or lost updates.
+ *
+ * All connector instances that share the same entity schema name share the same underlying
+ * buffer, making data written in one instance immediately visible in another, including
+ * across worker threads when the main thread forwards worker messages to the lock and
+ * buffer handlers.
  */
 export class MemoryEntityStorageConnector {
     /**
@@ -37,10 +46,21 @@ export class MemoryEntityStorageConnector {
      */
     _primaryKey;
     /**
-     * The storage for the in-memory items.
+     * The resolved schema name used as the shared buffer and lock key.
+     * Stored separately so it is always a plain string rather than string | undefined.
      * @internal
      */
-    _store;
+    _schemaName;
+    /**
+     * Initial capacity hint in bytes for the shared entity buffer.
+     * @internal
+     */
+    _initialCapacityBytes;
+    /**
+     * Maximum capacity in bytes for the shared entity buffer.
+     * @internal
+     */
+    _maxCapacityBytes;
     /**
      * Create a new instance of MemoryEntityStorageConnector.
      * @param options The options for the connector.
@@ -49,9 +69,11 @@ export class MemoryEntityStorageConnector {
         Guards.object(MemoryEntityStorageConnector.CLASS_NAME, "options", options);
         Guards.stringValue(MemoryEntityStorageConnector.CLASS_NAME, "options.entitySchema", options.entitySchema);
         this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
+        this._schemaName = this._entitySchema.type ?? options.entitySchema;
         this._partitionContextIds = options.partitionContextIds;
         this._primaryKey = EntitySchemaHelper.getPrimaryKey(this._entitySchema);
-        this._store = [];
+        this._initialCapacityBytes = options.initialCapacityBytes;
+        this._maxCapacityBytes = options.maxCapacityBytes;
     }
     /**
      * Returns the class name of the component.
@@ -60,6 +82,20 @@ export class MemoryEntityStorageConnector {
     className() {
         return MemoryEntityStorageConnector.CLASS_NAME;
     }
+    /**
+     * Returns the health status of the component.
+     * @returns The health status of the component.
+     */
+    async health() {
+        return [
+            {
+                source: MemoryEntityStorageConnector.CLASS_NAME,
+                status: HealthStatus.Ok,
+                description: "healthDescription",
+                data: { entityType: this._schemaName }
+            }
+        ];
+    }
     /**
      * Get the schema for the entities.
      * @returns The schema for the entities.
@@ -67,6 +103,18 @@ export class MemoryEntityStorageConnector {
     getSchema() {
         return this._entitySchema;
     }
+    /**
+     * Bootstrap the component by creating and initializing any resources it needs.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns True if the bootstrapping process was successful.
+     */
+    async bootstrap(nodeLoggingComponentType) {
+        await SharedObjectBuffer.create(this._schemaName, {
+            initialCapacityBytes: this._initialCapacityBytes,
+            maxCapacityBytes: this._maxCapacityBytes
+        });
+        return true;
+    }
     /**
      * Get an entity.
      * @param id The id of the entity to get, or the index value if secondaryIndex is set.
@@ -85,40 +133,84 @@ export class MemoryEntityStorageConnector {
                 value: partitionKey
             });
         }
-        const index = this.findItem(id, secondaryIndex, finalConditions);
-        const item = index >= 0 ? ObjectHelper.clone(this._store[index]) : undefined;
-        if (Is.objectValue(item)) {
-            ObjectHelper.propertyDelete(item, MemoryEntityStorageConnector._PARTITION_KEY);
-        }
-        return item;
+        return this.withLock(entities => {
+            const index = this.findItem(entities, id, secondaryIndex, finalConditions);
+            const item = index >= 0 ? entities[index] : undefined;
+            if (Is.objectValue(item)) {
+                return {
+                    result: EntityStorageHelper.unPrepareEntity(item, [
+                        MemoryEntityStorageConnector._PARTITION_KEY
+                    ])
+                };
+            }
+            return { result: undefined };
+        });
     }
     /**
      * Set an entity.
      * @param entity The entity to set.
      * @param conditions The optional conditions to match for the entities.
-     * @returns The id of the entity.
+     * @returns Resolves when the entity has been stored.
      */
     async set(entity, conditions) {
         Guards.object(MemoryEntityStorageConnector.CLASS_NAME, "entity", entity);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        EntitySchemaHelper.validateEntity(entity, this.getSchema());
         const finalConditions = conditions ?? [];
-        const finalEntity = ObjectHelper.clone(entity);
+        const prepared = EntityStorageHelper.prepareEntity(entity, this._entitySchema, Is.stringValue(partitionKey)
+            ? [{ property: MemoryEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : undefined, { nullBehavior: "omit" });
         if (Is.stringValue(partitionKey)) {
             finalConditions.push({
                 property: MemoryEntityStorageConnector._PARTITION_KEY,
                 value: partitionKey
             });
-            ObjectHelper.propertySet(finalEntity, MemoryEntityStorageConnector._PARTITION_KEY, partitionKey);
-        }
-        const existingIndex = this.findItem(finalEntity[this._primaryKey.property], undefined, finalConditions);
-        if (existingIndex >= 0) {
-            this._store[existingIndex] = finalEntity;
-        }
-        else {
-            this._store.push(finalEntity);
         }
+        return this.withLock(entities => {
+            const existingIndex = this.findItem(entities, prepared[this._primaryKey.property], undefined, finalConditions);
+            if (existingIndex >= 0) {
+                entities[existingIndex] = prepared;
+            }
+            else {
+                entities.push(prepared);
+            }
+            return { updated: entities, result: undefined };
+        });
+    }
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    async setBatch(entities) {
+        Guards.arrayValue(MemoryEntityStorageConnector.CLASS_NAME, "entities", entities);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        const preparedItems = entities.map(entity => EntityStorageHelper.prepareEntity(entity, this._entitySchema, Is.stringValue(partitionKey)
+            ? [{ property: MemoryEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : undefined, { nullBehavior: "omit" }));
+        return this.withLock(store => {
+            const indexMap = new Map();
+            for (let i = 0; i < store.length; i++) {
+                const stored = store[i];
+                const storedPartition = ObjectHelper.propertyGet(stored, MemoryEntityStorageConnector._PARTITION_KEY);
+                if (!Is.stringValue(partitionKey) || storedPartition === partitionKey) {
+                    indexMap.set(stored[this._primaryKey.property], i);
+                }
+            }
+            for (const prepared of preparedItems) {
+                const id = prepared[this._primaryKey.property];
+                const existingIndex = indexMap.get(id);
+                if (existingIndex !== undefined) {
+                    store[existingIndex] = prepared;
+                }
+                else {
+                    const newIndex = store.push(prepared) - 1;
+                    indexMap.set(id, newIndex);
+                }
+            }
+            return { updated: store, result: undefined };
+        });
     }
     /**
      * Remove the entity.
@@ -137,10 +229,13 @@ export class MemoryEntityStorageConnector {
                 value: partitionKey
             });
         }
-        const index = this.findItem(id, undefined, finalConditions);
-        if (index >= 0) {
-            this._store.splice(index, 1);
-        }
+        return this.withLock(entities => {
+            const index = this.findItem(entities, id, undefined, finalConditions);
+            if (index >= 0) {
+                entities.splice(index, 1);
+            }
+            return { updated: entities, result: undefined };
+        });
     }
     /**
      * Find all the entities which match the conditions.
@@ -155,7 +250,135 @@ export class MemoryEntityStorageConnector {
     async query(conditions, sortProperties, properties, cursor, limit) {
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        let allEntities = this._store.slice();
+        EntityStorageHelper.validateSortProperties(this._entitySchema, sortProperties);
+        EntityStorageHelper.validateProperties(this._entitySchema, properties);
+        if (!Is.empty(limit)) {
+            const validationFailures = [];
+            Validation.integer("limit", limit, validationFailures, undefined, { minValue: 1 });
+            Validation.asValidationError(MemoryEntityStorageConnector.CLASS_NAME, "query", validationFailures);
+        }
+        return this.withLock(store => {
+            let allEntities = store.slice();
+            const finalConditions = {
+                conditions: [],
+                logicalOperator: LogicalOperator.And
+            };
+            if (Is.stringValue(partitionKey)) {
+                finalConditions.conditions.push({
+                    property: MemoryEntityStorageConnector._PARTITION_KEY,
+                    comparison: ComparisonOperator.Equals,
+                    value: partitionKey
+                });
+            }
+            if (!Is.empty(conditions)) {
+                finalConditions.conditions.push(EntityStorageHelper.normalizeConditionValues(conditions));
+            }
+            const resultEntities = [];
+            const finalLimit = limit ?? MemoryEntityStorageConnector._DEFAULT_LIMIT;
+            let nextCursor;
+            if (allEntities.length > 0) {
+                const finalSortKeys = EntitySchemaHelper.buildSortProperties(this._entitySchema, sortProperties);
+                allEntities = EntitySorter.sort(allEntities, finalSortKeys);
+                const startIndex = Coerce.number(cursor) ?? 0;
+                for (let i = startIndex; i < allEntities.length; i++) {
+                    if (EntityConditions.check(allEntities[i], finalConditions) &&
+                        resultEntities.length < finalLimit) {
+                        const entity = Is.arrayValue(properties)
+                            ? ObjectHelper.pick(allEntities[i], properties)
+                            : allEntities[i];
+                        resultEntities.push(EntityStorageHelper.unPrepareEntity(entity, [
+                            MemoryEntityStorageConnector._PARTITION_KEY
+                        ]));
+                        if (resultEntities.length >= finalLimit) {
+                            if (i < allEntities.length - 1) {
+                                nextCursor = (i + 1).toString();
+                            }
+                            break;
+                        }
+                    }
+                }
+            }
+            return { result: { entities: resultEntities, cursor: nextCursor } };
+        });
+    }
+    /**
+     * Remove all entities from the storage.
+     * @returns Nothing.
+     */
+    async empty() {
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        return this.withLock(entities => {
+            if (Is.stringValue(partitionKey)) {
+                const filtered = entities.filter(item => ObjectHelper.propertyGet(item, MemoryEntityStorageConnector._PARTITION_KEY) !==
+                    partitionKey);
+                return { updated: filtered, result: undefined };
+            }
+            return { updated: [], result: undefined };
+        });
+    }
+    /**
+     * Remove multiple entities by id.
+     * @param ids The ids of the entities to remove.
+     * @returns Nothing.
+     */
+    async removeBatch(ids) {
+        Guards.arrayValue(MemoryEntityStorageConnector.CLASS_NAME, "ids", ids);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        const finalConditions = [];
+        if (Is.stringValue(partitionKey)) {
+            finalConditions.push({
+                property: MemoryEntityStorageConnector._PARTITION_KEY,
+                value: partitionKey
+            });
+        }
+        return this.withLock(entities => {
+            for (const id of ids) {
+                const index = this.findItem(entities, id, undefined, finalConditions);
+                if (index >= 0) {
+                    entities.splice(index, 1);
+                }
+            }
+            return { updated: entities, result: undefined };
+        });
+    }
+    /**
+     * Teardown the storage by clearing the underlying shared buffer for this schema.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns True if the teardown process was successful.
+     */
+    async teardown(nodeLoggingComponentType) {
+        const nodeLogging = ComponentFactory.getIfExists(nodeLoggingComponentType);
+        await nodeLogging?.log({
+            level: "info",
+            source: MemoryEntityStorageConnector.CLASS_NAME,
+            ts: Date.now(),
+            message: "storeTearingDown"
+        });
+        await Mutex.lock(this._schemaName, { throwOnTimeout: true });
+        try {
+            SharedObjectBuffer.remove(this._schemaName);
+        }
+        finally {
+            Mutex.unlock(this._schemaName);
+        }
+        await nodeLogging?.log({
+            level: "info",
+            source: MemoryEntityStorageConnector.CLASS_NAME,
+            ts: Date.now(),
+            message: "storeTornDown"
+        });
+        return true;
+    }
+    /**
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
+     */
+    async count(conditions) {
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
         const finalConditions = {
             conditions: [],
             logicalOperator: LogicalOperator.And
@@ -168,51 +391,124 @@ export class MemoryEntityStorageConnector {
             });
         }
         if (!Is.empty(conditions)) {
-            finalConditions.conditions.push(conditions);
-        }
-        const entities = [];
-        const finalLimit = limit ?? MemoryEntityStorageConnector._DEFAULT_LIMIT;
-        let nextCursor;
-        if (allEntities.length > 0) {
-            const finalSortKeys = EntitySchemaHelper.buildSortProperties(this._entitySchema, sortProperties);
-            allEntities = EntitySorter.sort(allEntities, finalSortKeys);
-            const startIndex = Coerce.number(cursor) ?? 0;
-            for (let i = startIndex; i < allEntities.length; i++) {
-                if (EntityConditions.check(allEntities[i], finalConditions) &&
-                    entities.length < finalLimit) {
-                    const entity = ObjectHelper.clone(ObjectHelper.pick(allEntities[i], properties));
-                    ObjectHelper.propertyDelete(entity, MemoryEntityStorageConnector._PARTITION_KEY);
-                    entities.push(entity);
-                    if (entities.length >= finalLimit) {
-                        if (i < allEntities.length - 1) {
-                            nextCursor = (i + 1).toString();
-                        }
-                        break;
-                    }
+            finalConditions.conditions.push(EntityStorageHelper.normalizeConditionValues(conditions));
+        }
+        return this.withLock(entities => {
+            if (finalConditions.conditions.length === 0) {
+                return { result: entities.length };
+            }
+            return {
+                result: entities.filter(item => EntityConditions.check(item, finalConditions)).length
+            };
+        });
+    }
+    /**
+     * Get all entities in the memory store.
+     * @returns All stored entities with partition keys removed.
+     */
+    async getStore() {
+        return this.withLock(entities => ({
+            result: entities.map(item => EntityStorageHelper.unPrepareEntity(item, [MemoryEntityStorageConnector._PARTITION_KEY]))
+        }));
+    }
+    /**
+     * Get a unique list of all the context ids from the storage.
+     * @returns The list of unique context ids.
+     */
+    async getPartitionContextIds() {
+        return this.withLock(entities => {
+            const contextIds = {};
+            for (const entity of entities) {
+                const partitionId = ObjectHelper.propertyGet(entity, MemoryEntityStorageConnector._PARTITION_KEY);
+                if (Is.stringValue(partitionId)) {
+                    contextIds[partitionId] = ContextIdHelper.shortSplit(this._partitionContextIds ?? [], partitionId);
                 }
             }
+            return { result: Object.values(contextIds) };
+        });
+    }
+    /**
+     * Create the target connector for performing the migration it will use a temporary storage location.
+     * @param newEntitySchema The name of the new entity schema to create the connector for.
+     * @returns Connector for performing the migration.
+     */
+    async createTargetConnector(newEntitySchema) {
+        // Resolve the target schema name the same way _schemaName is resolved in the constructor.
+        const targetSchemaEntry = EntitySchemaFactory.get(newEntitySchema);
+        const targetSchemaName = targetSchemaEntry.type ?? newEntitySchema;
+        // When migrating to a different schema, wipe the target buffer so that every
+        // migration starts from an empty store regardless of any previous connector
+        // instances that shared the same schema name.
+        if (targetSchemaName !== this._schemaName) {
+            await Mutex.lock(targetSchemaName, { throwOnTimeout: true });
+            try {
+                SharedObjectBuffer.remove(targetSchemaName);
+            }
+            finally {
+                Mutex.unlock(targetSchemaName);
+            }
         }
-        return {
-            entities,
-            cursor: nextCursor
-        };
+        return new MemoryEntityStorageConnector({
+            entitySchema: newEntitySchema,
+            partitionContextIds: this._partitionContextIds,
+            initialCapacityBytes: this._initialCapacityBytes,
+            maxCapacityBytes: this._maxCapacityBytes
+        });
+    }
+    /**
+     * Finalize the migration by tearing down the old connector and replacing it with the target connector.
+     * @param targetConnector The target connector to finalize the migration with.
+     * @param options The options to control how the migration is finalized.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is finalized.
+     */
+    async finalizeMigration(targetConnector, options, loggingComponentType) {
+        return targetConnector;
     }
     /**
-     * Get the memory store.
-     * @returns The store.
+     * Cleanup the migration if a migration fails or needs to be aborted.
+     * @param targetConnector The target connector to cleanup the migration with.
+     * @param options The options to control how the migration is cleaned up.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is cleaned up.
+     */
+    async cleanupMigration(targetConnector, options, loggingComponentType) { }
+    /**
+     * Acquires the schema-keyed lock, runs fn with the current entity array, optionally
+     * writes back a modified array, then releases the lock.
+     * @param fn The synchronous function to run while the lock is held.
+     * @returns The result produced by fn.
+     * @internal
      */
-    getStore() {
-        return this._store;
+    async withLock(fn) {
+        const key = this._schemaName;
+        await Mutex.lock(key, { throwOnTimeout: true });
+        try {
+            await SharedObjectBuffer.create(key, {
+                initialCapacityBytes: this._initialCapacityBytes,
+                maxCapacityBytes: this._maxCapacityBytes
+            });
+            const entities = (await SharedObjectBuffer.read(key)) ?? [];
+            const outcome = fn(entities);
+            if (outcome.updated !== undefined) {
+                await SharedObjectBuffer.write(key, outcome.updated);
+            }
+            return outcome.result;
+        }
+        finally {
+            Mutex.unlock(key);
+        }
     }
     /**
-     * Find the item in the store.
+     * Find the item in the provided entity array.
+     * @param entities The current entity array.
      * @param id The id to search for.
      * @param secondaryIndex The secondary index to search for.
      * @param conditions The optional conditions to match for the entities.
      * @returns The index of the item if found or -1.
      * @internal
      */
-    findItem(id, secondaryIndex, conditions) {
+    findItem(entities, id, secondaryIndex, conditions) {
         const finalConditions = [];
         if (!Is.empty(secondaryIndex)) {
             finalConditions.push({
@@ -222,7 +518,6 @@ export class MemoryEntityStorageConnector {
             });
         }
         if (Is.arrayValue(conditions)) {
-            // If we haven't added a secondary index condition we need to add the primary key condition.
             if (finalConditions.length === 0) {
                 finalConditions.push({
                     property: this._primaryKey.property,
@@ -237,14 +532,14 @@ export class MemoryEntityStorageConnector {
             })));
         }
         if (finalConditions.length > 0) {
-            for (let i = 0; i < this._store.length; i++) {
-                if (EntityConditions.check(this._store[i], { conditions: finalConditions })) {
+            for (let i = 0; i < entities.length; i++) {
+                if (EntityConditions.check(entities[i], { conditions: finalConditions })) {
                     return i;
                 }
             }
         }
         else {
-            return this._store.findIndex(e => e[this._primaryKey.property] === id);
+            return entities.findIndex(e => e[this._primaryKey.property] === id);
         }
         return -1;
     }