@twin.org/entity-storage-service 0.0.3-next.8 → 0.9.0-next.1

package/dist/es/schemaVersionService.js

@@ -0,0 +1,356 @@
+// Copyright 2026 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+import { ComponentFactory, GeneralError, Is } from "@twin.org/core";
+import { EntitySchemaFactory, EntitySchemaHelper } from "@twin.org/entity";
+import { EntityStorageConnectorFactory, MigrationHelper, SchemaMigrationFactory } from "@twin.org/entity-storage-models";
+import { SchemaVersion } from "./entities/schemaVersion.js";
+/**
+ * Service that checks and applies entity schema migrations at every node start-up.
+ *
+ * This service must be the first entry in coreTypeInitialisers.json. The engine iterates that
+ * array in order to determine start sequence — there is no engine-level priority mechanism, so
+ * registration position is the only guarantee that start() runs before any other service.
+ * By the time start() is called, all component bootstraps have completed (every table already
+ * exists) and EntitySchemaFactory / EntityStorageConnectorFactory are fully populated with every
+ * registered schema and connector.
+ *
+ * Migration mechanics: old schema versions are registered in EntitySchemaFactory by naming
+ * convention — current schema = "MyEntity", first history = "MyEntityV0", second = "MyEntityV1".
+ * The service groups schemas by base name (strips the trailing V number suffix) and resolves the
+ * migration chain automatically by diffing consecutive versioned schemas. For steps that require
+ * property renames or a custom transform hook, register an optional ISchemaMigration entry in
+ * SchemaMigrationFactory under the key "Base_from_to" (e.g. "MyEntity_0_1").
+ *
+ * Crash-window note: finalizeMigration and the subsequent version-record write are two
+ * separate operations. If the process dies between them the next boot re-runs the chain
+ * over already-migrated data. applyEntityTransform is NOT idempotent for structural changes
+ * (newly-added optional fields would be dropped on re-run). A transaction spanning both
+ * writes is a precondition for production; track this in the concurrency follow-up.
+ */
+export class SchemaVersionService {
+    /**
+     * Runtime name for the class.
+     */
+    static CLASS_NAME = "SchemaVersionService";
+    /**
+     * Regex to detect a versioned schema name and extract the base name and version number.
+     * Matches names like "MyEntityV0", "AuditableItemGraphV2", etc.
+     * @internal
+     */
+    static _VERSION_SUFFIX_RE = /^(.+)V(\d+)$/;
+    /**
+     * The connector used to read and write SchemaVersion records.
+     * Not readonly because finalizeMigration may return a replacement connector object.
+     * @internal
+     */
+    _versionConnector;
+    /**
+     * Optional config passed through constructor options.
+     * @internal
+     */
+    _config;
+    /**
+     * Create a new SchemaVersionService.
+     * @param options Optional constructor options.
+     */
+    constructor(options) {
+        this._versionConnector = EntityStorageConnectorFactory.get(options?.schemaVersionStorageType ?? "schema-version");
+        this._config = options?.config;
+    }
+    /**
+     * Returns the class name.
+     * @returns The class name.
+     */
+    className() {
+        return SchemaVersionService.CLASS_NAME;
+    }
+    /**
+     * Reads all registered entity schemas, groups versioned schemas by base name, reads the
+     * full schemaVersion table in one pass, then orchestrates chain migrations for any schema
+     * whose stored version is behind the current version declared in EntitySchemaFactory.
+     * SchemaVersion itself is processed first so the version store is migrated before any
+     * version records are written for other schemas.
+     *
+     * Runs after all component bootstraps, so every managed table already exists.
+     * @param nodeLoggingComponentType An optional logging component type.
+     */
+    async start(nodeLoggingComponentType) {
+        const logging = ComponentFactory.getIfExists(nodeLoggingComponentType);
+        const migrationOptions = {
+            batchSize: this._config?.batchSize,
+            onProgress: async (progressItem, itemTotal, itemIndex) => {
+                await this.logProgress(logging, progressItem, itemTotal, itemIndex);
+            }
+        };
+        // 1. Collect all registered schema names and partition into current vs historical.
+        const allNames = EntitySchemaFactory.names();
+        const historicalByBase = new Map();
+        const currentSchemas = new Map();
+        for (const name of allNames) {
+            const match = SchemaVersionService._VERSION_SUFFIX_RE.exec(name);
+            if (match) {
+                const baseName = match[1];
+                const version = Number.parseInt(match[2], 10);
+                let versions = historicalByBase.get(baseName);
+                if (!versions) {
+                    versions = new Map();
+                    historicalByBase.set(baseName, versions);
+                }
+                versions.set(version, EntitySchemaFactory.get(name));
+            }
+            else {
+                currentSchemas.set(name, EntitySchemaFactory.get(name));
+            }
+        }
+        // 2. Read ALL stored version records, paging through the full table.
+        const storedVersions = new Map();
+        let cursor;
+        do {
+            const queryResult = await this._versionConnector.query(undefined, undefined, undefined, cursor);
+            for (const record of queryResult.entities ?? []) {
+                if (Is.object(record)) {
+                    storedVersions.set(record.schemaName, record.version);
+                }
+            }
+            cursor = queryResult.cursor;
+        } while (Is.stringValue(cursor));
+        // 3. Process SchemaVersion first so the version store itself is fully migrated
+        //    before any version records are written for other schemas.
+        const schemaVersionName = "SchemaVersion";
+        const schemaVersionSchema = currentSchemas.get(schemaVersionName);
+        if (schemaVersionSchema) {
+            currentSchemas.delete(schemaVersionName);
+            await this.processSchema(schemaVersionName, schemaVersionSchema, storedVersions, historicalByBase.get(schemaVersionName), migrationOptions, nodeLoggingComponentType, logging);
+        }
+        // 4. Process all remaining schemas.
+        for (const [schemaName, schema] of currentSchemas) {
+            await this.processSchema(schemaName, schema, storedVersions, historicalByBase.get(schemaName), migrationOptions, nodeLoggingComponentType, logging);
+        }
+    }
+    /**
+     * Checks and applies any pending migration for a single entity schema.
+     * Extracted to avoid continue statements in the outer loop.
+     * @param schemaName The base schema name.
+     * @param schema The current schema definition.
+     * @param storedVersions The full map of stored version records.
+     * @param history The versioned-schema map for this schema (historicalByBase.get(schemaName)), or undefined if none exist.
+     * @param migrationOptions The migration options to pass through to MigrationHelper.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @param logging An optional logging component to pass through to MigrationHelper for migration progress logging.
+     * @internal
+     */
+    async processSchema(schemaName, schema, storedVersions, history, migrationOptions, loggingComponentType, logging) {
+        const currentVersion = EntitySchemaHelper.getVersion(schema);
+        // Find the entity-storage connector whose schema type matches this schema name.
+        // For SchemaVersion itself, use the injected connector directly rather than re-discovering
+        // it through the factory, which could resolve a different instance than _versionConnector.
+        const connectorEntry = schemaName === "SchemaVersion"
+            ? { connector: this._versionConnector, factoryKey: undefined }
+            : this.findConnector(schemaName);
+        if (!connectorEntry) {
+            // No connector registered for this schema — nothing to migrate.
+            return;
+        }
+        const { connector, factoryKey } = connectorEntry;
+        // Resolve the stored version, applying the backwards-compat baseline when no record exists.
+        const stored = storedVersions.get(schemaName);
+        let resolvedStoredVersion;
+        if (stored === undefined) {
+            // No version record: treat as v0 regardless of whether the table has data.
+            // On SQL connectors the table may have a stale column structure even when empty;
+            // running the chain over zero rows still calls finalizeMigration, which reconciles
+            // the table shape via a connector swap.
+            // Deployment precondition: any pre-existing data is genuinely at v0. A deployment
+            // that hand-applied a later schema before this service was introduced would be
+            // incorrectly replayed v0→…→current and should be seeded with an explicit record.
+            resolvedStoredVersion = 0;
+            await this.writeVersion(schemaName, 0);
+        }
+        else {
+            resolvedStoredVersion = stored;
+        }
+        // No-op: stored version already matches current.
+        if (resolvedStoredVersion === currentVersion) {
+            await logging?.log({
+                source: SchemaVersionService.CLASS_NAME,
+                level: "info",
+                message: "noMigrationRequired",
+                data: {
+                    schemaName,
+                    version: resolvedStoredVersion
+                }
+            });
+            return;
+        }
+        await logging?.log({
+            source: SchemaVersionService.CLASS_NAME,
+            level: "info",
+            message: "migrationRequired",
+            data: {
+                schemaName,
+                from: currentVersion,
+                to: resolvedStoredVersion
+            }
+        });
+        // Downgrade — not supported.
+        if (resolvedStoredVersion > currentVersion) {
+            throw new GeneralError(SchemaVersionService.CLASS_NAME, "storedVersionNewer", {
+                schemaName,
+                stored: resolvedStoredVersion,
+                current: currentVersion
+            });
+        }
+        // Migration is needed. If the connector does not support it, throw immediately so
+        // the problem surfaces at boot rather than at runtime when writes hit the wrong table shape.
+        if (!("createTargetConnector" in connector)) {
+            throw new GeneralError(SchemaVersionService.CLASS_NAME, "connectorNotMigrationCapable", {
+                schemaName,
+                stored: resolvedStoredVersion,
+                current: currentVersion
+            });
+        }
+        const migrationConnector = connector;
+        // Upgrade — resolve and run the chain.
+        const steps = [];
+        for (let v = resolvedStoredVersion; v < currentVersion; v++) {
+            const fromSchema = history?.get(v);
+            if (!fromSchema) {
+                throw new GeneralError(SchemaVersionService.CLASS_NAME, "noMigrationStep", {
+                    schemaName,
+                    stored: resolvedStoredVersion,
+                    current: currentVersion,
+                    missingFromVersion: v,
+                    missingToVersion: v + 1
+                });
+            }
+            const toSchema = v + 1 < currentVersion ? history?.get(v + 1) : schema;
+            if (!toSchema) {
+                throw new GeneralError(SchemaVersionService.CLASS_NAME, "noMigrationStepTarget", {
+                    schemaName,
+                    stored: resolvedStoredVersion,
+                    current: currentVersion,
+                    missingFromVersion: v,
+                    missingToVersion: v + 1
+                });
+            }
+            const overrideKey = `${schemaName}_${v}_${v + 1}`;
+            const override = SchemaMigrationFactory.getIfExists(overrideKey);
+            steps.push({
+                fromProperties: fromSchema.properties ?? [],
+                toProperties: toSchema.properties ?? [],
+                renames: override?.renames,
+                transformEntityProperty: override?.transformEntityProperty
+            });
+        }
+        const { finalConnector } = await MigrationHelper.migrateWithChain(migrationConnector, schemaName, steps, migrationOptions, loggingComponentType);
+        // Some connectors (e.g. in-memory) return a brand-new object from finalizeMigration
+        // rather than mutating the source in place.  Re-register the factory entry so that any
+        // subsequent EntityStorageConnectorFactory.get() call returns the migrated instance.
+        if (finalConnector !== connector) {
+            if (factoryKey) {
+                EntityStorageConnectorFactory.register(factoryKey, () => finalConnector);
+            }
+            // For SchemaVersion keep _versionConnector in sync so writeVersion below uses
+            // the migrated instance.
+            if (schemaName === "SchemaVersion") {
+                this._versionConnector = finalConnector;
+            }
+        }
+        // Advance the stored version only after finalizeMigration has succeeded.
+        // See crash-window note in the class comment.
+        await this.writeVersion(schemaName, currentVersion);
+    }
+    /**
+     * Upserts a SchemaVersion record for the given schema name.
+     * @param schemaName The schema type name.
+     * @param version The version to record.
+     * @internal
+     */
+    async writeVersion(schemaName, version) {
+        await this._versionConnector.set({
+            schemaName,
+            version,
+            updatedAt: new Date().toISOString()
+        });
+    }
+    /**
+     * Searches EntityStorageConnectorFactory for the connector whose registered schema type
+     * matches the given schema name.
+     * @param schemaName The entity type name to look up.
+     * @returns The matching connector and its factory key, or undefined if none is registered.
+     * @internal
+     */
+    findConnector(schemaName) {
+        for (const name of EntityStorageConnectorFactory.names()) {
+            try {
+                const connector = EntityStorageConnectorFactory.get(name);
+                if (connector.getSchema?.().type === schemaName) {
+                    return { connector, factoryKey: name };
+                }
+            }
+            catch {
+                // Connector not yet created or registration issue — skip.
+            }
+        }
+        return undefined;
+    }
+    /**
+     * Logs migration progress using the provided logging component, if available.
+     * @param logging The logging component to use for logging progress, if available.
+     * @param progressItem The progress item being updated.
+     * @param itemTotal The total number of items to process for this progress item.
+     * @param itemIndex The index of the current item being processed for this progress item.
+     * @internal
+     */
+    async logProgress(logging, progressItem, itemTotal, itemIndex) {
+        if (progressItem === "partitionStart") {
+            await logging?.log({
+                source: SchemaVersionService.CLASS_NAME,
+                level: "info",
+                message: "partitionStart",
+                data: { progressItem, itemTotal, itemIndex }
+            });
+        }
+        else if (progressItem === "partitionProgress") {
+            await logging?.log({
+                source: SchemaVersionService.CLASS_NAME,
+                level: "info",
+                message: "partitionProgress",
+                data: { progressItem, itemTotal, itemIndex }
+            });
+        }
+        else if (progressItem === "partitionEnd") {
+            await logging?.log({
+                source: SchemaVersionService.CLASS_NAME,
+                level: "info",
+                message: "partitionEnd",
+                data: { progressItem, itemTotal, itemIndex }
+            });
+        }
+        else if (progressItem === "partitionItemsStart") {
+            await logging?.log({
+                source: SchemaVersionService.CLASS_NAME,
+                level: "info",
+                message: "partitionItemsStart",
+                data: { progressItem, itemTotal, itemIndex }
+            });
+        }
+        else if (progressItem === "partitionItemsProgress") {
+            await logging?.log({
+                source: SchemaVersionService.CLASS_NAME,
+                level: "info",
+                message: "partitionItemsProgress",
+                data: { progressItem, itemTotal, itemIndex }
+            });
+        }
+        else if (progressItem === "partitionItemsEnd") {
+            await logging?.log({
+                source: SchemaVersionService.CLASS_NAME,
+                level: "info",
+                message: "partitionItemsEnd",
+                data: { progressItem, itemTotal, itemIndex }
+            });
+        }
+    }
+}
+//# sourceMappingURL=schemaVersionService.js.map

package/dist/es/schemaVersionService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemaVersionService.js","sourceRoot":"","sources":["../../src/schemaVersionService.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,uCAAuC;AACvC,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,EAAE,EAAmB,MAAM,gBAAgB,CAAC;AACrF,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAsB,MAAM,kBAAkB,CAAC;AAC/F,OAAO,EACN,6BAA6B,EAK7B,eAAe,EACf,sBAAsB,EACtB,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAI5D;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,OAAO,oBAAoB;IAChC;;OAEG;IACI,MAAM,CAAU,UAAU,0BAA0C;IAE3E;;;;OAIG;IACK,MAAM,CAAU,kBAAkB,GAAG,cAAc,CAAC;IAE5D;;;;OAIG;IACK,iBAAiB,CAAyC;IAElE;;;OAGG;IACc,OAAO,CAA+B;IAEvD;;;OAGG;IACH,YAAY,OAAiD;QAC5D,IAAI,CAAC,iBAAiB,GAAG,6BAA6B,CAAC,GAAG,CACzD,OAAO,EAAE,wBAAwB,IAAI,gBAAgB,CACrD,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,MAAM,CAAC;IAChC,CAAC;IAED;;;OAGG;IACI,SAAS;QACf,OAAO,oBAAoB,CAAC,UAAU,CAAC;IACxC,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,KAAK,CAAC,wBAAiC;QACnD,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,CAAoB,wBAAwB,CAAC,CAAC;QAE1F,MAAM,gBAAgB,GAAsB;YAC3C,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS;YAClC,UAAU,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE;gBACxD,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YACrE,CAAC;SACD,CAAC;QAEF,mFAAmF;QACnF,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,EAAE,CAAC;QAE7C,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAsC,CAAC;QACvE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAyB,CAAC;QAExD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjE,IAAI,KAAK,EAAE,CAAC;gBACX,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9C,IAAI,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;oBACrB,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC1C,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;YACtD,CAAC;iBAAM,CAAC;gBACP,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;YACzD,CAAC;QACF,CAAC;QAED,qEAAqE;QACrE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QACjD,IAAI,MAA0B,CAAC;QAC/B,GAAG,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CACrD,SAAS,EACT,SAAS,EACT,SAAS,EACT,MAAM,CACN,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;gBACjD,IAAI,EAAE,CAAC,MAAM,CAAgB,MAAM,CAAC,EAAE,CAAC;oBACtC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;gBACvD,CAAC;YACF,CAAC;YACD,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;QAC7B,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QAEjC,+EAA+E;QAC/E,+DAA+D;QAC/D,MAAM,iBAAiB,kBAAwB,CAAC;QAChD,MAAM,mBAAmB,GAAG,cAAc,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAClE,IAAI,mBAAmB,EAAE,CAAC;YACzB,cAAc,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACzC,MAAM,IAAI,CAAC,aAAa,CACvB,iBAAiB,EACjB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,CAAC,GAAG,CAAC,iBAAiB,CAAC,EACvC,gBAAgB,EAChB,wBAAwB,EACxB,OAAO,CACP,CAAC;QACH,CAAC;QAED,oCAAoC;QACpC,KAAK,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YACnD,MAAM,IAAI,CAAC,aAAa,CACvB,UAAU,EACV,MAAM,EACN,cAAc,EACd,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,EAChC,gBAAgB,EAChB,wBAAwB,EACxB,OAAO,CACP,CAAC;QACH,CAAC;IACF,CAAC;IAED;;;;;;;;;;;OAWG;IACK,KAAK,CAAC,aAAa,CAC1B,UAAkB,EAClB,MAAqB,EACrB,cAAmC,EACnC,OAA+C,EAC/C,gBAAmC,EACnC,oBAAwC,EACxC,OAAsC;QAEtC,MAAM,cAAc,GAAG,kBAAkB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE7D,gFAAgF;QAChF,2FAA2F;QAC3F,2FAA2F;QAC3F,MAAM,cAAc,GACnB,UAAU,oBAA0B;YACnC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,iBAA4C,EAAE,UAAU,EAAE,SAAS,EAAE;YACzF,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,gEAAgE;YAChE,OAAO;QACR,CAAC;QACD,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QAEjD,4FAA4F;QAC5F,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC9C,IAAI,qBAA6B,CAAC;QAElC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAC1B,2EAA2E;YAC3E,iFAAiF;YACjF,mFAAmF;YACnF,wCAAwC;YACxC,kFAAkF;YAClF,+EAA+E;YAC/E,kFAAkF;YAClF,qBAAqB,GAAG,CAAC,CAAC;YAC1B,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC;aAAM,CAAC;YACP,qBAAqB,GAAG,MAAM,CAAC;QAChC,CAAC;QAED,iDAAiD;QACjD,IAAI,qBAAqB,KAAK,cAAc,EAAE,CAAC;YAC9C,MAAM,OAAO,EAAE,GAAG,CAAC;gBAClB,MAAM,EAAE,oBAAoB,CAAC,UAAU;gBACvC,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,qBAAqB;gBAC9B,IAAI,EAAE;oBACL,UAAU;oBACV,OAAO,EAAE,qBAAqB;iBAC9B;aACD,CAAC,CAAC;YACH,OAAO;QACR,CAAC;QAED,MAAM,OAAO,EAAE,GAAG,CAAC;YAClB,MAAM,EAAE,oBAAoB,CAAC,UAAU;YACvC,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,mBAAmB;YAC5B,IAAI,EAAE;gBACL,UAAU;gBACV,IAAI,EAAE,cAAc;gBACpB,EAAE,EAAE,qBAAqB;aACzB;SACD,CAAC,CAAC;QAEH,6BAA6B;QAC7B,IAAI,qBAAqB,GAAG,cAAc,EAAE,CAAC;YAC5C,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,UAAU,EAAE,oBAAoB,EAAE;gBAC7E,UAAU;gBACV,MAAM,EAAE,qBAAqB;gBAC7B,OAAO,EAAE,cAAc;aACvB,CAAC,CAAC;QACJ,CAAC;QAED,kFAAkF;QAClF,6FAA6F;QAC7F,IAAI,CAAC,CAAC,uBAAuB,IAAI,SAAS,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,UAAU,EAAE,8BAA8B,EAAE;gBACvF,UAAU;gBACV,MAAM,EAAE,qBAAqB;gBAC7B,OAAO,EAAE,cAAc;aACvB,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,kBAAkB,GAAG,SAA6C,CAAC;QAEzE,uCAAuC;QACvC,MAAM,KAAK,GAA6B,EAAE,CAAC;QAE3C,KAAK,IAAI,CAAC,GAAG,qBAAqB,EAAE,CAAC,GAAG,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAG,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACjB,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,UAAU,EAAE,iBAAiB,EAAE;oBAC1E,UAAU;oBACV,MAAM,EAAE,qBAAqB;oBAC7B,OAAO,EAAE,cAAc;oBACvB,kBAAkB,EAAE,CAAC;oBACrB,gBAAgB,EAAE,CAAC,GAAG,CAAC;iBACvB,CAAC,CAAC;YACJ,CAAC;YAED,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACvE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,UAAU,EAAE,uBAAuB,EAAE;oBAChF,UAAU;oBACV,MAAM,EAAE,qBAAqB;oBAC7B,OAAO,EAAE,cAAc;oBACvB,kBAAkB,EAAE,CAAC;oBACrB,gBAAgB,EAAE,CAAC,GAAG,CAAC;iBACvB,CAAC,CAAC;YACJ,CAAC;YAED,MAAM,WAAW,GAAG,GAAG,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,QAAQ,GAAG,sBAAsB,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YAEjE,KAAK,CAAC,IAAI,CAAC;gBACV,cAAc,EAAE,UAAU,CAAC,UAAU,IAAI,EAAE;gBAC3C,YAAY,EAAE,QAAQ,CAAC,UAAU,IAAI,EAAE;gBACvC,OAAO,EAAE,QAAQ,EAAE,OAAO;gBAC1B,uBAAuB,EAAE,QAAQ,EAAE,uBAAuB;aAC1D,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAChE,kBAAkB,EAClB,UAAU,EACV,KAAK,EACL,gBAAgB,EAChB,oBAAoB,CACpB,CAAC;QAEF,oFAAoF;QACpF,uFAAuF;QACvF,qFAAqF;QACrF,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,UAAU,EAAE,CAAC;gBAChB,6BAA6B,CAAC,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC;YAC1E,CAAC;YACD,8EAA8E;YAC9E,yBAAyB;YACzB,IAAI,UAAU,oBAA0B,EAAE,CAAC;gBAC1C,IAAI,CAAC,iBAAiB,GAAG,cAAwD,CAAC;YACnF,CAAC;QACF,CAAC;QAED,yEAAyE;QACzE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IACrD,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,YAAY,CAAC,UAAkB,EAAE,OAAe;QAC7D,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC;YAChC,UAAU;YACV,OAAO;YACP,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACnC,CAAC,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CACpB,UAAkB;QAElB,KAAK,MAAM,IAAI,IAAI,6BAA6B,CAAC,KAAK,EAAE,EAAE,CAAC;YAC1D,IAAI,CAAC;gBACJ,MAAM,SAAS,GAAG,6BAA6B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC1D,IAAI,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;oBACjD,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;gBACxC,CAAC;YACF,CAAC;YAAC,MAAM,CAAC;gBACR,0DAA0D;YAC3D,CAAC;QACF,CAAC;QACD,OAAO,SAAS,CAAC;IAClB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,WAAW,CACxB,OAAsC,EACtC,YAAoB,EACpB,SAAiB,EACjB,SAAiB;QAEjB,IAAI,YAAY,KAAK,gBAAgB,EAAE,CAAC;YACvC,MAAM,OAAO,EAAE,GAAG,CAAC;gBAClB,MAAM,EAAE,oBAAoB,CAAC,UAAU;gBACvC,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,gBAAgB;gBACzB,IAAI,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE;aAC5C,CAAC,CAAC;QACJ,CAAC;aAAM,IAAI,YAAY,KAAK,mBAAmB,EAAE,CAAC;YACjD,MAAM,OAAO,EAAE,GAAG,CAAC;gBAClB,MAAM,EAAE,oBAAoB,CAAC,UAAU;gBACvC,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,mBAAmB;gBAC5B,IAAI,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE;aAC5C,CAAC,CAAC;QACJ,CAAC;aAAM,IAAI,YAAY,KAAK,cAAc,EAAE,CAAC;YAC5C,MAAM,OAAO,EAAE,GAAG,CAAC;gBAClB,MAAM,EAAE,oBAAoB,CAAC,UAAU;gBACvC,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,cAAc;gBACvB,IAAI,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE;aAC5C,CAAC,CAAC;QACJ,CAAC;aAAM,IAAI,YAAY,KAAK,qBAAqB,EAAE,CAAC;YACnD,MAAM,OAAO,EAAE,GAAG,CAAC;gBAClB,MAAM,EAAE,oBAAoB,CAAC,UAAU;gBACvC,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,qBAAqB;gBAC9B,IAAI,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE;aAC5C,CAAC,CAAC;QACJ,CAAC;aAAM,IAAI,YAAY,KAAK,wBAAwB,EAAE,CAAC;YACtD,MAAM,OAAO,EAAE,GAAG,CAAC;gBAClB,MAAM,EAAE,oBAAoB,CAAC,UAAU;gBACvC,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,wBAAwB;gBACjC,IAAI,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE;aAC5C,CAAC,CAAC;QACJ,CAAC;aAAM,IAAI,YAAY,KAAK,mBAAmB,EAAE,CAAC;YACjD,MAAM,OAAO,EAAE,GAAG,CAAC;gBAClB,MAAM,EAAE,oBAAoB,CAAC,UAAU;gBACvC,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,mBAAmB;gBAC5B,IAAI,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE;aAC5C,CAAC,CAAC;QACJ,CAAC;IACF,CAAC","sourcesContent":["// Copyright 2026 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport { ComponentFactory, GeneralError, Is, type IComponent } from \"@twin.org/core\";\nimport { EntitySchemaFactory, EntitySchemaHelper, type IEntitySchema } from \"@twin.org/entity\";\nimport {\n\tEntityStorageConnectorFactory,\n\ttype IEntityStorageConnector,\n\ttype IEntityStorageMigrationConnector,\n\ttype IMigrationOptions,\n\ttype IResolvedMigrationStep,\n\tMigrationHelper,\n\tSchemaMigrationFactory\n} from \"@twin.org/entity-storage-models\";\nimport type { ILoggingComponent } from \"@twin.org/logging-models\";\nimport { nameof } from \"@twin.org/nameof\";\nimport { SchemaVersion } from \"./entities/schemaVersion.js\";\nimport type { ISchemaVersionServiceConfig } from \"./models/ISchemaVersionServiceConfig.js\";\nimport type { ISchemaVersionServiceConstructorOptions } from \"./models/ISchemaVersionServiceConstructorOptions.js\";\n\n/**\n * Service that checks and applies entity schema migrations at every node start-up.\n *\n * This service must be the first entry in coreTypeInitialisers.json. The engine iterates that\n * array in order to determine start sequence — there is no engine-level priority mechanism, so\n * registration position is the only guarantee that start() runs before any other service.\n * By the time start() is called, all component bootstraps have completed (every table already\n * exists) and EntitySchemaFactory / EntityStorageConnectorFactory are fully populated with every\n * registered schema and connector.\n *\n * Migration mechanics: old schema versions are registered in EntitySchemaFactory by naming\n * convention — current schema = \"MyEntity\", first history = \"MyEntityV0\", second = \"MyEntityV1\".\n * The service groups schemas by base name (strips the trailing V number suffix) and resolves the\n * migration chain automatically by diffing consecutive versioned schemas. For steps that require\n * property renames or a custom transform hook, register an optional ISchemaMigration entry in\n * SchemaMigrationFactory under the key \"Base_from_to\" (e.g. \"MyEntity_0_1\").\n *\n * Crash-window note: finalizeMigration and the subsequent version-record write are two\n * separate operations. If the process dies between them the next boot re-runs the chain\n * over already-migrated data. applyEntityTransform is NOT idempotent for structural changes\n * (newly-added optional fields would be dropped on re-run). A transaction spanning both\n * writes is a precondition for production; track this in the concurrency follow-up.\n */\nexport class SchemaVersionService implements IComponent {\n\t/**\n\t * Runtime name for the class.\n\t */\n\tpublic static readonly CLASS_NAME: string = nameof<SchemaVersionService>();\n\n\t/**\n\t * Regex to detect a versioned schema name and extract the base name and version number.\n\t * Matches names like \"MyEntityV0\", \"AuditableItemGraphV2\", etc.\n\t * @internal\n\t */\n\tprivate static readonly _VERSION_SUFFIX_RE = /^(.+)V(\\d+)$/;\n\n\t/**\n\t * The connector used to read and write SchemaVersion records.\n\t * Not readonly because finalizeMigration may return a replacement connector object.\n\t * @internal\n\t */\n\tprivate _versionConnector: IEntityStorageConnector<SchemaVersion>;\n\n\t/**\n\t * Optional config passed through constructor options.\n\t * @internal\n\t */\n\tprivate readonly _config?: ISchemaVersionServiceConfig;\n\n\t/**\n\t * Create a new SchemaVersionService.\n\t * @param options Optional constructor options.\n\t */\n\tconstructor(options?: ISchemaVersionServiceConstructorOptions) {\n\t\tthis._versionConnector = EntityStorageConnectorFactory.get(\n\t\t\toptions?.schemaVersionStorageType ?? \"schema-version\"\n\t\t);\n\t\tthis._config = options?.config;\n\t}\n\n\t/**\n\t * Returns the class name.\n\t * @returns The class name.\n\t */\n\tpublic className(): string {\n\t\treturn SchemaVersionService.CLASS_NAME;\n\t}\n\n\t/**\n\t * Reads all registered entity schemas, groups versioned schemas by base name, reads the\n\t * full schemaVersion table in one pass, then orchestrates chain migrations for any schema\n\t * whose stored version is behind the current version declared in EntitySchemaFactory.\n\t * SchemaVersion itself is processed first so the version store is migrated before any\n\t * version records are written for other schemas.\n\t *\n\t * Runs after all component bootstraps, so every managed table already exists.\n\t * @param nodeLoggingComponentType An optional logging component type.\n\t */\n\tpublic async start(nodeLoggingComponentType?: string): Promise<void> {\n\t\tconst logging = ComponentFactory.getIfExists<ILoggingComponent>(nodeLoggingComponentType);\n\n\t\tconst migrationOptions: IMigrationOptions = {\n\t\t\tbatchSize: this._config?.batchSize,\n\t\t\tonProgress: async (progressItem, itemTotal, itemIndex) => {\n\t\t\t\tawait this.logProgress(logging, progressItem, itemTotal, itemIndex);\n\t\t\t}\n\t\t};\n\n\t\t// 1. Collect all registered schema names and partition into current vs historical.\n\t\tconst allNames = EntitySchemaFactory.names();\n\n\t\tconst historicalByBase = new Map<string, Map<number, IEntitySchema>>();\n\t\tconst currentSchemas = new Map<string, IEntitySchema>();\n\n\t\tfor (const name of allNames) {\n\t\t\tconst match = SchemaVersionService._VERSION_SUFFIX_RE.exec(name);\n\t\t\tif (match) {\n\t\t\t\tconst baseName = match[1];\n\t\t\t\tconst version = Number.parseInt(match[2], 10);\n\t\t\t\tlet versions = historicalByBase.get(baseName);\n\t\t\t\tif (!versions) {\n\t\t\t\t\tversions = new Map();\n\t\t\t\t\thistoricalByBase.set(baseName, versions);\n\t\t\t\t}\n\t\t\t\tversions.set(version, EntitySchemaFactory.get(name));\n\t\t\t} else {\n\t\t\t\tcurrentSchemas.set(name, EntitySchemaFactory.get(name));\n\t\t\t}\n\t\t}\n\n\t\t// 2. Read ALL stored version records, paging through the full table.\n\t\tconst storedVersions = new Map<string, number>();\n\t\tlet cursor: string | undefined;\n\t\tdo {\n\t\t\tconst queryResult = await this._versionConnector.query(\n\t\t\t\tundefined,\n\t\t\t\tundefined,\n\t\t\t\tundefined,\n\t\t\t\tcursor\n\t\t\t);\n\t\t\tfor (const record of queryResult.entities ?? []) {\n\t\t\t\tif (Is.object<SchemaVersion>(record)) {\n\t\t\t\t\tstoredVersions.set(record.schemaName, record.version);\n\t\t\t\t}\n\t\t\t}\n\t\t\tcursor = queryResult.cursor;\n\t\t} while (Is.stringValue(cursor));\n\n\t\t// 3. Process SchemaVersion first so the version store itself is fully migrated\n\t\t//    before any version records are written for other schemas.\n\t\tconst schemaVersionName = nameof(SchemaVersion);\n\t\tconst schemaVersionSchema = currentSchemas.get(schemaVersionName);\n\t\tif (schemaVersionSchema) {\n\t\t\tcurrentSchemas.delete(schemaVersionName);\n\t\t\tawait this.processSchema(\n\t\t\t\tschemaVersionName,\n\t\t\t\tschemaVersionSchema,\n\t\t\t\tstoredVersions,\n\t\t\t\thistoricalByBase.get(schemaVersionName),\n\t\t\t\tmigrationOptions,\n\t\t\t\tnodeLoggingComponentType,\n\t\t\t\tlogging\n\t\t\t);\n\t\t}\n\n\t\t// 4. Process all remaining schemas.\n\t\tfor (const [schemaName, schema] of currentSchemas) {\n\t\t\tawait this.processSchema(\n\t\t\t\tschemaName,\n\t\t\t\tschema,\n\t\t\t\tstoredVersions,\n\t\t\t\thistoricalByBase.get(schemaName),\n\t\t\t\tmigrationOptions,\n\t\t\t\tnodeLoggingComponentType,\n\t\t\t\tlogging\n\t\t\t);\n\t\t}\n\t}\n\n\t/**\n\t * Checks and applies any pending migration for a single entity schema.\n\t * Extracted to avoid continue statements in the outer loop.\n\t * @param schemaName The base schema name.\n\t * @param schema The current schema definition.\n\t * @param storedVersions The full map of stored version records.\n\t * @param history The versioned-schema map for this schema (historicalByBase.get(schemaName)), or undefined if none exist.\n\t * @param migrationOptions The migration options to pass through to MigrationHelper.\n\t * @param loggingComponentType The optional component type to use for logging the migration progress.\n\t * @param logging An optional logging component to pass through to MigrationHelper for migration progress logging.\n\t * @internal\n\t */\n\tprivate async processSchema(\n\t\tschemaName: string,\n\t\tschema: IEntitySchema,\n\t\tstoredVersions: Map<string, number>,\n\t\thistory: Map<number, IEntitySchema> | undefined,\n\t\tmigrationOptions: IMigrationOptions,\n\t\tloggingComponentType: string | undefined,\n\t\tlogging: ILoggingComponent | undefined\n\t): Promise<void> {\n\t\tconst currentVersion = EntitySchemaHelper.getVersion(schema);\n\n\t\t// Find the entity-storage connector whose schema type matches this schema name.\n\t\t// For SchemaVersion itself, use the injected connector directly rather than re-discovering\n\t\t// it through the factory, which could resolve a different instance than _versionConnector.\n\t\tconst connectorEntry =\n\t\t\tschemaName === nameof(SchemaVersion)\n\t\t\t\t? { connector: this._versionConnector as IEntityStorageConnector, factoryKey: undefined }\n\t\t\t\t: this.findConnector(schemaName);\n\t\tif (!connectorEntry) {\n\t\t\t// No connector registered for this schema — nothing to migrate.\n\t\t\treturn;\n\t\t}\n\t\tconst { connector, factoryKey } = connectorEntry;\n\n\t\t// Resolve the stored version, applying the backwards-compat baseline when no record exists.\n\t\tconst stored = storedVersions.get(schemaName);\n\t\tlet resolvedStoredVersion: number;\n\n\t\tif (stored === undefined) {\n\t\t\t// No version record: treat as v0 regardless of whether the table has data.\n\t\t\t// On SQL connectors the table may have a stale column structure even when empty;\n\t\t\t// running the chain over zero rows still calls finalizeMigration, which reconciles\n\t\t\t// the table shape via a connector swap.\n\t\t\t// Deployment precondition: any pre-existing data is genuinely at v0. A deployment\n\t\t\t// that hand-applied a later schema before this service was introduced would be\n\t\t\t// incorrectly replayed v0→…→current and should be seeded with an explicit record.\n\t\t\tresolvedStoredVersion = 0;\n\t\t\tawait this.writeVersion(schemaName, 0);\n\t\t} else {\n\t\t\tresolvedStoredVersion = stored;\n\t\t}\n\n\t\t// No-op: stored version already matches current.\n\t\tif (resolvedStoredVersion === currentVersion) {\n\t\t\tawait logging?.log({\n\t\t\t\tsource: SchemaVersionService.CLASS_NAME,\n\t\t\t\tlevel: \"info\",\n\t\t\t\tmessage: \"noMigrationRequired\",\n\t\t\t\tdata: {\n\t\t\t\t\tschemaName,\n\t\t\t\t\tversion: resolvedStoredVersion\n\t\t\t\t}\n\t\t\t});\n\t\t\treturn;\n\t\t}\n\n\t\tawait logging?.log({\n\t\t\tsource: SchemaVersionService.CLASS_NAME,\n\t\t\tlevel: \"info\",\n\t\t\tmessage: \"migrationRequired\",\n\t\t\tdata: {\n\t\t\t\tschemaName,\n\t\t\t\tfrom: currentVersion,\n\t\t\t\tto: resolvedStoredVersion\n\t\t\t}\n\t\t});\n\n\t\t// Downgrade — not supported.\n\t\tif (resolvedStoredVersion > currentVersion) {\n\t\t\tthrow new GeneralError(SchemaVersionService.CLASS_NAME, \"storedVersionNewer\", {\n\t\t\t\tschemaName,\n\t\t\t\tstored: resolvedStoredVersion,\n\t\t\t\tcurrent: currentVersion\n\t\t\t});\n\t\t}\n\n\t\t// Migration is needed. If the connector does not support it, throw immediately so\n\t\t// the problem surfaces at boot rather than at runtime when writes hit the wrong table shape.\n\t\tif (!(\"createTargetConnector\" in connector)) {\n\t\t\tthrow new GeneralError(SchemaVersionService.CLASS_NAME, \"connectorNotMigrationCapable\", {\n\t\t\t\tschemaName,\n\t\t\t\tstored: resolvedStoredVersion,\n\t\t\t\tcurrent: currentVersion\n\t\t\t});\n\t\t}\n\n\t\tconst migrationConnector = connector as IEntityStorageMigrationConnector;\n\n\t\t// Upgrade — resolve and run the chain.\n\t\tconst steps: IResolvedMigrationStep[] = [];\n\n\t\tfor (let v = resolvedStoredVersion; v < currentVersion; v++) {\n\t\t\tconst fromSchema = history?.get(v);\n\t\t\tif (!fromSchema) {\n\t\t\t\tthrow new GeneralError(SchemaVersionService.CLASS_NAME, \"noMigrationStep\", {\n\t\t\t\t\tschemaName,\n\t\t\t\t\tstored: resolvedStoredVersion,\n\t\t\t\t\tcurrent: currentVersion,\n\t\t\t\t\tmissingFromVersion: v,\n\t\t\t\t\tmissingToVersion: v + 1\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tconst toSchema = v + 1 < currentVersion ? history?.get(v + 1) : schema;\n\t\t\tif (!toSchema) {\n\t\t\t\tthrow new GeneralError(SchemaVersionService.CLASS_NAME, \"noMigrationStepTarget\", {\n\t\t\t\t\tschemaName,\n\t\t\t\t\tstored: resolvedStoredVersion,\n\t\t\t\t\tcurrent: currentVersion,\n\t\t\t\t\tmissingFromVersion: v,\n\t\t\t\t\tmissingToVersion: v + 1\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tconst overrideKey = `${schemaName}_${v}_${v + 1}`;\n\t\t\tconst override = SchemaMigrationFactory.getIfExists(overrideKey);\n\n\t\t\tsteps.push({\n\t\t\t\tfromProperties: fromSchema.properties ?? [],\n\t\t\t\ttoProperties: toSchema.properties ?? [],\n\t\t\t\trenames: override?.renames,\n\t\t\t\ttransformEntityProperty: override?.transformEntityProperty\n\t\t\t});\n\t\t}\n\n\t\tconst { finalConnector } = await MigrationHelper.migrateWithChain(\n\t\t\tmigrationConnector,\n\t\t\tschemaName,\n\t\t\tsteps,\n\t\t\tmigrationOptions,\n\t\t\tloggingComponentType\n\t\t);\n\n\t\t// Some connectors (e.g. in-memory) return a brand-new object from finalizeMigration\n\t\t// rather than mutating the source in place.  Re-register the factory entry so that any\n\t\t// subsequent EntityStorageConnectorFactory.get() call returns the migrated instance.\n\t\tif (finalConnector !== connector) {\n\t\t\tif (factoryKey) {\n\t\t\t\tEntityStorageConnectorFactory.register(factoryKey, () => finalConnector);\n\t\t\t}\n\t\t\t// For SchemaVersion keep _versionConnector in sync so writeVersion below uses\n\t\t\t// the migrated instance.\n\t\t\tif (schemaName === nameof(SchemaVersion)) {\n\t\t\t\tthis._versionConnector = finalConnector as IEntityStorageConnector<SchemaVersion>;\n\t\t\t}\n\t\t}\n\n\t\t// Advance the stored version only after finalizeMigration has succeeded.\n\t\t// See crash-window note in the class comment.\n\t\tawait this.writeVersion(schemaName, currentVersion);\n\t}\n\n\t/**\n\t * Upserts a SchemaVersion record for the given schema name.\n\t * @param schemaName The schema type name.\n\t * @param version The version to record.\n\t * @internal\n\t */\n\tprivate async writeVersion(schemaName: string, version: number): Promise<void> {\n\t\tawait this._versionConnector.set({\n\t\t\tschemaName,\n\t\t\tversion,\n\t\t\tupdatedAt: new Date().toISOString()\n\t\t});\n\t}\n\n\t/**\n\t * Searches EntityStorageConnectorFactory for the connector whose registered schema type\n\t * matches the given schema name.\n\t * @param schemaName The entity type name to look up.\n\t * @returns The matching connector and its factory key, or undefined if none is registered.\n\t * @internal\n\t */\n\tprivate findConnector(\n\t\tschemaName: string\n\t): { connector: IEntityStorageConnector; factoryKey: string } | undefined {\n\t\tfor (const name of EntityStorageConnectorFactory.names()) {\n\t\t\ttry {\n\t\t\t\tconst connector = EntityStorageConnectorFactory.get(name);\n\t\t\t\tif (connector.getSchema?.().type === schemaName) {\n\t\t\t\t\treturn { connector, factoryKey: name };\n\t\t\t\t}\n\t\t\t} catch {\n\t\t\t\t// Connector not yet created or registration issue — skip.\n\t\t\t}\n\t\t}\n\t\treturn undefined;\n\t}\n\n\t/**\n\t * Logs migration progress using the provided logging component, if available.\n\t * @param logging The logging component to use for logging progress, if available.\n\t * @param progressItem The progress item being updated.\n\t * @param itemTotal The total number of items to process for this progress item.\n\t * @param itemIndex The index of the current item being processed for this progress item.\n\t * @internal\n\t */\n\tprivate async logProgress(\n\t\tlogging: ILoggingComponent | undefined,\n\t\tprogressItem: string,\n\t\titemTotal: number,\n\t\titemIndex: number\n\t): Promise<void> {\n\t\tif (progressItem === \"partitionStart\") {\n\t\t\tawait logging?.log({\n\t\t\t\tsource: SchemaVersionService.CLASS_NAME,\n\t\t\t\tlevel: \"info\",\n\t\t\t\tmessage: \"partitionStart\",\n\t\t\t\tdata: { progressItem, itemTotal, itemIndex }\n\t\t\t});\n\t\t} else if (progressItem === \"partitionProgress\") {\n\t\t\tawait logging?.log({\n\t\t\t\tsource: SchemaVersionService.CLASS_NAME,\n\t\t\t\tlevel: \"info\",\n\t\t\t\tmessage: \"partitionProgress\",\n\t\t\t\tdata: { progressItem, itemTotal, itemIndex }\n\t\t\t});\n\t\t} else if (progressItem === \"partitionEnd\") {\n\t\t\tawait logging?.log({\n\t\t\t\tsource: SchemaVersionService.CLASS_NAME,\n\t\t\t\tlevel: \"info\",\n\t\t\t\tmessage: \"partitionEnd\",\n\t\t\t\tdata: { progressItem, itemTotal, itemIndex }\n\t\t\t});\n\t\t} else if (progressItem === \"partitionItemsStart\") {\n\t\t\tawait logging?.log({\n\t\t\t\tsource: SchemaVersionService.CLASS_NAME,\n\t\t\t\tlevel: \"info\",\n\t\t\t\tmessage: \"partitionItemsStart\",\n\t\t\t\tdata: { progressItem, itemTotal, itemIndex }\n\t\t\t});\n\t\t} else if (progressItem === \"partitionItemsProgress\") {\n\t\t\tawait logging?.log({\n\t\t\t\tsource: SchemaVersionService.CLASS_NAME,\n\t\t\t\tlevel: \"info\",\n\t\t\t\tmessage: \"partitionItemsProgress\",\n\t\t\t\tdata: { progressItem, itemTotal, itemIndex }\n\t\t\t});\n\t\t} else if (progressItem === \"partitionItemsEnd\") {\n\t\t\tawait logging?.log({\n\t\t\t\tsource: SchemaVersionService.CLASS_NAME,\n\t\t\t\tlevel: \"info\",\n\t\t\t\tmessage: \"partitionItemsEnd\",\n\t\t\t\tdata: { progressItem, itemTotal, itemIndex }\n\t\t\t});\n\t\t}\n\t}\n}\n"]}

package/dist/types/entities/schemaVersion.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Tracks the currently applied schema version for each managed entity schema.
+ * One record per schema name. Written once on first boot, then updated after
+ * each successful migration.
+ */
+export declare class SchemaVersion {
+    /**
+     * The entity schema type name — primary key.
+     */
+    schemaName: string;
+    /**
+     * The currently deployed version of this schema.
+     */
+    version: number;
+    /**
+     * ISO 8601 timestamp of the last version write.
+     */
+    updatedAt: string;
+}

package/dist/types/entityStorageRoutes.d.ts

@@ -1,5 +1,5 @@
 import { type IHttpRequestContext, type INoContentResponse, type IRestRoute, type ITag } from "@twin.org/api-models";
-import type { IEntityStorageGetRequest, IEntityStorageGetResponse, IEntityStorageListRequest, IEntityStorageListResponse, IEntityStorageRemoveRequest, IEntityStorageSetRequest } from "@twin.org/entity-storage-models";
+import type { IEntityStorageCountRequest, IEntityStorageCountResponse, IEntityStorageEmptyRequest, IEntityStorageGetRequest, IEntityStorageGetResponse, IEntityStorageListRequest, IEntityStorageListResponse, IEntityStorageRemoveBatchRequest, IEntityStorageRemoveRequest, IEntityStorageSetBatchRequest, IEntityStorageSetRequest } from "@twin.org/entity-storage-models";
 import type { IEntityStorageRoutesExamples } from "./models/IEntityStorageRoutesExamples.js";
 /**
  * The tag to associate with the routes.
@@ -28,6 +28,22 @@ export declare function generateRestRoutesEntityStorage(baseRouteName: string, c
  * @returns The response object with additional http response properties.
  */
 export declare function entityStorageSet(httpRequestContext: IHttpRequestContext, componentName: string, request: IEntityStorageSetRequest): Promise<INoContentResponse>;
+/**
+ * Set multiple entries in entity storage.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function entityStorageSetBatch(httpRequestContext: IHttpRequestContext, componentName: string, request: IEntityStorageSetBatchRequest): Promise<INoContentResponse>;
+/**
+ * Remove all entries from entity storage.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function entityStorageEmpty(httpRequestContext: IHttpRequestContext, componentName: string, request: IEntityStorageEmptyRequest): Promise<INoContentResponse>;
 /**
  * Get the entry from entity storage.
  * @param httpRequestContext The request context for the API.
@@ -52,3 +68,19 @@ export declare function entityStorageRemove(httpRequestContext: IHttpRequestCont
  * @returns The response object with additional http response properties.
  */
 export declare function entityStorageList(httpRequestContext: IHttpRequestContext, componentName: string, request: IEntityStorageListRequest): Promise<IEntityStorageListResponse>;
+/**
+ * Count the entries in entity storage.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function entityStorageCount(httpRequestContext: IHttpRequestContext, componentName: string, request: IEntityStorageCountRequest): Promise<IEntityStorageCountResponse>;
+/**
+ * Remove multiple entries from entity storage by id.
+ * @param httpRequestContext The request context for the API.
+ * @param componentName The name of the component to use in the routes.
+ * @param request The request.
+ * @returns The response object with additional http response properties.
+ */
+export declare function entityStorageRemoveBatch(httpRequestContext: IHttpRequestContext, componentName: string, request: IEntityStorageRemoveBatchRequest): Promise<INoContentResponse>;

package/dist/types/entityStorageService.d.ts

@@ -22,22 +22,57 @@ export declare class EntityStorageService<T = any> implements IEntityStorageComp
     /**
      * Set an entity.
      * @param entity The entity to set.
+     * @param conditions The optional conditions to match for the entities.
      * @returns The id of the entity.
      */
-    set(entity: T): Promise<void>;
+    set(entity: T, conditions?: {
+        property: keyof T;
+        value: unknown;
+    }[]): Promise<void>;
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    setBatch(entities: T[]): Promise<void>;
     /**
      * Get an entity.
      * @param id The id of the entity to get, or the index value if secondaryIndex is set.
      * @param secondaryIndex Get the item using a secondary index.
+     * @param conditions The optional conditions to match for the entities.
      * @returns The object if it can be found or undefined.
      */
-    get(id: string, secondaryIndex?: keyof T): Promise<T | undefined>;
+    get(id: string, secondaryIndex?: keyof T, conditions?: {
+        property: keyof T;
+        value: unknown;
+    }[]): Promise<T | undefined>;
     /**
      * Remove the entity.
      * @param id The id of the entity to remove.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns Nothing.
+     */
+    remove(id: string, conditions?: {
+        property: keyof T;
+        value: unknown;
+    }[]): Promise<void>;
+    /**
+     * Remove multiple entities by id.
+     * @param ids The ids of the entities to remove.
      * @returns Nothing.
      */
-    remove(id: string): Promise<void>;
+    removeBatch(ids: string[]): Promise<void>;
+    /**
+     * Remove all entities from the storage.
+     * @returns Nothing.
+     */
+    empty(): Promise<void>;
+    /**
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
+     */
+    count(conditions?: EntityCondition<T>): Promise<number>;
     /**
      * Query all the entities which match the conditions.
      * @param conditions The conditions to match for the entities.

package/dist/types/index.d.ts

@@ -1,6 +1,11 @@
+export * from "./entities/schemaVersion.js";
 export * from "./entityStorageRoutes.js";
 export * from "./entityStorageService.js";
-export * from "./models/IEntityStorageServiceConfig.js";
 export * from "./models/IEntityStorageRoutesExamples.js";
+export * from "./models/IEntityStorageServiceConfig.js";
 export * from "./models/IEntityStorageServiceConstructorOptions.js";
+export * from "./models/ISchemaVersionServiceConfig.js";
+export * from "./models/ISchemaVersionServiceConstructorOptions.js";
 export * from "./restEntryPoints.js";
+export * from "./schema.js";
+export * from "./schemaVersionService.js";

package/dist/types/models/IEntityStorageRoutesExamples.d.ts

@@ -1,5 +1,5 @@
 import type { IRestRouteRequestExample, IRestRouteResponseExample } from "@twin.org/api-models";
-import type { IEntityStorageGetRequest, IEntityStorageGetResponse, IEntityStorageListRequest, IEntityStorageListResponse, IEntityStorageRemoveRequest, IEntityStorageSetRequest } from "@twin.org/entity-storage-models";
+import type { IEntityStorageCountRequest, IEntityStorageCountResponse, IEntityStorageGetRequest, IEntityStorageGetResponse, IEntityStorageListRequest, IEntityStorageListResponse, IEntityStorageRemoveRequest, IEntityStorageSetRequest } from "@twin.org/entity-storage-models";
 /**
  * Examples for the entity storage routes.
  */
@@ -30,4 +30,11 @@ export interface IEntityStorageRoutesExamples {
         requestExamples: IRestRouteRequestExample<IEntityStorageListRequest>[];
         responseExamples: IRestRouteResponseExample<IEntityStorageListResponse>[];
     };
+    /**
+     * Examples for the count route.
+     */
+    count?: {
+        requestExamples: IRestRouteRequestExample<IEntityStorageCountRequest>[];
+        responseExamples: IRestRouteResponseExample<IEntityStorageCountResponse>[];
+    };
 }

package/dist/types/models/ISchemaVersionServiceConfig.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Constructor options config for SchemaVersionService.
+ */
+export interface ISchemaVersionServiceConfig {
+    /**
+     * The batch size for processing schema versions.
+     */
+    batchSize?: number;
+}

package/dist/types/models/ISchemaVersionServiceConstructorOptions.d.ts

@@ -0,0 +1,15 @@
+import type { ISchemaVersionServiceConfig } from "./ISchemaVersionServiceConfig.js";
+/**
+ * Constructor options for SchemaVersionService.
+ */
+export interface ISchemaVersionServiceConstructorOptions {
+    /**
+     * The version storage type.
+     * @default schema-version
+     */
+    schemaVersionStorageType?: string;
+    /**
+     * Optional config.
+     */
+    config?: ISchemaVersionServiceConfig;
+}

package/dist/types/schema.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Initialize the schema for the entity storage models.
+ */
+export declare function initSchema(): void;