@twin.org/entity-storage-models 0.0.3-next.2 → 0.0.3-next.20

package/dist/types/helpers/migrationHelper.d.ts

@@ -0,0 +1,61 @@
+import { type IContextIds } from "@twin.org/context";
+import { type IEntitySchemaDiff } from "@twin.org/entity";
+import type { IEntityStorageConnector } from "../models/IEntityStorageConnector.js";
+import type { IEntityStorageMigrationConnector } from "../models/IEntityStorageMigrationConnector.js";
+import type { IMigrationOptions } from "../models/IMigrationOptions.js";
+/**
+ * Helper class for performing schema migrations between two connectors.
+ */
+export declare class MigrationHelper {
+    /**
+     * Runtime name for the class.
+     */
+    static readonly CLASS_NAME: string;
+    /**
+     * Performs a migration between two connectors, using the provided options and schema diff to control the migration behaviour.
+     * @param sourceConnector The connector to migrate from to allow the migration helper to create the new connector and finalize the migration.
+     * @param targetEntitySchemaName The name of the new entity schema.
+     * @param renames An optional list of property renames to apply during migration.
+     * @param options Options controlling migration behaviour.
+     * @param loggingComponentType An optional logging component type to use for bootstrapping and starting connectors if necessary.
+     * @returns The connector for the new schema and the number of entities successfully migrated, the sourceConnector will no longer be usable, finalConnector will be undefined if no migration was necessary.
+     */
+    static migrate<T, U>(sourceConnector: IEntityStorageMigrationConnector<T>, targetEntitySchemaName: string, renames?: {
+        from: string;
+        to: string;
+    }[], options?: IMigrationOptions<T, U>, loggingComponentType?: string): Promise<{
+        finalConnector?: IEntityStorageConnector<U>;
+        migrated: number;
+    }>;
+    /**
+     * Generic per-partition migration loop.
+     * @param source Connector to read from (current schema, already bootstrapped).
+     * @param target Connector to write to (new schema, already bootstrapped).
+     * @param partitionContextIds The context ids to use for the migration, used for partitioning and can be used in the transform function when `options.transformEntityProperty` is provided.
+     * @param schemaDiff The schema diff.
+     * @param options Optional migration controls (batchSize, transformEntity, onProgress).
+     * @returns The number of entities successfully migrated.
+     */
+    static migrateEntities<T = unknown, U = T>(source: IEntityStorageMigrationConnector<T>, target: IEntityStorageConnector<U>, partitionContextIds: IContextIds[], schemaDiff: IEntitySchemaDiff<T, U>, options?: IMigrationOptions<T, U>): Promise<number>;
+    /**
+     * Generic per-partition migration loop.
+     * @param source Connector to read from (current schema, already bootstrapped).
+     * @param target Connector to write to (new schema, already bootstrapped).
+     * @param partitionTotal The total number of partitions to migrate, used for progress reporting.
+     * @param partitionIndex The index of the current partition being migrated, used for progress reporting.
+     * @param schemaDiff Schema diff used to add nullable defaults and drop removed fields when `options.transformEntity` is not provided.
+     * @param options Optional migration controls (batchSize, transformEntity, onProgress).
+     * @returns The number of entities successfully migrated.
+     */
+    static migratePartition<T = unknown, U = unknown>(source: IEntityStorageConnector<T>, target: IEntityStorageConnector<U>, partitionTotal: number, partitionIndex: number, schemaDiff: IEntitySchemaDiff<T, U>, options?: IMigrationOptions<T, U>): Promise<number>;
+    /**
+     * Applies the entity transformation for migration, using the provided options and schema diff.
+     * @param entity The entity to transform.
+     * @param schemaDiff The schema diff between the old and new schemas.
+     * @param options The migration options.
+     * @returns The transformed entity ready to be written to the new schema.
+     * @throws GeneralError if a transformation is required for an object or array property but no `options.transformEntityProperty` function is provided.
+     * @throws GeneralError if coercion of a modified property results in undefined for a non-optional target property.
+     */
+    static applyEntityTransform<T = unknown, U = unknown>(entity: Partial<T>, schemaDiff: IEntitySchemaDiff<T, U>, options?: IMigrationOptions<T, U>): U;
+}

package/dist/types/index.d.ts

@@ -1,9 +1,18 @@
 export * from "./factories/entityStorageConnectorFactory.js";
+export * from "./helpers/entityStorageHelper.js";
+export * from "./helpers/migrationHelper.js";
+export * from "./models/api/IEntityStorageCountRequest.js";
+export * from "./models/api/IEntityStorageCountResponse.js";
+export * from "./models/api/IEntityStorageEmptyRequest.js";
 export * from "./models/api/IEntityStorageGetRequest.js";
 export * from "./models/api/IEntityStorageGetResponse.js";
 export * from "./models/api/IEntityStorageListRequest.js";
 export * from "./models/api/IEntityStorageListResponse.js";
+export * from "./models/api/IEntityStorageRemoveBatchRequest.js";
 export * from "./models/api/IEntityStorageRemoveRequest.js";
+export * from "./models/api/IEntityStorageSetBatchRequest.js";
 export * from "./models/api/IEntityStorageSetRequest.js";
 export * from "./models/IEntityStorageComponent.js";
 export * from "./models/IEntityStorageConnector.js";
+export * from "./models/IEntityStorageMigrationConnector.js";
+export * from "./models/IMigrationOptions.js";

package/dist/types/models/IEntityStorageComponent.d.ts

@@ -7,22 +7,46 @@ export interface IEntityStorageComponent<T = unknown> extends IComponent {
     /**
      * Set an entity.
      * @param entity The entity to set.
+     * @param conditions The optional conditions to match for the entities.
      * @returns The id of the entity.
      */
-    set(entity: T): Promise<void>;
+    set(entity: T, conditions?: {
+        property: keyof T;
+        value: unknown;
+    }[]): Promise<void>;
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    setBatch(entities: T[]): Promise<void>;
     /**
      * Get an entity.
      * @param id The id of the entity to get, or the index value if secondaryIndex is set.
      * @param secondaryIndex Get the item using a secondary index.
+     * @param conditions The optional conditions to match for the entities.
      * @returns The object if it can be found or undefined.
      */
-    get(id: string, secondaryIndex?: keyof T): Promise<T | undefined>;
+    get(id: string, secondaryIndex?: keyof T, conditions?: {
+        property: keyof T;
+        value: unknown;
+    }[]): Promise<T | undefined>;
     /**
      * Remove the entity.
      * @param id The id of the entity to remove.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns Nothing.
+     */
+    remove(id: string, conditions?: {
+        property: keyof T;
+        value: unknown;
+    }[]): Promise<void>;
+    /**
+     * Remove multiple entities by id.
+     * @param ids The ids of the entities to remove.
      * @returns Nothing.
      */
-    remove(id: string): Promise<void>;
+    removeBatch(ids: string[]): Promise<void>;
     /**
      * Query all the entities which match the conditions.
      * @param conditions The conditions to match for the entities.
@@ -44,4 +68,15 @@ export interface IEntityStorageComponent<T = unknown> extends IComponent {
          */
         cursor?: string;
     }>;
+    /**
+     * Remove all entities from the storage.
+     * @returns Nothing.
+     */
+    empty(): Promise<void>;
+    /**
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
+     */
+    count(conditions?: EntityCondition<T>): Promise<number>;
 }

package/dist/types/models/IEntityStorageConnector.d.ts

@@ -19,6 +19,12 @@ export interface IEntityStorageConnector<T = unknown> extends IComponent {
         property: keyof T;
         value: unknown;
     }[]): Promise<void>;
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    setBatch(entities: T[]): Promise<void>;
     /**
      * Get an entity.
      * @param id The id of the entity to get, or the index value if secondaryIndex is set.
@@ -40,6 +46,12 @@ export interface IEntityStorageConnector<T = unknown> extends IComponent {
         property: keyof T;
         value: unknown;
     }[]): Promise<void>;
+    /**
+     * Remove multiple entities by id.
+     * @param ids The ids of the entities to remove.
+     * @returns Nothing.
+     */
+    removeBatch(ids: string[]): Promise<void>;
     /**
      * Query all the entities which match the conditions.
      * @param conditions The conditions to match for the entities.
@@ -63,4 +75,15 @@ export interface IEntityStorageConnector<T = unknown> extends IComponent {
          */
         cursor?: string;
     }>;
+    /**
+     * Remove all entities from the storage.
+     * @returns Nothing.
+     */
+    empty(): Promise<void>;
+    /**
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
+     */
+    count(conditions?: EntityCondition<T>): Promise<number>;
 }

package/dist/types/models/IEntityStorageMigrationConnector.d.ts

@@ -0,0 +1,35 @@
+import type { IContextIds } from "@twin.org/context";
+import type { IEntityStorageConnector } from "./IEntityStorageConnector.js";
+import type { IMigrationOptions } from "./IMigrationOptions.js";
+/**
+ * Interface describing an entity storage migration connector.
+ */
+export interface IEntityStorageMigrationConnector<T = unknown> extends IEntityStorageConnector<T> {
+    /**
+     * Get a unique list of all the context ids from the storage.
+     * @returns The list of unique context ids.
+     */
+    getPartitionContextIds(): Promise<IContextIds[]>;
+    /**
+     * Create the target connector for performing the migration it will use a temporary storage location.
+     * @param newEntitySchema The name of the new entity schema to create the connector for.
+     * @returns Connector for performing the migration.
+     */
+    createTargetConnector<U>(newEntitySchema: string): Promise<IEntityStorageConnector<U>>;
+    /**
+     * Finalize the migration by tearing down the old connector and replacing it with the target connector.
+     * @param targetConnector The target connector to finalize the migration with.
+     * @param options The options to control how the migration is finalized.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is finalized and returns the final connector.
+     */
+    finalizeMigration<U>(targetConnector: IEntityStorageConnector<U>, options?: IMigrationOptions<T, U>, loggingComponentType?: string): Promise<IEntityStorageConnector<U>>;
+    /**
+     * Cleanup the migration if a migration fails or needs to be aborted.
+     * @param targetConnector The target connector to cleanup the migration with.
+     * @param options The options to control how the migration is cleaned up.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is cleaned up.
+     */
+    cleanupMigration<U>(targetConnector: IEntityStorageConnector<U> | undefined, options?: IMigrationOptions<T, U>, loggingComponentType?: string): Promise<void>;
+}

package/dist/types/models/IMigrationOptions.d.ts

@@ -0,0 +1,32 @@
+import type { IEntitySchemaProperty } from "@twin.org/entity";
+/**
+ * Options controlling how a schema migration is executed.
+ */
+export interface IMigrationOptions<T, U> {
+    /**
+     * Number of entities to read and write per batch.
+     * @default 100
+     */
+    batchSize?: number;
+    /**
+     * Optional transformation for properties, usually only called for object and array types.
+     * @param schema1Property The property schema in the old schema.
+     * @param schemaProperty2 The property schema in the new schema.
+     * @param value The value of the property in the old schema.
+     * @returns The transformed value to match the new schema.
+     */
+    transformEntityProperty?: (schema1Property: IEntitySchemaProperty<T>, schemaProperty2: IEntitySchemaProperty<U>, value: unknown) => unknown;
+    /**
+     * Called for each partition for progress tracking.
+     * @param rowTotal The total number of rows to migrate.
+     * @param rowIndex The number of rows migrated so far.
+     */
+    onPartitionProgress?: (rowTotal: number, rowIndex: number) => Promise<void>;
+    /**
+     * Called for overall progress tracking.
+     * @param stepKey The key representing the current step in the migration.
+     * @param itemTotal The total number of items in this progress.
+     * @param itemIndex The number of items processed so far.
+     */
+    onStepProgress?: (stepKey: string, itemTotal: number, itemIndex: number) => Promise<void>;
+}

package/dist/types/models/api/IEntityStorageCountRequest.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Count the entries in entity storage.
+ */
+export interface IEntityStorageCountRequest {
+    /**
+     * The query parameters.
+     */
+    query?: {
+        /**
+         * The optional conditions to filter the count, JSON encoded EntityCondition.
+         */
+        conditions?: string;
+    };
+}

package/dist/types/models/api/IEntityStorageCountResponse.d.ts

@@ -0,0 +1,14 @@
+/**
+ * The response for counting entries in entity storage.
+ */
+export interface IEntityStorageCountResponse {
+    /**
+     * The body of the response.
+     */
+    body: {
+        /**
+         * The total count of entities.
+         */
+        count: number;
+    };
+}

package/dist/types/models/api/IEntityStorageEmptyRequest.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Remove all entries from entity storage.
+ */
+export interface IEntityStorageEmptyRequest {
+}

package/dist/types/models/api/IEntityStorageGetRequest.d.ts

@@ -19,5 +19,9 @@ export interface IEntityStorageGetRequest {
          * The secondary index to query with the id.
          */
         secondaryIndex?: string;
+        /**
+         * The optional conditions to match for the entity, JSON encoded array of property/value pairs.
+         */
+        conditions?: string;
     };
 }

package/dist/types/models/api/IEntityStorageRemoveBatchRequest.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Remove multiple entries from entity storage by id.
+ */
+export interface IEntityStorageRemoveBatchRequest {
+    /**
+     * The ids of the entities to remove.
+     */
+    body: string[];
+}

package/dist/types/models/api/IEntityStorageRemoveRequest.d.ts

@@ -11,4 +11,13 @@ export interface IEntityStorageRemoveRequest {
          */
         id: string;
     };
+    /**
+     * The query parameters.
+     */
+    query?: {
+        /**
+         * The optional conditions to match for the entity, JSON encoded array of property/value pairs.
+         */
+        conditions?: string;
+    };
 }

package/dist/types/models/api/IEntityStorageSetBatchRequest.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Set multiple entries in entity storage.
+ */
+export interface IEntityStorageSetBatchRequest {
+    /**
+     * The entities to set.
+     */
+    body: unknown[];
+}

package/dist/types/models/api/IEntityStorageSetRequest.d.ts

@@ -6,4 +6,13 @@ export interface IEntityStorageSetRequest {
      * The data to be used in the entity.
      */
     body: unknown;
+    /**
+     * The query parameters.
+     */
+    query?: {
+        /**
+         * The optional conditions to match for the entity, JSON encoded array of property/value pairs.
+         */
+        conditions?: string;
+    };
 }