@twin.org/entity-storage-connector-gcp-firestore 0.0.1-next.12

package/dist/esm/index.mjs

@@ -0,0 +1,390 @@
+import { Firestore } from '@google-cloud/firestore';
+import { Guards, Is, ObjectHelper, Converter, BaseError, GeneralError } from '@twin.org/core';
+import { EntitySchemaFactory, EntitySchemaHelper, SortDirection, ComparisonOperator } from '@twin.org/entity';
+import { LoggingConnectorFactory } from '@twin.org/logging-models';
+
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Class for performing entity storage operations using Firestore.
+ */
+class FirestoreEntityStorageConnector {
+    /**
+     * Limit the number of entities when finding.
+     * @internal
+     */
+    static _PAGE_SIZE = 40;
+    /**
+     * Runtime name for the class.
+     */
+    CLASS_NAME = "FirestoreEntityStorageConnector";
+    /**
+     * The schema for the entity.
+     * @internal
+     */
+    _entitySchema;
+    /**
+     * The primary key.
+     * @internal
+     */
+    _primaryKey;
+    /**
+     * The configuration for the connector.
+     * @internal
+     */
+    _config;
+    /**
+     * The Firestore client.
+     * @internal
+     */
+    _firestoreClient;
+    /**
+     * The Firestore collection.
+     * @internal
+     */
+    _collection;
+    /**
+     * Create a new instance of FirestoreEntityStorageConnector.
+     * @param options The options for the connector.
+     * @param options.entitySchema The schema for the entity.
+     * @param options.loggingConnectorType The type of logging connector to use, defaults to no logging.
+     * @param options.config The configuration for the connector.
+     */
+    constructor(options) {
+        Guards.object(this.CLASS_NAME, "options", options);
+        Guards.stringValue(this.CLASS_NAME, "options.entitySchema", options.entitySchema);
+        Guards.object(this.CLASS_NAME, "options.config", options.config);
+        Guards.stringValue(this.CLASS_NAME, "options.config.projectId", options.config.projectId);
+        Guards.stringValue(this.CLASS_NAME, "options.config.collectionName", options.config.collectionName);
+        let credentials;
+        if (!Is.empty(options.config.credentials)) {
+            Guards.stringBase64(this.CLASS_NAME, "options.config.credentials", options.config.credentials);
+            credentials = ObjectHelper.fromBytes(Converter.base64ToBytes(options.config.credentials));
+        }
+        this._config = options.config;
+        this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
+        this._primaryKey = EntitySchemaHelper.getPrimaryKey(this._entitySchema);
+        const firestoreOptions = {
+            projectId: this._config.projectId,
+            databaseId: this._config.databaseId,
+            collectionName: this._config.collectionName,
+            maxIdleChannels: this._config.settings?.maxIdleChannels,
+            timeout: this._config.settings?.timeout,
+            credentials
+        };
+        if (Is.stringValue(this._config.endpoint)) {
+            firestoreOptions.host = this._config.endpoint;
+            firestoreOptions.ssl = false;
+        }
+        this._firestoreClient = new Firestore(firestoreOptions);
+        this._collection = this._firestoreClient.collection(this._config.collectionName);
+    }
+    /**
+     * Bootstrap the component by creating and initializing any resources it needs.
+     * @param nodeLoggingConnectorType The node logging connector type, defaults to "node-logging".
+     * @returns True if the bootstrapping process was successful.
+     */
+    async bootstrap(nodeLoggingConnectorType) {
+        const nodeLogging = LoggingConnectorFactory.getIfExists(nodeLoggingConnectorType ?? "node-logging");
+        try {
+            await nodeLogging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                ts: Date.now(),
+                message: "firestoreCreating",
+                data: {
+                    projectId: this._config.projectId,
+                    collectionName: this._config.collectionName
+                }
+            });
+            // Firestore doesn't require explicit collection creation
+            // Perform a small write operation to ensure connectivity
+            const testDoc = this._firestoreClient.collection(this._config.collectionName).doc("test");
+            await testDoc.set({ test: true });
+            await testDoc.delete();
+            await nodeLogging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                ts: Date.now(),
+                message: "firestoreCreated",
+                data: {
+                    projectId: this._config.projectId,
+                    collectionName: this._config.collectionName
+                }
+            });
+            return true;
+        }
+        catch (err) {
+            await nodeLogging?.log({
+                level: "error",
+                source: this.CLASS_NAME,
+                ts: Date.now(),
+                message: "firestoreCreationFailed",
+                error: BaseError.fromError(err),
+                data: {
+                    projectId: this._config.projectId,
+                    collectionName: this._config.collectionName
+                }
+            });
+            return false;
+        }
+    }
+    /**
+     * Get the schema for the entities.
+     * @returns The schema for the entities.
+     */
+    getSchema() {
+        return this._entitySchema;
+    }
+    /**
+     * Get an entity.
+     * @param id The id of the entity to get.
+     * @param secondaryIndex The optional secondary index to use.
+     * @param conditions The optional conditions to apply to the query.
+     * @returns The object if it can be found or undefined.
+     */
+    async get(id, secondaryIndex, conditions) {
+        Guards.stringValue(this.CLASS_NAME, "id", id);
+        try {
+            if (!Is.stringValue(secondaryIndex) && !Is.arrayValue(conditions)) {
+                const docRef = this._collection.doc(id);
+                const doc = await docRef.get();
+                if (doc.exists) {
+                    return doc.data();
+                }
+            }
+            // Use secondaryIndex and/or conditions to construct a query
+            let query = this._collection;
+            if (secondaryIndex) {
+                query = query.where(secondaryIndex, "==", id);
+            }
+            else {
+                // If no secondaryIndex, include primary key in conditions
+                query = query.where(this._primaryKey.property, "==", id);
+            }
+            if (Is.arrayValue(conditions)) {
+                for (const condition of conditions) {
+                    query = query.where(condition.property, "==", condition.value);
+                }
+            }
+            const querySnapshot = await query.limit(1).get();
+            if (!querySnapshot.empty) {
+                return querySnapshot.docs[0].data();
+            }
+        }
+        catch (err) {
+            throw new GeneralError(this.CLASS_NAME, "getEntityFailed", { id }, err);
+        }
+    }
+    /**
+     * Set an entity.
+     * @param entity The entity to set.
+     * @param conditions The optional conditions to apply to the update.
+     * @returns Nothing.
+     */
+    async set(entity, conditions) {
+        Guards.object(this.CLASS_NAME, "entity", entity);
+        try {
+            const id = entity[this._primaryKey.property];
+            const entityCopy = { ...entity };
+            // Handle indexing field
+            if (entityCopy.valueArray && Is.array(entityCopy.valueArray)) {
+                const valueArrayFields = entityCopy.valueArray
+                    .filter((item) => Is.notEmpty(item))
+                    .map(item => `${item.field}:${item.value}`);
+                entityCopy.valueArrayFields = valueArrayFields;
+            }
+            const docRef = this._collection.doc(id);
+            if (!Is.arrayValue(conditions)) {
+                await docRef.set(entityCopy);
+            }
+            else {
+                await this._firestoreClient.runTransaction(async (transaction) => {
+                    const docSnapshot = await transaction.get(docRef);
+                    if (!docSnapshot.exists) {
+                        transaction.set(docRef, entityCopy);
+                    }
+                    else {
+                        const data = docSnapshot.data();
+                        let conditionsMet = true;
+                        for (const condition of conditions) {
+                            if (data[condition.property] !== condition.value) {
+                                conditionsMet = false;
+                                break;
+                            }
+                        }
+                        if (conditionsMet) {
+                            transaction.set(docRef, entityCopy);
+                        }
+                    }
+                });
+            }
+        }
+        catch (err) {
+            throw new GeneralError(this.CLASS_NAME, "setEntityFailed", { entity }, err);
+        }
+    }
+    /**
+     * Remove the entity.
+     * @param id The id of the entity to remove.
+     * @param conditions The optional conditions to apply to the delete.
+     * @returns Nothing.
+     */
+    async remove(id, conditions) {
+        Guards.stringValue(this.CLASS_NAME, "id", id);
+        try {
+            const docRef = this._collection.doc(id);
+            if (!Is.arrayValue(conditions)) {
+                await docRef.delete();
+            }
+            else {
+                await this._firestoreClient.runTransaction(async (transaction) => {
+                    const docSnapshot = await transaction.get(docRef);
+                    if (docSnapshot.exists) {
+                        const data = docSnapshot.data();
+                        let conditionsMet = true;
+                        for (const condition of conditions) {
+                            if (data[condition.property] !== condition.value) {
+                                conditionsMet = false;
+                                break;
+                            }
+                        }
+                        if (conditionsMet) {
+                            transaction.delete(docRef);
+                        }
+                    }
+                });
+            }
+        }
+        catch (err) {
+            throw new GeneralError(this.CLASS_NAME, "removeEntityFailed", { id }, err);
+        }
+    }
+    /**
+     * Find all the entities which match the conditions.
+     * @param conditions The conditions to match for the entities.
+     * @param sortProperties The optional sort order.
+     * @param properties The optional properties to return, defaults to all.
+     * @param cursor The cursor to request the next page of entities.
+     * @param pageSize The suggested number of entities to return in each chunk.
+     * @returns The matching entities and a cursor for the next page.
+     */
+    async query(conditions, sortProperties, properties, cursor, pageSize) {
+        const queryDescription = [];
+        try {
+            let query = this._collection;
+            if (conditions) {
+                query = this.applyConditions(query, conditions);
+                queryDescription.push(`Conditions: ${JSON.stringify(conditions)}`);
+            }
+            if (Is.arrayValue(sortProperties)) {
+                for (const { property, sortDirection } of sortProperties) {
+                    query = query.orderBy(property, sortDirection === SortDirection.Ascending ? "asc" : "desc");
+                }
+                queryDescription.push(`Sort: ${JSON.stringify(sortProperties)}`);
+            }
+            if (Is.stringValue(cursor)) {
+                const cursorDoc = await this._firestoreClient.doc(cursor).get();
+                if (cursorDoc?.exists) {
+                    query = query.startAfter(cursorDoc);
+                }
+                queryDescription.push(`Cursor: ${cursor}`);
+            }
+            const limit = pageSize ?? FirestoreEntityStorageConnector._PAGE_SIZE;
+            query = query.limit(limit);
+            queryDescription.push(`Limit: ${limit}`);
+            if (properties) {
+                query = query.select(...properties);
+                queryDescription.push(`Properties: ${properties.join(", ")}`);
+            }
+            const querySnapshot = await query.get();
+            const entities = querySnapshot.docs.map((doc) => doc.data());
+            let nextCursor;
+            if (entities.length === limit) {
+                nextCursor = querySnapshot.docs[querySnapshot.docs.length - 1].ref.path;
+            }
+            return {
+                entities,
+                cursor: nextCursor
+            };
+        }
+        catch (err) {
+            throw new GeneralError(this.CLASS_NAME, "queryFailed", { queryDescription: queryDescription.join("; ") }, err);
+        }
+    }
+    /**
+     * Delete all entities in the collection.
+     * @returns Nothing.
+     * @internal
+     */
+    async collectionDelete() {
+        const collection = this._collection;
+        const batchSize = 500;
+        const query = collection.limit(batchSize);
+        try {
+            await this.deleteQueryBatch(query, batchSize);
+        }
+        catch (error) {
+            throw new GeneralError(this.CLASS_NAME, "collectionDeleteFailed", { collectionName: this._config.collectionName }, error);
+        }
+    }
+    /**
+     * Apply conditions to a Firestore query.
+     * @param query The initial query.
+     * @param condition The condition to apply.
+     * @returns The updated query.
+     * @internal
+     */
+    applyConditions(query, condition) {
+        if ("conditions" in condition) {
+            // It's a group of conditions
+            for (const c of condition.conditions) {
+                query = this.applyConditions(query, c);
+            }
+            return query;
+        }
+        // It's a single condition
+        const { property, value, comparison } = condition;
+        switch (comparison) {
+            case ComparisonOperator.Equals:
+                return query.where(property, "==", value);
+            case ComparisonOperator.NotEquals:
+                return query.where(property, "!=", value);
+            case ComparisonOperator.GreaterThan:
+                return query.where(property, ">", value);
+            case ComparisonOperator.LessThan:
+                return query.where(property, "<", value);
+            case ComparisonOperator.GreaterThanOrEqual:
+                return query.where(property, ">=", value);
+            case ComparisonOperator.LessThanOrEqual:
+                return query.where(property, "<=", value);
+            case ComparisonOperator.In:
+                return query.where(property, "in", value);
+            case ComparisonOperator.Includes:
+                return query.where(property, "array-contains", value);
+            default:
+                throw new GeneralError(this.CLASS_NAME, "unsupportedComparisonOperator", { comparison });
+        }
+    }
+    /**
+     * Delete all entities in the collection.
+     * @returns Nothing.
+     * @internal
+     */
+    async deleteQueryBatch(query, batchSize) {
+        const snapshot = await query.get();
+        if (snapshot.size === 0) {
+            return;
+        }
+        const batch = this._firestoreClient.batch();
+        for (const doc of snapshot.docs) {
+            batch.delete(doc.ref);
+        }
+        await batch.commit();
+        if (snapshot.size === batchSize) {
+            await this.deleteQueryBatch(query, batchSize);
+        }
+    }
+}
+
+export { FirestoreEntityStorageConnector };

package/dist/types/firestoreEntityStorageConnector.d.ts

@@ -0,0 +1,88 @@
+import { type EntityCondition, type IEntitySchema, SortDirection } from "@twin.org/entity";
+import type { IEntityStorageConnector } from "@twin.org/entity-storage-models";
+import type { IFirestoreEntityStorageConnectorConfig } from "./models/IFirestoreEntityStorageConnectorConfig";
+/**
+ * Class for performing entity storage operations using Firestore.
+ */
+export declare class FirestoreEntityStorageConnector<T = unknown> implements IEntityStorageConnector<T> {
+    /**
+     * Runtime name for the class.
+     */
+    readonly CLASS_NAME: string;
+    /**
+     * Create a new instance of FirestoreEntityStorageConnector.
+     * @param options The options for the connector.
+     * @param options.entitySchema The schema for the entity.
+     * @param options.loggingConnectorType The type of logging connector to use, defaults to no logging.
+     * @param options.config The configuration for the connector.
+     */
+    constructor(options: {
+        entitySchema: string;
+        loggingConnectorType?: string;
+        config: IFirestoreEntityStorageConnectorConfig;
+    });
+    /**
+     * Bootstrap the component by creating and initializing any resources it needs.
+     * @param nodeLoggingConnectorType The node logging connector type, defaults to "node-logging".
+     * @returns True if the bootstrapping process was successful.
+     */
+    bootstrap(nodeLoggingConnectorType?: string): Promise<boolean>;
+    /**
+     * Get the schema for the entities.
+     * @returns The schema for the entities.
+     */
+    getSchema(): IEntitySchema;
+    /**
+     * Get an entity.
+     * @param id The id of the entity to get.
+     * @param secondaryIndex The optional secondary index to use.
+     * @param conditions The optional conditions to apply to the query.
+     * @returns The object if it can be found or undefined.
+     */
+    get(id: string, secondaryIndex?: keyof T, conditions?: {
+        property: keyof T;
+        value: unknown;
+    }[]): Promise<T | undefined>;
+    /**
+     * Set an entity.
+     * @param entity The entity to set.
+     * @param conditions The optional conditions to apply to the update.
+     * @returns Nothing.
+     */
+    set(entity: T, conditions?: {
+        property: keyof T;
+        value: unknown;
+    }[]): Promise<void>;
+    /**
+     * Remove the entity.
+     * @param id The id of the entity to remove.
+     * @param conditions The optional conditions to apply to the delete.
+     * @returns Nothing.
+     */
+    remove(id: string, conditions?: {
+        property: keyof T;
+        value: unknown;
+    }[]): Promise<void>;
+    /**
+     * Find all the entities which match the conditions.
+     * @param conditions The conditions to match for the entities.
+     * @param sortProperties The optional sort order.
+     * @param properties The optional properties to return, defaults to all.
+     * @param cursor The cursor to request the next page of entities.
+     * @param pageSize The suggested number of entities to return in each chunk.
+     * @returns The matching entities and a cursor for the next page.
+     */
+    query(conditions?: EntityCondition<T>, sortProperties?: {
+        property: keyof T;
+        sortDirection: SortDirection;
+    }[], properties?: (keyof T)[], cursor?: string, pageSize?: number): Promise<{
+        /**
+         * The entities, which can be partial if a limited keys list was provided.
+         */
+        entities: Partial<T>[];
+        /**
+         * An optional cursor, when defined can be used to call find to get more entities.
+         */
+        cursor?: string;
+    }>;
+}

package/dist/types/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./firestoreEntityStorageConnector";
+export * from "./models/IFirestoreEntityStorageConnectorConfig";

package/dist/types/models/IEntityWithIndexing.d.ts

@@ -0,0 +1,14 @@
+import type { IValueType } from "./IValueType";
+/**
+ * Interface representing an entity with indexing fields.
+ */
+export interface IEntityWithIndexing {
+    /**
+     * The value array.
+     */
+    valueArray?: IValueType[];
+    /**
+     * The value array fields.
+     */
+    valueArrayFields?: string[];
+}

package/dist/types/models/IFirestoreEntityStorageConnectorConfig.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Configuration for the Firestore Entity Storage Connector.
+ */
+export interface IFirestoreEntityStorageConnectorConfig {
+    /**
+     * The GCP project ID.
+     */
+    projectId: string;
+    /**
+     * The database ID, if omitted default database will be used.
+     */
+    databaseId?: string;
+    /**
+     * The name of the collection for the storage.
+     */
+    collectionName: string;
+    /**
+     * The GCP credentials, a base64 encoded version of the JWTInput data type.
+     */
+    credentials?: string;
+    /**
+     * It's usually only used with an emulator (e.g., "localhost:8080").
+     */
+    endpoint?: string;
+    /**
+     * Optional settings for Firestore client initialization.
+     */
+    settings?: {
+        /**
+         * The maximum number of idle channels to keep open.
+         */
+        maxIdleChannels?: number;
+        /**
+         * The custom timeout for requests (in milliseconds).
+         */
+        timeout?: number;
+    };
+}

package/dist/types/models/IValueType.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Interface representing a value type with a field and a value.
+ */
+export interface IValueType {
+    /**
+     * The field name.
+     */
+    field: string;
+    /**
+     * The value.
+     */
+    value: string;
+}

package/docs/changelog.md

@@ -0,0 +1,5 @@
+# @twin.org/entity-storage-connector-gcp-firestore - Changelog
+
+## v0.0.1-next.12
+
+- Initial Release

package/docs/examples.md

@@ -0,0 +1 @@
+# @twin.org/entity-storage-connector-gcp-firestore - Examples