@twin.org/entity-storage-connector-scylladb 0.0.3-next.3 → 0.0.3-next.30

package/README.md

@@ -1,6 +1,6 @@
 # Entity Storage Connector ScyllaDB
 
-Entity Storage connector implementation using ScyllaDB.
+This package provides a ScyllaDB backend for distributed workloads that need low latency and high throughput. It is designed to work with the wider storage ecosystem so applications can keep behaviour consistent across connectors and environments.
 
 ## Installation
 
@@ -8,18 +8,12 @@ Entity Storage connector implementation using ScyllaDB.
 npm install @twin.org/entity-storage-connector-scylladb
 ```
 
-## Testing
+## Docker
 
-The tests developed are functional tests and need an instance of ScyllaDB up and running. To run ScyllaDB locally:
+To perform testing of this component it may be necessary to launch a local instance to communicate with.
 
 ```shell
-docker run -p 9500:9042 --name twin-entity-storage-scylladb --hostname scylla -d scylladb/scylla:5.4.9 --smp 1
-```
-
-Afterwards you can run the tests as follows:
-
-```shell
-npm run test
+docker run -d --name twin-entity-storage-scylladb -p 9042:9042 scylladb/scylla:5.4.9
 ```
 
 ## Examples

package/dist/es/abstractScyllaDBConnector.js

@@ -1,8 +1,9 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
-import { Coerce, ComponentFactory, GeneralError, Guards, Is, ObjectHelper, StringHelper } from "@twin.org/core";
+import { Coerce, ComponentFactory, GeneralError, Guards, Is, Validation } from "@twin.org/core";
 import { ComparisonOperator, EntitySchemaFactory, EntitySchemaHelper, LogicalOperator, SortDirection } from "@twin.org/entity";
+import { EntityStorageHelper } from "@twin.org/entity-storage-models";
 import { types as CassandraTypes, Client } from "cassandra-driver";
 /**
  * Store entities using ScyllaDB.
@@ -26,7 +27,7 @@ export class AbstractScyllaDBConnector {
      * Limit the number of entities when finding.
      * @internal
      */
-    static _DEFAULT_LIMIT = 40;
+    static DEFAULT_LIMIT = 40;
     /**
      * The name of the database table.
      * @internal
@@ -57,6 +58,13 @@ export class AbstractScyllaDBConnector {
      * @internal
      */
     _primaryKey;
+    /**
+     * Cached persistent client (keyspace-scoped). Reused across all operations on this
+     * connector instance so the expensive cassandra-driver `connect()` only runs once.
+     * Closed by `closePersistentClient()` which callers (e.g. `teardown()`) must invoke.
+     * @internal
+     */
+    _persistentClient;
     /**
      * Create a new instance of AbstractScyllaDBConnector.
      * @param options The options for the connector.
@@ -72,12 +80,12 @@ export class AbstractScyllaDBConnector {
         Guards.arrayValue(AbstractScyllaDBConnector.CLASS_NAME, "options.config.hosts", options.config.hosts);
         Guards.stringValue(AbstractScyllaDBConnector.CLASS_NAME, "options.config.localDataCenter", options.config.localDataCenter);
         Guards.stringValue(AbstractScyllaDBConnector.CLASS_NAME, "options.config.keyspace", options.config.keyspace);
-        this._logging = ComponentFactory.getIfExists(options.loggingComponentType ?? "logging");
+        this._logging = ComponentFactory.getIfExists(options.loggingComponentType);
         this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
         this._partitionContextIds = options.partitionContextIds;
         this._primaryKey = EntitySchemaHelper.getPrimaryKey(this._entitySchema);
         this._config = options.config;
-        this._fullTableName = StringHelper.camelCase(Is.stringValue(options.config.tableName) ? options.config.tableName : options.entitySchema);
+        this._fullTableName = options.config.tableName;
     }
     /**
      * Returns the class name of the component.
@@ -117,7 +125,7 @@ export class AbstractScyllaDBConnector {
                 value: id
             });
             const { sqlCondition, conditionValues } = this.buildConditions(conditions);
-            let sql = `SELECT * FROM "${this._fullTableName}" WHERE ${sqlCondition}`;
+            let sql = `SELECT * FROM "${this.safeTableName(this._fullTableName)}" WHERE ${sqlCondition}`;
             if (secondaryIndex) {
                 sql += " ALLOW FILTERING";
             }
@@ -157,100 +165,51 @@ export class AbstractScyllaDBConnector {
         let connection;
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        EntityStorageHelper.validateSortProperties(this._entitySchema, sortProperties);
+        EntityStorageHelper.validateProperties(this._entitySchema, properties);
+        if (!Is.empty(limit)) {
+            const validationFailures = [];
+            Validation.integer("limit", limit, validationFailures, undefined, { minValue: 1 });
+            Validation.asValidationError(AbstractScyllaDBConnector.CLASS_NAME, "query", validationFailures);
+        }
+        // CQL ORDER BY is only valid on clustering columns (isPrimary). Secondary-index
+        // properties cannot be used; throw before reaching the try-catch so the error
+        // surfaces directly to the caller without being wrapped as findFailed.
+        if (Is.arrayValue(sortProperties)) {
+            for (const sortProperty of sortProperties) {
+                const propertySchema = this._entitySchema.properties?.find(p => p.property === sortProperty.property);
+                if (!propertySchema?.isPrimary) {
+                    throw new GeneralError(AbstractScyllaDBConnector.CLASS_NAME, "sortOnlyPrimaryKey", {
+                        property: sortProperty.property
+                    });
+                }
+            }
+        }
+        // Validates and throws for unsupported conditions before entering the try-catch
+        // so that comparisonNotSupported errors surface directly to the caller.
+        const { whereClause, params, noResults } = this.buildCqlConditions(conditions, partitionKey);
+        if (noResults) {
+            return { entities: [], cursor: undefined };
+        }
         try {
-            let returnSize = limit ?? AbstractScyllaDBConnector._DEFAULT_LIMIT;
-            let sql = `SELECT * FROM "${this._fullTableName}"`;
+            const returnSize = limit ?? AbstractScyllaDBConnector.DEFAULT_LIMIT;
+            let sql = `SELECT * FROM "${this.safeTableName(this._fullTableName)}"`;
             if (Is.array(properties)) {
                 const fields = [];
                 for (const property of properties) {
                     fields.push(property.toString());
                 }
-                const selectFields = fields.join(",");
-                sql = sql.replace("*", selectFields);
-            }
-            const conds = [];
-            let conditionQuery = "";
-            // The params to be used to execute the query
-            const params = [];
-            let finalConditionQuery = `"${AbstractScyllaDBConnector.PARTITION_KEY}" = ?`;
-            params.push(partitionKey ?? AbstractScyllaDBConnector.PARTITION_KEY_VALUE);
-            let theConditions = [];
-            if (!Is.undefined(conditions)) {
-                if ("conditions" in conditions) {
-                    theConditions = conditions.conditions;
-                }
-                else {
-                    theConditions.push(conditions);
-                }
-            }
-            // TODO: This code needs refactoring to support conditions for sub properties.
-            for (const cond of theConditions) {
-                const condition = cond;
-                const descriptor = this._entitySchema.properties?.find(p => p.property === condition.property);
-                if (condition.comparison === ComparisonOperator.Includes ||
-                    condition.comparison === ComparisonOperator.NotIncludes) {
-                    const propValue = `'%${condition.value?.toString()}%'`;
-                    if (condition.comparison === ComparisonOperator.Includes) {
-                        conds.push(`"${condition.property}" LIKE ${propValue}`);
-                    }
-                    else if (condition.comparison === ComparisonOperator.NotIncludes) {
-                        conds.push(`"${condition.property}" NOT LIKE ${propValue}`);
-                    }
-                }
-                else if (condition.comparison === ComparisonOperator.In) {
-                    let value = [];
-                    if (!Is.arrayValue(condition.value)) {
-                        value.push(this.propertyToDbValue(condition.value, descriptor));
-                    }
-                    else {
-                        value = condition.value.map(v => this.propertyToDbValue(v, descriptor));
-                    }
-                    params.push(value);
-                    conds.push(`"${condition.property}" IN ?`);
-                }
-                else {
-                    const propValue = condition.value;
-                    params.push(propValue);
-                    if (condition.comparison === ComparisonOperator.Equals) {
-                        conds.push(`"${condition.property}" = ?`);
-                    }
-                    else if (condition.comparison === ComparisonOperator.NotEquals) {
-                        conds.push(`"${condition.property}" <> ?`);
-                    }
-                    else if (condition.comparison === ComparisonOperator.GreaterThan) {
-                        conds.push(`"${condition.property}" > ?`);
-                    }
-                    else if (condition.comparison === ComparisonOperator.LessThan) {
-                        conds.push(`"${condition.property}" < ?`);
-                    }
-                    else if (condition.comparison === ComparisonOperator.GreaterThanOrEqual) {
-                        conds.push(`"${condition.property}" >= ?`);
-                    }
-                    else if (condition.comparison === ComparisonOperator.LessThanOrEqual) {
-                        conds.push(`"${condition.property}" <= ?`);
-                    }
-                }
-                const operator = conditions.logicalOperator ?? LogicalOperator.And;
-                conditionQuery = `${conds.join(` ${operator} `)}`;
+                sql = sql.replace("*", fields.join(","));
             }
-            if (conditionQuery.length > 0) {
-                finalConditionQuery += ` AND (${conditionQuery})`;
-            }
-            sql += ` WHERE ${finalConditionQuery}`;
-            connection = await this.openConnection();
-            // TODO: Only supported one sort property at the moment. This code would need to be revised in a follow-up
+            sql += ` WHERE ${whereClause}`;
             if (Is.array(sortProperties) && sortProperties.length >= 1) {
-                const sortKey = sortProperties[0].property ?? this._primaryKey.property;
-                const sortDir = sortProperties[0].sortDirection ??
-                    this._entitySchema.properties?.find(e => e.property === sortKey)?.sortDirection;
-                let sqlSortDir = "asc";
-                if (sortDir === SortDirection.Descending) {
-                    sqlSortDir = "desc";
-                }
-                sql += ` ORDER BY "${String(sortKey)}" ${sqlSortDir.toUpperCase()}`;
-                // Disabling paging in order by situations
-                returnSize = 0;
+                const orderClauses = sortProperties.map(sp => {
+                    const dir = sp.sortDirection === SortDirection.Descending ? "DESC" : "ASC";
+                    return `"${String(sp.property)}" ${dir}`;
+                });
+                sql += ` ORDER BY ${orderClauses.join(", ")}`;
             }
+            connection = await this.openConnection();
             sql += " ALLOW FILTERING";
             await this._logging?.log({
                 level: "info",
@@ -264,13 +223,49 @@ export class AbstractScyllaDBConnector {
             for (const row of result.rows) {
                 entities.push(this.convertRowToObject(this._entitySchema.properties, row));
             }
+            // ScyllaDB may return a pageState even when the current page is the last one
+            // (when rows.length == fetchSize). Peek at the next page to verify there are
+            // actually more rows before surfacing the cursor to the caller.
+            let nextCursor;
+            if (returnSize > 0 && result.rows.length >= returnSize && Is.stringValue(result.pageState)) {
+                const peek = await this.queryDB(connection, sql, params, result.pageState, 1);
+                if (peek.rows.length > 0) {
+                    nextCursor = result.pageState;
+                }
+            }
             return {
                 entities,
-                cursor: Is.stringValue(result.pageState) ? result.pageState : undefined
+                cursor: nextCursor
             };
         }
         catch (error) {
-            throw new GeneralError(AbstractScyllaDBConnector.CLASS_NAME, "findFailed", { table: this._fullTableName }, error);
+            throw new GeneralError(AbstractScyllaDBConnector.CLASS_NAME, "findFailed", { table: this.safeTableName(this._fullTableName) }, error);
+        }
+        finally {
+            await this.closeConnection(connection);
+        }
+    }
+    /**
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
+     */
+    async count(conditions) {
+        let connection;
+        try {
+            const contextIds = await ContextIdStore.getContextIds();
+            const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+            const { whereClause, params, noResults } = this.buildCqlConditions(conditions, partitionKey);
+            if (noResults) {
+                return 0;
+            }
+            const sql = `SELECT COUNT(*) FROM "${this.safeTableName(this._fullTableName)}" WHERE ${whereClause} ALLOW FILTERING`;
+            connection = await this.openConnection();
+            const result = await this.queryDB(connection, sql, params);
+            return Number(result.rows[0]?.get("count") ?? 0);
+        }
+        catch (err) {
+            throw new GeneralError(AbstractScyllaDBConnector.CLASS_NAME, "countFailed", undefined, err);
         }
         finally {
             await this.closeConnection(connection);
@@ -278,12 +273,16 @@ export class AbstractScyllaDBConnector {
     }
     /**
      * Open a new database connection.
-     * @param config The config for the connection.
      * @param skipKeySpace Don't include the keyspace in the connection.
      * @returns The new connection.
      * @internal
      */
     async openConnection(skipKeySpace = false) {
+        // Reuse the cached keyspace-scoped client when available (avoids repeated
+        // cassandra-driver cluster-discovery on every operation).
+        if (!skipKeySpace && this._persistentClient !== undefined) {
+            return this._persistentClient;
+        }
         const client = new Client({
             contactPoints: this._config.hosts,
             localDataCenter: this._config.localDataCenter,
@@ -293,25 +292,48 @@ export class AbstractScyllaDBConnector {
             }
         });
         await client.connect();
+        if (!skipKeySpace) {
+            this._persistentClient = client;
+        }
         return client;
     }
     /**
      * Close database connection.
+     * When `connection` is the cached persistent client it is kept alive so it
+     * can be reused by future operations; call `closePersistentClient()` to
+     * explicitly shut it down (e.g. from `teardown()`).
      * @param connection The connection to close.
+     * @returns Nothing.
      * @internal
      */
     async closeConnection(connection) {
         if (!connection) {
             return;
         }
+        if (connection === this._persistentClient) {
+            // Keep the persistent client alive for reuse.
+            return;
+        }
         return connection.shutdown();
     }
+    /**
+     * Shut down and clear the persistent client. Call this from `teardown()`
+     * implementations to release the underlying TCP connection.
+     * @internal
+     */
+    async closePersistentClient() {
+        if (this._persistentClient !== undefined) {
+            await this._persistentClient.shutdown();
+            this._persistentClient = undefined;
+        }
+    }
     /**
      * Query the database.
      * @param connection The connection to query.
      * @param sql The sql statement to execute.
      * @param params The params to use when executing the query.
-     * @param state The state to use when it comes to pagination.
+     * @param pageState The page state to use when it comes to pagination.
+     * @param limit The maximum number of rows to return.
      * @returns The rows.
      * @internal
      */
@@ -321,7 +343,7 @@ export class AbstractScyllaDBConnector {
             connection.eachRow(sql, params, {
                 prepare: true,
                 autoPage: false,
-                fetchSize: limit ?? AbstractScyllaDBConnector._DEFAULT_LIMIT,
+                fetchSize: limit ?? AbstractScyllaDBConnector.DEFAULT_LIMIT,
                 pageState
             }, (n, row) => {
                 rows.push(row);
@@ -339,6 +361,8 @@ export class AbstractScyllaDBConnector {
      * Execute on the database.
      * @param connection The connection to execute.
      * @param sql The sql statement to execute.
+     * @param params The optional params to use when executing the statement.
+     * @returns The result set.
      * @internal
      */
     async execute(connection, sql, params) {
@@ -348,6 +372,7 @@ export class AbstractScyllaDBConnector {
      * Create keyspace if it doesn't exist.
      * @param connection The connection to perform the query with.
      * @param keyspaceName The name of the keyspace to create.
+     * @returns The result set.
      * @internal
      */
     async createKeyspace(connection, keyspaceName) {
@@ -392,6 +417,7 @@ export class AbstractScyllaDBConnector {
      * @param value The value to convert to original form.
      * @param fieldDescriptor The descriptor for the field.
      * @returns The value as a property for the object.
+     * @throws GeneralError if parsing JSON fails.
      * @internal
      */
     dbValueToProperty(value, fieldDescriptor) {
@@ -436,9 +462,9 @@ export class AbstractScyllaDBConnector {
         return value;
     }
     /**
-     * Format a value for the DB. As the driver takes care of conversion from Javascript
+     * Format a value for the DB.
      * @param value The value to format.
-     * @param fieldDescriptor The descriptor for the field
+     * @param fieldDescriptor The descriptor for the field.
      * @returns The value after conversion.
      * @internal
      */
@@ -475,7 +501,7 @@ export class AbstractScyllaDBConnector {
                 obj[field.property] = this.dbValueToProperty(value, field);
             }
         }
-        return ObjectHelper.removeEmptyProperties(obj, { removeNull: true });
+        return EntityStorageHelper.unPrepareEntity(obj, [AbstractScyllaDBConnector.PARTITION_KEY]);
     }
     /**
      * Wrap a string for DB format.
@@ -541,5 +567,131 @@ export class AbstractScyllaDBConnector {
         }
         return { sqlCondition: sqlConditions.join(" AND "), conditionValues };
     }
+    /**
+     * Get a safe table name by replacing any non-alphanumeric characters.
+     * @param name The name to sanitize.
+     * @returns The safe table name.
+     */
+    safeTableName(name) {
+        return name.replace(/[^\dA-Za-z]/g, "");
+    }
+    /**
+     * Parse, validate, and build a CQL WHERE clause from an EntityCondition tree.
+     * The partition key equality is always the first clause; user conditions follow.
+     * @param conditions The optional conditions to match for the entities.
+     * @param partitionKey The partition key value to filter by.
+     * @returns The complete WHERE clause (without the WHERE keyword) and bound params.
+     * @throws GeneralError if OR conditions, dot-notation paths, null comparisons, NotEquals, or NotIncludes operators are used.
+     * @internal
+     */
+    buildCqlConditions(conditions, partitionKey) {
+        let conditionsList = [];
+        if (conditions !== undefined) {
+            if ("conditions" in conditions) {
+                if (conditions.logicalOperator === LogicalOperator.Or) {
+                    throw new GeneralError(AbstractScyllaDBConnector.CLASS_NAME, "orConditionNotSupported");
+                }
+                conditionsList = conditions.conditions;
+            }
+            else {
+                conditionsList = [conditions];
+            }
+        }
+        for (const cond of conditionsList) {
+            const comparator = cond;
+            if (String(comparator.property).includes(".")) {
+                throw new GeneralError(AbstractScyllaDBConnector.CLASS_NAME, "comparisonNotSupported", {
+                    property: comparator.property,
+                    reason: "dot-notation nested property paths are not supported in CQL"
+                });
+            }
+            if ((comparator.comparison === ComparisonOperator.Equals ||
+                comparator.comparison === ComparisonOperator.NotEquals) &&
+                (comparator.value === null || comparator.value === undefined)) {
+                throw new GeneralError(AbstractScyllaDBConnector.CLASS_NAME, "comparisonNotSupported", {
+                    property: comparator.property,
+                    reason: "null/undefined comparisons are not supported in CQL WHERE clauses"
+                });
+            }
+            if (comparator.comparison === ComparisonOperator.NotEquals) {
+                throw new GeneralError(AbstractScyllaDBConnector.CLASS_NAME, "notEqualsNotSupported", {
+                    property: comparator.property
+                });
+            }
+            if (comparator.comparison === ComparisonOperator.NotIncludes) {
+                throw new GeneralError(AbstractScyllaDBConnector.CLASS_NAME, "notIncludesNotSupported", {
+                    property: comparator.property
+                });
+            }
+        }
+        const conds = [];
+        const params = [partitionKey ?? AbstractScyllaDBConnector.PARTITION_KEY_VALUE];
+        for (const cond of conditionsList) {
+            const condition = cond;
+            const descriptor = this._entitySchema.properties?.find(p => p.property === condition.property);
+            if (condition.comparison === ComparisonOperator.Includes ||
+                condition.comparison === ComparisonOperator.NotIncludes) {
+                const serialized = this.propertyToDbValue(condition.value, descriptor);
+                const propValue = `'%${Is.stringValue(serialized) ? serialized : ""}%'`;
+                if (condition.comparison === ComparisonOperator.Includes) {
+                    conds.push(`"${condition.property}" LIKE ${propValue}`);
+                }
+                else if (condition.comparison === ComparisonOperator.NotIncludes) {
+                    conds.push(`"${condition.property}" NOT LIKE ${propValue}`);
+                }
+            }
+            else if (condition.comparison === ComparisonOperator.In) {
+                // Guard must come first: Is.arrayValue([]) returns false for an empty array,
+                // so an empty value would be wrapped as a single element below and bypass
+                // the length check. Check Is.array (true for any array) before branching (#141).
+                if (Is.array(condition.value) && condition.value.length === 0) {
+                    return {
+                        whereClause: "",
+                        params: [],
+                        noResults: true
+                    };
+                }
+                let value = [];
+                if (!Is.arrayValue(condition.value)) {
+                    value.push(this.propertyToDbValue(condition.value, descriptor));
+                }
+                else {
+                    value = condition.value.map(v => this.propertyToDbValue(v, descriptor));
+                }
+                params.push(value);
+                conds.push(`"${condition.property}" IN ?`);
+            }
+            else {
+                const propValue = this.propertyToDbValue(condition.value, descriptor);
+                params.push(propValue);
+                if (condition.comparison === ComparisonOperator.Equals) {
+                    conds.push(`"${condition.property}" = ?`);
+                }
+                else if (condition.comparison === ComparisonOperator.NotEquals) {
+                    conds.push(`"${condition.property}" != ?`);
+                }
+                else if (condition.comparison === ComparisonOperator.GreaterThan) {
+                    conds.push(`"${condition.property}" > ?`);
+                }
+                else if (condition.comparison === ComparisonOperator.LessThan) {
+                    conds.push(`"${condition.property}" < ?`);
+                }
+                else if (condition.comparison === ComparisonOperator.GreaterThanOrEqual) {
+                    conds.push(`"${condition.property}" >= ?`);
+                }
+                else if (condition.comparison === ComparisonOperator.LessThanOrEqual) {
+                    conds.push(`"${condition.property}" <= ?`);
+                }
+            }
+        }
+        const operator = "conditions" in (conditions ?? {})
+            ? (conditions.logicalOperator ?? LogicalOperator.And)
+            : LogicalOperator.And;
+        let whereClause = `"${AbstractScyllaDBConnector.PARTITION_KEY}" = ?`;
+        if (conds.length > 0) {
+            whereClause += ` AND ${conds.join(` ${operator} `)}`;
+        }
+        return { whereClause, params };
+    }
 }
 //# sourceMappingURL=abstractScyllaDBConnector.js.map