@twin.org/entity-storage-connector-postgresql 0.0.3-next.1 → 0.0.3-next.11

package/README.md

@@ -1,6 +1,6 @@
-# TWIN Entity Storage Connector PostgreSql
+# Entity Storage Connector PostgreSQL
 
-Entity Storage connector implementation using PostgreSql storage.
+This package provides a PostgreSQL backend for relational persistence, transactions and advanced SQL features. It is designed to work with the wider storage ecosystem so applications can keep behaviour consistent across connectors and environments.
 
 ## Installation
 
@@ -8,18 +8,13 @@ Entity Storage connector implementation using PostgreSql storage.
 npm install @twin.org/entity-storage-connector-postgresql
 ```
 
-## Testing
+## Docker
 
-The tests developed are functional tests and need an instance of PostgreSql up and running. To run PostgreSql locally:
+To perform testing of this component it may be necessary to launch a local instance to communicate with.
 
-```sh
-docker run -p 5444:5432 --name twin-entity-storage-postgresql --hostname postgres -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=password -d postgres
-```
-
-Afterwards you can run the tests as follows:
-
-```sh
-npm run test
+```shell
+docker pull postgres:latest
+docker run -d --name twin-entity-storage-postgresql -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=password -p 5444:5432 postgres:latest
 ```
 
 ## Examples

package/dist/es/postgreSqlEntityStorageConnector.js

@@ -1,7 +1,7 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
-import { BaseError, Coerce, ComponentFactory, GeneralError, Guards, Is, ObjectHelper } from "@twin.org/core";
+import { BaseError, Coerce, ComponentFactory, GeneralError, Guards, HealthStatus, Is, ObjectHelper } from "@twin.org/core";
 import { ComparisonOperator, EntitySchemaFactory, EntitySchemaHelper, EntitySchemaPropertyType, LogicalOperator, SortDirection } from "@twin.org/entity";
 import postgres from "postgres";
 /**
@@ -115,7 +115,7 @@ export class PostgreSqlEntityStorageConnector {
                         tableName: this._config.tableName
                     }
                 });
-                const createTableQuery = `CREATE TABLE ${this._config.tableName} (${this.mapPostgreSqlProperties(this._entitySchema)})`;
+                const createTableQuery = `CREATE TABLE "${this._config.tableName}" (${this.mapPostgreSqlProperties(this._entitySchema)})`;
                 await dbConnection.unsafe(createTableQuery);
                 await this.waitForTableExists();
             }
@@ -153,6 +153,45 @@ export class PostgreSqlEntityStorageConnector {
     className() {
         return PostgreSqlEntityStorageConnector.CLASS_NAME;
     }
+    /**
+     * Get the health of the component.
+     * @returns The health of the component.
+     */
+    async health() {
+        try {
+            const sql = await this.createConnection();
+            await sql `SELECT 1 FROM ${sql(this._config.tableName)} LIMIT 0`;
+            return [
+                {
+                    source: PostgreSqlEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Ok,
+                    description: "healthDescription",
+                    data: { tableName: this._config.tableName }
+                }
+            ];
+        }
+        catch {
+            return [
+                {
+                    source: PostgreSqlEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Error,
+                    description: "healthDescription",
+                    message: "connectionFailed",
+                    data: { tableName: this._config.tableName }
+                }
+            ];
+        }
+    }
+    /**
+     * The component needs to be stopped when the node is closed.
+     * @returns Nothing.
+     */
+    async stop() {
+        if (this._connection) {
+            await this._connection.end();
+            this._connection = undefined;
+        }
+    }
     /**
      * Get the schema for the entities.
      * @returns The schema for the entities.
@@ -202,9 +241,17 @@ export class PostgreSqlEntityStorageConnector {
                         if ((prop.type === EntitySchemaPropertyType.Object ||
                             prop.type === EntitySchemaPropertyType.Array) &&
                             typeof row[propColumn] === "string") {
-                            const rowValue = JSON.parse(rows[0][propColumn]);
+                            let value;
+                            try {
+                                value = JSON.parse(rows[0][propColumn]);
+                            }
+                            catch {
+                                // If JSON.parse fails, keep the value as string
+                                // This handles cases where plain text was stored in Object/Array fields
+                                value = rows[0][propColumn];
+                            }
                             delete rows[0][propColumn];
-                            rows[0][prop.property] = rowValue;
+                            rows[0][prop.property] = value;
                         }
                         if (row[propColumn] === null) {
                             rows[0][prop.property] = undefined;
@@ -249,7 +296,6 @@ export class PostgreSqlEntityStorageConnector {
                 type: EntitySchemaPropertyType.String
             });
             ObjectHelper.propertySet(finalEntity, PostgreSqlEntityStorageConnector._PARTITION_KEY, partitionKey ?? PostgreSqlEntityStorageConnector._PARTITION_KEY_VALUE);
-            ObjectHelper.propertySet(finalEntity, PostgreSqlEntityStorageConnector._PARTITION_KEY, partitionKey ?? PostgreSqlEntityStorageConnector._PARTITION_KEY_VALUE);
             const keys = [];
             const values = [];
             for (const prop of props) {
@@ -277,6 +323,68 @@ export class PostgreSqlEntityStorageConnector {
             }, err);
         }
     }
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    async setBatch(entities) {
+        Guards.arrayValue(PostgreSqlEntityStorageConnector.CLASS_NAME, "entities", entities);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        for (const entity of entities) {
+            EntitySchemaHelper.validateEntity(entity, this.getSchema());
+        }
+        try {
+            const props = [...(this._entitySchema.properties ?? [])];
+            props.unshift({
+                property: PostgreSqlEntityStorageConnector._PARTITION_KEY,
+                type: EntitySchemaPropertyType.String
+            });
+            const keys = props.map(p => p.property);
+            const allValues = [];
+            const rowPlaceholders = [];
+            for (const entity of entities) {
+                const finalEntity = ObjectHelper.clone(entity);
+                ObjectHelper.propertySet(finalEntity, PostgreSqlEntityStorageConnector._PARTITION_KEY, partitionKey ?? PostgreSqlEntityStorageConnector._PARTITION_KEY_VALUE);
+                const rowValues = [];
+                for (const prop of props) {
+                    const val = finalEntity[prop.property];
+                    allValues.push(Is.empty(val) ? null : val);
+                    rowValues.push(`$${allValues.length}`);
+                }
+                rowPlaceholders.push(`(${rowValues.join(", ")})`);
+            }
+            let sql = `INSERT INTO "${this._config.tableName}"`;
+            sql += ` (${keys.map(key => `"${key}"`).join(", ")})`;
+            sql += ` VALUES ${rowPlaceholders.join(", ")}`;
+            sql += ` ON CONFLICT ("${PostgreSqlEntityStorageConnector._PARTITION_KEY}", "${this._primaryKeyProperty.property}")`;
+            sql += ` DO UPDATE SET ${keys.map(key => `"${key}" = EXCLUDED."${key}"`).join(", ")};`;
+            const dbConnection = await this.createConnection();
+            await dbConnection.unsafe(sql, allValues);
+        }
+        catch (err) {
+            throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "setBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Empty all the entities.
+     * @returns Nothing.
+     */
+    async empty() {
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        try {
+            const sql = `DELETE FROM "${this._config.tableName}" WHERE "${PostgreSqlEntityStorageConnector._PARTITION_KEY}" = $1`;
+            const dbConnection = await this.createConnection();
+            await dbConnection.unsafe(sql, [
+                partitionKey ?? PostgreSqlEntityStorageConnector._PARTITION_KEY_VALUE
+            ]);
+        }
+        catch (err) {
+            throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "emptyFailed", undefined, err);
+        }
+    }
     /**
      * Remove the entity.
      * @param id The id of the entity to remove.
@@ -313,6 +421,68 @@ export class PostgreSqlEntityStorageConnector {
             }, err);
         }
     }
+    /**
+     * Remove multiple entities by their primary key IDs.
+     * @param ids The ids of the entities to remove.
+     * @returns Nothing.
+     */
+    async removeBatch(ids) {
+        Guards.arrayValue(PostgreSqlEntityStorageConnector.CLASS_NAME, "ids", ids);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        try {
+            const sql = `DELETE FROM "${this._config.tableName}" WHERE "${PostgreSqlEntityStorageConnector._PARTITION_KEY}" = $1 AND "${this._primaryKeyProperty.property}" = ANY($2)`;
+            const dbConnection = await this.createConnection();
+            await dbConnection.unsafe(sql, [
+                partitionKey ?? PostgreSqlEntityStorageConnector._PARTITION_KEY_VALUE,
+                ids
+            ]);
+        }
+        catch (err) {
+            throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "removeBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Teardown the entity storage by dropping the table.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns True if the teardown process was successful.
+     */
+    async teardown(nodeLoggingComponentType) {
+        const nodeLogging = ComponentFactory.getIfExists(nodeLoggingComponentType);
+        await nodeLogging?.log({
+            level: "info",
+            source: PostgreSqlEntityStorageConnector.CLASS_NAME,
+            ts: Date.now(),
+            message: "tableDropping",
+            data: { tableName: this._config.tableName }
+        });
+        try {
+            const tableExists = await this.tableExists();
+            if (tableExists) {
+                const dbConnection = await this.createConnection();
+                await dbConnection.unsafe(`DROP TABLE "${this._config.tableName}";`);
+                await this.waitForTableNotExists();
+            }
+            await nodeLogging?.log({
+                level: "info",
+                source: PostgreSqlEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "tableDropped",
+                data: { tableName: this._config.tableName }
+            });
+            return true;
+        }
+        catch (err) {
+            await nodeLogging?.log({
+                level: "error",
+                source: PostgreSqlEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "teardownFailed",
+                error: BaseError.fromError(err)
+            });
+            return false;
+        }
+    }
     /**
      * Find all the entities which match the conditions.
      * @param conditions The conditions to match for the entities.
@@ -367,9 +537,17 @@ export class PostgreSqlEntityStorageConnector {
                         if ((prop.type === EntitySchemaPropertyType.Object ||
                             prop.type === EntitySchemaPropertyType.Array) &&
                             Is.string(row[propColumn])) {
-                            const rowValue = JSON.parse(row[propColumn]);
+                            let value;
+                            try {
+                                value = JSON.parse(row[propColumn]);
+                            }
+                            catch {
+                                // If JSON.parse fails, keep the value as string
+                                // This handles cases where plain text was stored in Object/Array fields
+                                value = row[propColumn];
+                            }
                             delete row[propColumn];
-                            row[prop.property] = rowValue;
+                            row[prop.property] = value;
                         }
                         if (row[propColumn] === null) {
                             row[prop.property] = undefined;
@@ -394,21 +572,19 @@ export class PostgreSqlEntityStorageConnector {
         }
     }
     /**
-     * Drop the table.
-     * @returns Nothing.
+     * Count all the entities which match the conditions.
+     * @returns The total count of entities in the storage.
      */
-    async tableDrop() {
+    async count() {
         try {
-            const tableExists = await this.tableExists();
-            if (!tableExists) {
-                return;
-            }
-            const dbConnection = await this.createConnection();
-            await dbConnection.unsafe(`DROP TABLE ${this._config.tableName};`);
-            await this.waitForTableNotExists();
+            const contextIds = await ContextIdStore.getContextIds();
+            const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+            const sql = await this.createConnection();
+            const result = await sql `SELECT COUNT(*) AS count FROM ${sql(this._config.tableName)} WHERE "partitionId" = ${partitionKey ?? PostgreSqlEntityStorageConnector._PARTITION_KEY_VALUE}`;
+            return Number(result[0].count);
         }
-        catch {
-            // Ignore errors
+        catch (err) {
+            throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "countFailed", undefined, err);
         }
     }
     /**
@@ -567,40 +743,99 @@ export class PostgreSqlEntityStorageConnector {
             const placeholders = inValues.map((_, index) => `$${valueIndex + index}`).join(", ");
             return `"${prop}" IN (${placeholders})`;
         }
+        // null/undefined must use IS NULL / IS NOT NULL — never a parameterised placeholder.
+        // Passing undefined through propertyToDbValue() coerces it to NaN for number fields
+        // (Number(undefined) === NaN), and null coerces to 0 (Number(null) === 0), both of
+        // which produce semantically wrong or invalid SQL.
+        if (comparator.value === null || comparator.value === undefined) {
+            if (comparator.comparison === ComparisonOperator.Equals ||
+                comparator.comparison === ComparisonOperator.NotEquals) {
+                const nullCheck = comparator.comparison === ComparisonOperator.Equals ? "IS NULL" : "IS NOT NULL";
+                if (comparator.property.split(".").length > 1) {
+                    const rootProp = comparator.property.split(".")[0];
+                    const nestedParts = comparator.property.split(".").slice(1);
+                    const jsonPath = nestedParts
+                        .map((p, i, arr) => (i === arr.length - 1 ? `->> '${p}'` : `-> '${p}'`))
+                        .join("");
+                    const jsonTextExpr = `("${rootProp}"::jsonb ${jsonPath})`;
+                    return `${jsonTextExpr} ${nullCheck}`;
+                }
+                return `"${prop}" ${nullCheck}`;
+            }
+        }
         const dbValue = this.propertyToDbValue(comparator.value, type);
         values.push(dbValue);
         if (comparator.property.split(".").length > 1) {
-            const jsonPath = comparator.property
-                .split(".")
-                .slice(1)
+            const rootProp = comparator.property.split(".")[0];
+            const nestedParts = comparator.property.split(".").slice(1);
+            const rootSchema = this._entitySchema.properties?.find(p => p.property === rootProp);
+            const isArray = rootSchema?.type === EntitySchemaPropertyType.Array;
+            const jsonPath = nestedParts
                 .map((p, i, arr) => (i === arr.length - 1 ? `->> '${p}'` : `-> '${p}'`))
                 .join("");
-            return `("${comparator.property.split(".")[0]}"::jsonb ${jsonPath}) = $${valueIndex}`;
-        }
-        else if (comparator.comparison === ComparisonOperator.Equals) {
-            return `"${prop}" = $${valueIndex}`;
-        }
-        else if (comparator.comparison === ComparisonOperator.NotEquals) {
-            return `"${prop}" <> $${valueIndex}`;
-        }
-        else if (comparator.comparison === ComparisonOperator.GreaterThan) {
-            return `"${prop}" > $${valueIndex}`;
-        }
-        else if (comparator.comparison === ComparisonOperator.LessThan) {
-            return `"${prop}" < $${valueIndex}`;
-        }
-        else if (comparator.comparison === ComparisonOperator.GreaterThanOrEqual) {
-            return `"${prop}" >= $${valueIndex}`;
-        }
-        else if (comparator.comparison === ComparisonOperator.LessThanOrEqual) {
-            return `"${prop}" <= $${valueIndex}`;
+            const jsonTextExpr = `("${rootProp}"::jsonb ${jsonPath})`;
+            switch (comparator.comparison) {
+                case ComparisonOperator.Includes: {
+                    values.pop();
+                    values.push(`%${String(comparator.value).toLowerCase()}%`);
+                    if (isArray) {
+                        const elemPath = nestedParts
+                            .map((p, i, arr) => (i === arr.length - 1 ? `->>'${p}'` : `->'${p}'`))
+                            .join("");
+                        return `EXISTS (SELECT 1 FROM jsonb_array_elements("${rootProp}") elem WHERE LOWER(elem${elemPath}) ILIKE $${valueIndex})`;
+                    }
+                    return `LOWER(${jsonTextExpr}) ILIKE $${valueIndex}`;
+                }
+                case ComparisonOperator.NotEquals:
+                    return `${jsonTextExpr} <> $${valueIndex}`;
+                case ComparisonOperator.GreaterThan:
+                    return `${jsonTextExpr} > $${valueIndex}`;
+                case ComparisonOperator.LessThan:
+                    return `${jsonTextExpr} < $${valueIndex}`;
+                case ComparisonOperator.GreaterThanOrEqual:
+                    return `${jsonTextExpr} >= $${valueIndex}`;
+                case ComparisonOperator.LessThanOrEqual:
+                    return `${jsonTextExpr} <= $${valueIndex}`;
+                default:
+                    return `${jsonTextExpr} = $${valueIndex}`;
+            }
         }
-        else if (comparator.comparison === ComparisonOperator.Includes) {
-            return `EXISTS (SELECT 1 FROM jsonb_array_elements("${prop}") elem WHERE elem @> $${valueIndex}::jsonb)`;
+        switch (comparator.comparison) {
+            case ComparisonOperator.Equals:
+                if (Is.object(comparator.value) || Is.array(comparator.value)) {
+                    return `"${prop}" = $${valueIndex}::jsonb`;
+                }
+                return `"${prop}" = $${valueIndex}`;
+            case ComparisonOperator.NotEquals:
+                if (Is.object(comparator.value) || Is.array(comparator.value)) {
+                    return `"${prop}" != $${valueIndex}::jsonb`;
+                }
+                return `"${prop}" <> $${valueIndex}`;
+            case ComparisonOperator.GreaterThan:
+                return `"${prop}" > $${valueIndex}`;
+            case ComparisonOperator.LessThan:
+                return `"${prop}" < $${valueIndex}`;
+            case ComparisonOperator.GreaterThanOrEqual:
+                return `"${prop}" >= $${valueIndex}`;
+            case ComparisonOperator.LessThanOrEqual:
+                return `"${prop}" <= $${valueIndex}`;
+            case ComparisonOperator.Includes: {
+                if (type === EntitySchemaPropertyType.String) {
+                    return `"${prop}" ILIKE '%' || $${valueIndex} || '%'`;
+                }
+                if (type === EntitySchemaPropertyType.Array || type === EntitySchemaPropertyType.Object) {
+                    return `EXISTS (SELECT 1 FROM jsonb_array_elements("${prop}") elem WHERE elem @> $${valueIndex}::jsonb)`;
+                }
+                throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "comparisonNotSupported", {
+                    comparison: comparator.comparison,
+                    type
+                });
+            }
+            default:
+                throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "comparisonNotSupported", {
+                    comparison: comparator.comparison
+                });
         }
-        throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "comparisonNotSupported", {
-            comparison: comparator.comparison
-        });
     }
     /**
      * Format a value to insert into DB.
@@ -610,21 +845,19 @@ export class PostgreSqlEntityStorageConnector {
      * @internal
      */
     propertyToDbValue(value, type) {
-        if (type === "string") {
+        if (type === EntitySchemaPropertyType.String) {
             return String(value);
         }
-        else if (type === "number") {
+        else if (type === EntitySchemaPropertyType.Number) {
             return Number(value);
         }
-        else if (type === "boolean") {
+        else if (type === EntitySchemaPropertyType.Boolean) {
             return Boolean(value);
         }
-        else if (type === "array") {
+        else if (type === EntitySchemaPropertyType.Object ||
+            type === EntitySchemaPropertyType.Array) {
             return value;
         }
-        if (Is.object(value)) {
-            return JSON.stringify(value);
-        }
         return value;
     }
     /**
@@ -681,7 +914,48 @@ export class PostgreSqlEntityStorageConnector {
         });
         const columnDefinitions = props
             .map(prop => {
-            const sqlType = sqlTypeMap[prop.type] || "TEXT";
+            let sqlType = sqlTypeMap[prop.type] || "TEXT";
+            if (prop.format) {
+                switch (prop.type) {
+                    case EntitySchemaPropertyType.String:
+                        switch (prop.format) {
+                            case "uuid":
+                                sqlType = "UUID";
+                                break;
+                        }
+                        break;
+                    case EntitySchemaPropertyType.Number:
+                        switch (prop.format) {
+                            case "float":
+                                sqlType = "REAL";
+                                break;
+                            case "double":
+                                sqlType = "DOUBLE PRECISION";
+                                break;
+                        }
+                        break;
+                    case EntitySchemaPropertyType.Integer:
+                        switch (prop.format) {
+                            case "int8":
+                            case "uint8":
+                                sqlType = "SMALLINT";
+                                break;
+                            case "int16":
+                                sqlType = "SMALLINT";
+                                break;
+                            case "uint16":
+                            case "int32":
+                                sqlType = "INTEGER";
+                                break;
+                            case "uint32":
+                            case "int64":
+                            case "uint64":
+                                sqlType = "BIGINT";
+                                break;
+                        }
+                        break;
+                }
+            }
             const columnName = String(prop.property);
             const nullable = prop.optional ? " NULL" : " NOT NULL";
             if (prop.isPrimary) {