@twin.org/entity-storage-connector-postgresql 0.10.0 → 0.10.1-next.10

package/dist/es/postgreSqlEntityStorageConnector.js

@@ -29,11 +29,22 @@ export class PostgreSqlEntityStorageConnector {
      * @internal
      */
     static _PARTITION_KEY_VALUE = "root";
+    /**
+     * Maximum length of the partition id column. The column leads the primary key and every
+     * index, so it is bounded rather than stored as unconstrained text.
+     * @internal
+     */
+    static _PARTITION_KEY_MAX_LENGTH = 255;
     /**
      * Maximum number of rows per INSERT statement in setBatch.
      * @internal
      */
     static _BATCH_CHUNK_SIZE = 1000;
+    /**
+     * The column the group ranking is emitted as when picking one row per group.
+     * @internal
+     */
+    static _GROUP_RANK_COLUMN = "__groupRank";
     /**
      * PostgreSQL's maximum identifier length in characters; longer names are silently truncated.
      * @internal
@@ -200,13 +211,18 @@ export class PostgreSqlEntityStorageConnector {
                     }
                 });
             }
+            const indexes = await this.readIndexes(dbConnection);
             for (const prop of this._entitySchema.properties ?? []) {
                 if ((prop.isSecondary === true || !Is.empty(prop.sortDirection)) &&
                     prop.type !== EntitySchemaPropertyType.Object &&
                     prop.type !== EntitySchemaPropertyType.Array) {
-                    await this.ensureIndex(dbConnection, prop, nodeLogging);
+                    await this.ensureIndex(dbConnection, indexes, prop, nodeLogging);
                 }
             }
+            const indexGroups = EntitySchemaHelper.getIndexGroups(this._entitySchema);
+            for (const indexProperties of Object.values(indexGroups)) {
+                await this.ensureCompositeIndex(dbConnection, indexes, indexProperties);
+            }
         }
         catch (error) {
             await nodeLogging?.log({
@@ -644,7 +660,7 @@ export class PostgreSqlEntityStorageConnector {
      * @returns The connector implementation version.
      */
     connectorVersion() {
-        return 0;
+        return 1;
     }
     /**
      * Get all the distinct partition context ids from the storage.
@@ -707,17 +723,20 @@ export class PostgreSqlEntityStorageConnector {
         });
     }
     /**
-     * Finalize the migration by renaming the migration table to the original table name.
+     * Finalize the migration by dropping the source table and renaming the migration table into its name in one transaction.
      * @param targetConnector The connector pointing to the migration table.
      * @param options The optional migration options.
      * @param loggingComponentType The node logging component type.
      * @returns A connector pointing to the final (renamed) table.
      */
     async finalizeMigration(targetConnector, options, loggingComponentType) {
-        // Teardown the existing table with the original name to free up the name for the new table
-        await this.teardown(loggingComponentType);
-        const dbConnection = await targetConnector.getClient();
-        await dbConnection.unsafe(`ALTER TABLE "${targetConnector._config.tableName}" RENAME TO "${this._config.tableName}"`);
+        // One transaction drops the source and renames the migration table, so a failure or a
+        // process death leaves either the untouched source or the complete migrated table in place.
+        const dbConnection = await this.getClient();
+        await dbConnection.begin(async (transaction) => {
+            await transaction.unsafe(`DROP TABLE "${this._config.tableName}"`);
+            await transaction.unsafe(`ALTER TABLE "${targetConnector._config.tableName}" RENAME TO "${this._config.tableName}"`);
+        });
         const finalConnector = new PostgreSqlEntityStorageConnector({
             entitySchema: targetConnector._entitySchemaName,
             config: this._config,
@@ -874,6 +893,37 @@ export class PostgreSqlEntityStorageConnector {
             throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "queryFailed", { sql }, err);
         }
     }
+    /**
+     * Find all the entities which match the conditions, attaching to each one the entities from a
+     * second storage connector whose join property matches. The join behaves like a left join by
+     * default, a primary entity with no matches is still returned with an empty joined list, unless
+     * joinRequired asks for an inner join and those entities are left out altogether. Both connectors
+     * must be PostgreSQL connectors reading from the same database so the work can be done in a
+     * single statement.
+     * @param joinConnector The connector holding the entities to join to.
+     * @param joinOptions The properties to join on, the conditions, sort order, projection and
+     * paging for the primary entities, the optional grouping and group conditions, and the optional
+     * conditions, sort order and projection for the joined entities.
+     * @returns All the entities for the storage matching the conditions with their joined entities,
+     * and a cursor which can be used to request more entities.
+     * @throws GeneralError if the join connector does not read from the same server and database.
+     */
+    async queryJoin(joinConnector, joinOptions) {
+        Guards.object(PostgreSqlEntityStorageConnector.CLASS_NAME, "joinConnector", joinConnector);
+        // The join runs as one statement against this connector's connection, so the other side has
+        // to be a PostgreSQL connector reading from the same server and database.
+        const typedJoinConnector = joinConnector;
+        if (joinConnector.className?.() !== PostgreSqlEntityStorageConnector.CLASS_NAME ||
+            typedJoinConnector._config?.host !== this._config.host ||
+            typedJoinConnector._config?.port !== this._config.port ||
+            typedJoinConnector._config?.database !== this._config.database) {
+            throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "joinConnectorMismatch", {
+                database: this._config.database
+            });
+        }
+        EntityStorageHelper.validateJoinOptions(this._entitySchema, typedJoinConnector._entitySchema, joinOptions);
+        return this.queryJoinPage(typedJoinConnector, joinOptions, joinOptions.groupProperty);
+    }
     /**
      * Count all the entities which match the conditions.
      * @param conditions The optional conditions to match for the entities.
@@ -898,6 +948,399 @@ export class PostgreSqlEntityStorageConnector {
             throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "countFailed", { sql: queryStr }, err);
         }
     }
+    /**
+     * Read a page of primary entities and their joined entities in a single statement. The page of
+     * primary rows is selected in a derived table so the limit and the cursor apply to the primary
+     * entities rather than to the rows the join multiplies them into. When grouping, the derived
+     * table keeps only the first row of each group in the sort order, which makes one row stand for
+     * the whole group and lets the same key set cursor step past every row the group holds.
+     * @param joinConnector The connector holding the entities to join to.
+     * @param joinOptions The join configuration.
+     * @param groupProperty The optional property to group the primary entities by.
+     * @returns The entities with their joined entities, and the next page cursor.
+     * @internal
+     */
+    async queryJoinPage(joinConnector, joinOptions, groupProperty) {
+        const returnSize = joinOptions.limit ?? PostgreSqlEntityStorageConnector._DEFAULT_LIMIT;
+        const pkPropName = String(this._primaryKeyProperty.property);
+        const joinColumn = String(joinOptions.property);
+        const joinedPrimaryKey = String(joinConnector._primaryKeyProperty.property);
+        const normalizedOptions = EntityStorageHelper.normalizeJoinOptions(joinOptions);
+        const keySetValues = EntityStorageHelper.decodeCursor(normalizedOptions, joinOptions.cursor);
+        let sql = "";
+        try {
+            const keySetCols = this.buildKeySetColumns(joinOptions.sortProperties);
+            // The key set columns, the join column and the group column are needed to page, to join
+            // and to re-attach the members of a group, so they are read even when the caller did not
+            // ask for them, then removed from the entities.
+            const primary = this.buildColumnSelection(this._entitySchema, joinOptions.properties, keySetCols
+                .map(c => c.prop)
+                .concat(joinColumn)
+                .concat(Is.empty(groupProperty) ? [] : [String(groupProperty)]));
+            const joined = this.buildColumnSelection(joinConnector._entitySchema, joinOptions.joinProperties, [joinedPrimaryKey]);
+            const partitionKey = await this.resolvePartitionKey();
+            const values = [];
+            // PostgreSQL numbers its placeholders, so every clause is built in the order it appears
+            // in the statement and its values are appended as it goes.
+            const where = this.buildWhereClause(joinOptions.conditions, partitionKey, "t", values.length + 1);
+            values.push(...where.values);
+            const whereClauses = [...where.whereClauses];
+            if (!Is.empty(groupProperty)) {
+                whereClauses.push(`t."${String(groupProperty)}" IS NOT NULL`);
+            }
+            const narrowing = await this.buildNarrowingClauses(joinConnector, joinOptions, groupProperty, partitionKey, "t", values.length + 1);
+            whereClauses.push(...narrowing.clauses);
+            values.push(...narrowing.values);
+            const columnList = primary.columns.map(c => `"${c}"`).join(", ");
+            const pageOrderBy = keySetCols.map(c => `"${c.prop}" ${c.asc ? "ASC" : "DESC"}`).join(", ");
+            const keySet = this.buildKeySetClause(keySetCols, keySetValues, values.length + 1);
+            values.push(...keySet.values);
+            let pageSql;
+            if (Is.empty(groupProperty)) {
+                pageSql = `SELECT ${columnList} FROM "${this._config.tableName}" AS t WHERE ${[...whereClauses, ...keySet.clauses].join(" AND ")} ORDER BY ${pageOrderBy} LIMIT ${returnSize + 1}`;
+            }
+            else {
+                // Ranking inside each group and keeping the first row collapses the group to the one
+                // row the result stands on, so the ordering, the projection and the cursor all work
+                // on ordinary rows rather than on a distinct list of group values.
+                const rankedSql = `SELECT ${columnList}, ROW_NUMBER() OVER (PARTITION BY t."${String(groupProperty)}" ORDER BY ${keySetCols.map(c => `t."${c.prop}" ${c.asc ? "ASC" : "DESC"}`).join(", ")}) AS "${PostgreSqlEntityStorageConnector._GROUP_RANK_COLUMN}" FROM "${this._config.tableName}" AS t WHERE ${whereClauses.join(" AND ")}`;
+                const rankedWhere = [
+                    `ranked."${PostgreSqlEntityStorageConnector._GROUP_RANK_COLUMN}" = 1`,
+                    ...keySet.clauses
+                ];
+                pageSql = `SELECT ${columnList} FROM (${rankedSql}) AS ranked WHERE ${rankedWhere.join(" AND ")} ORDER BY ${pageOrderBy} LIMIT ${returnSize + 1}`;
+            }
+            // A group stands on one row for ordering and paging, but its joined list has to hold the
+            // matches of every entity in the group, so the other members are re-attached to the page
+            // and the join hangs off them. The same joined entity reached through more than one
+            // member is collapsed when the rows are collected.
+            let fromClause = `(${pageSql}) AS p`;
+            let joinFromAlias = "p";
+            if (!Is.empty(groupProperty)) {
+                const members = this.buildWhereClause(joinOptions.conditions, partitionKey, "m", values.length + 1);
+                values.push(...members.values);
+                const memberNarrowing = await this.buildNarrowingClauses(joinConnector, joinOptions, undefined, partitionKey, "m", values.length + 1);
+                values.push(...memberNarrowing.values);
+                fromClause += ` LEFT JOIN "${this._config.tableName}" AS m ON ${[
+                    `m."${String(groupProperty)}" = p."${String(groupProperty)}"`,
+                    ...members.whereClauses,
+                    ...memberNarrowing.clauses
+                ].join(" AND ")}`;
+                joinFromAlias = "m";
+            }
+            const join = await joinConnector.buildJoinClause(String(joinOptions.joinProperty), joinOptions.joinConditions, "j", joinFromAlias, joinColumn, values.length + 1);
+            values.push(...join.values);
+            // PostgreSQL returns a flat row, so each column is aliased by position and mapped back
+            // afterwards rather than relying on the table it came from.
+            const selectClause = primary.columns
+                .map((c, i) => `p."${c}" AS "p${i}"`)
+                .concat(joined.columns.map((c, i) => `j."${c}" AS "j${i}"`))
+                .join(", ");
+            const outerOrderBy = keySetCols
+                .map(c => `p."${c.prop}" ${c.asc ? "ASC" : "DESC"}`)
+                .concat(this.buildJoinOrderBy(joinOptions, "j"))
+                .join(", ");
+            sql = `SELECT ${selectClause} FROM ${fromClause} LEFT JOIN "${joinConnector._config.tableName}" AS j ON ${join.clause} ORDER BY ${outerOrderBy}`;
+            const dbConnection = await this.getClient();
+            const rows = await dbConnection.unsafe(sql, values);
+            const groups = this.collectJoinRows(rows, primary.columns, joined.columns, pkPropName, joinedPrimaryKey);
+            const hasMore = groups.length > returnSize;
+            const pageGroups = hasMore ? groups.slice(0, returnSize) : groups;
+            const entities = [];
+            for (const group of pageGroups) {
+                const entity = this.prepareJoinEntity(group.key, this._entitySchema, primary.internal);
+                entities.push({
+                    ...entity,
+                    joined: group.joined.map(j => this.prepareJoinEntity(j, joinConnector._entitySchema, joined.internal))
+                });
+            }
+            let nextCursor;
+            if (hasMore && pageGroups.length > 0) {
+                const lastRow = pageGroups[pageGroups.length - 1].key;
+                nextCursor = EntityStorageHelper.encodeCursor(normalizedOptions, keySetCols.map(c => lastRow[c.prop]));
+            }
+            return { entities, cursor: nextCursor };
+        }
+        catch (err) {
+            throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "queryJoinFailed", { sql }, err);
+        }
+    }
+    /**
+     * Build the clauses which narrow the rows a page is built from, beyond the plain conditions.
+     * An inner join requires the row to have at least one joined entity, and a group condition
+     * requires the group the row belongs to to hold an entity which matches it.
+     * @param joinConnector The connector holding the entities to join to.
+     * @param joinOptions The join configuration.
+     * @param groupProperty The optional property the entities are grouped by.
+     * @param partitionKey The partition key of this connector.
+     * @param alias The alias of the row being narrowed.
+     * @param startIndex The number the first placeholder takes.
+     * @returns The clauses and their bound values.
+     * @internal
+     */
+    async buildNarrowingClauses(joinConnector, joinOptions, groupProperty, partitionKey, alias, startIndex) {
+        const clauses = [];
+        const values = [];
+        if (joinOptions.joinRequired ?? false) {
+            const exists = await joinConnector.buildJoinExistsClause(String(joinOptions.joinProperty), joinOptions.joinConditions, "jx", alias, String(joinOptions.property), startIndex + values.length);
+            clauses.push(exists.clause);
+            values.push(...exists.values);
+        }
+        if (!Is.empty(groupProperty) && Is.arrayValue(joinOptions.groupConditions)) {
+            const groupColumn = String(groupProperty);
+            for (let i = 0; i < joinOptions.groupConditions.length; i++) {
+                const memberAlias = `gx${i}`;
+                const group = this.buildWhereClause(joinOptions.groupConditions[i], partitionKey, memberAlias, startIndex + values.length);
+                clauses.push(`EXISTS (SELECT 1 FROM "${this._config.tableName}" AS ${memberAlias} WHERE ${memberAlias}."${groupColumn}" = ${alias}."${groupColumn}" AND ${group.whereClauses.join(" AND ")})`);
+                values.push(...group.values);
+            }
+        }
+        return { clauses, values };
+    }
+    /**
+     * Build the clause which requires a primary entity to have at least one joined entity. Called
+     * on the connector holding the joined entities so its partition key and property types apply.
+     * @param joinProperty The column on this connector's table to join to.
+     * @param joinConditions The optional conditions to match for the joined entities.
+     * @param alias The alias of the joined table inside the clause.
+     * @param primaryAlias The alias of the primary entity being narrowed.
+     * @param primaryColumn The column on the primary entity to join from.
+     * @param startIndex The number the first placeholder takes.
+     * @returns The clause and its bound values.
+     * @internal
+     */
+    async buildJoinExistsClause(joinProperty, joinConditions, alias, primaryAlias, primaryColumn, startIndex) {
+        const partitionKey = await this.resolvePartitionKey();
+        const where = this.buildWhereClause(joinConditions, partitionKey, alias, startIndex);
+        return {
+            clause: `EXISTS (SELECT 1 FROM "${this._config.tableName}" AS ${alias} WHERE ${alias}."${joinProperty}" = ${primaryAlias}."${primaryColumn}" AND ${where.whereClauses.join(" AND ")})`,
+            values: where.values
+        };
+    }
+    /**
+     * Build the ON clause which attaches the joined table, including its own partition key and any
+     * conditions the caller supplied for the joined entities. Called on the connector holding the
+     * joined entities so the partition key and the property types come from its own schema and
+     * configuration.
+     * @param joinProperty The column on this connector's table to join to.
+     * @param joinConditions The optional conditions to match for the joined entities.
+     * @param alias The alias of the joined table.
+     * @param primaryAlias The alias of the table holding the primary entities.
+     * @param primaryColumn The column on the primary table to join from.
+     * @param startIndex The number the first placeholder takes.
+     * @returns The ON clause and its bound values.
+     * @internal
+     */
+    async buildJoinClause(joinProperty, joinConditions, alias, primaryAlias, primaryColumn, startIndex) {
+        const partitionKey = await this.resolvePartitionKey();
+        const where = this.buildWhereClause(joinConditions, partitionKey, alias, startIndex);
+        return {
+            clause: [
+                `${alias}."${joinProperty}" = ${primaryAlias}."${primaryColumn}"`,
+                ...where.whereClauses
+            ].join(" AND "),
+            values: where.values
+        };
+    }
+    /**
+     * Build the ORDER BY fragments which order the joined entities within each primary entity.
+     * @param joinOptions The join configuration.
+     * @param alias The alias of the joined table.
+     * @returns The order by fragments, empty when no sort order was requested.
+     * @internal
+     */
+    buildJoinOrderBy(joinOptions, alias) {
+        return (joinOptions.joinSortProperties ?? []).map(s => `${alias}."${String(s.property)}" ${s.sortDirection === SortDirection.Ascending ? "ASC" : "DESC"}`);
+    }
+    /**
+     * Build the ordered keySet columns used for the page order and the cursor, matching the
+     * behaviour of query so both paginate the same way.
+     * @param sortProperties The optional sort order.
+     * @returns The ordered columns with their direction.
+     * @internal
+     */
+    buildKeySetColumns(sortProperties) {
+        const pkPropName = String(this._primaryKeyProperty.property);
+        const keySetCols = [];
+        for (const sortProperty of sortProperties ?? []) {
+            keySetCols.push({
+                prop: String(sortProperty.property),
+                asc: sortProperty.sortDirection === SortDirection.Ascending
+            });
+        }
+        if (!keySetCols.some(c => c.prop === pkPropName)) {
+            keySetCols.push({ prop: pkPropName, asc: true });
+        }
+        return keySetCols;
+    }
+    /**
+     * Build the keySet condition which continues the page from a previous cursor.
+     * @param keySetCols The ordered keySet columns.
+     * @param lastValues The key set values of the last entity of the previous page.
+     * @param startIndex The number the first placeholder takes.
+     * @returns The clauses and their bound values.
+     * @internal
+     */
+    buildKeySetClause(keySetCols, lastValues, startIndex) {
+        if (!Is.arrayValue(lastValues)) {
+            return { clauses: [], values: [] };
+        }
+        const values = [];
+        const orParts = [];
+        for (let i = 0; i < keySetCols.length; i++) {
+            const parts = [];
+            for (let j = 0; j < i; j++) {
+                values.push(lastValues[j]);
+                parts.push(`"${keySetCols[j].prop}" = $${startIndex + values.length - 1}`);
+            }
+            const op = keySetCols[i].asc ? ">" : "<";
+            values.push(lastValues[i]);
+            parts.push(`"${keySetCols[i].prop}" ${op} $${startIndex + values.length - 1}`);
+            orParts.push(parts.length === 1 ? parts[0] : `(${parts.join(" AND ")})`);
+        }
+        return { clauses: [`(${orParts.join(" OR ")})`], values };
+    }
+    /**
+     * Work out which columns to read for one side of the join, honouring the caller's projection
+     * but adding the columns the join itself needs.
+     * @param schema The schema of the entities being read.
+     * @param properties The optional projection requested by the caller.
+     * @param required The columns the join needs regardless of the projection.
+     * @returns The columns to read and the ones which were only added internally.
+     * @internal
+     */
+    buildColumnSelection(schema, properties, required) {
+        const columns = [];
+        const internal = [];
+        if (Is.arrayValue(properties)) {
+            for (const prop of properties) {
+                const column = String(prop);
+                if (!columns.includes(column)) {
+                    columns.push(column);
+                }
+            }
+            for (const column of required) {
+                if (!columns.includes(column)) {
+                    columns.push(column);
+                    internal.push(column);
+                }
+            }
+        }
+        else {
+            for (const prop of schema.properties ?? []) {
+                const column = String(prop.property);
+                if (!columns.includes(column)) {
+                    columns.push(column);
+                }
+            }
+        }
+        return { columns, internal };
+    }
+    /**
+     * Collapse the flat rows returned by the join into one entry per primary entity, preserving the
+     * order the database returned them in.
+     * @param rows The rows from the join statement, whose columns are aliased by position.
+     * @param primaryColumns The primary columns in the order they were aliased.
+     * @param joinedColumns The joined columns in the order they were aliased.
+     * @param identityColumn The column which identifies a primary entity.
+     * @param joinedPrimaryKey The primary key column of the joined entities.
+     * @returns One entry per primary entity with its joined entities.
+     * @internal
+     */
+    collectJoinRows(rows, primaryColumns, joinedColumns, identityColumn, joinedPrimaryKey) {
+        const groups = [];
+        if (!Is.array(rows)) {
+            return groups;
+        }
+        let current;
+        let currentIdentity;
+        let seenJoined = new Set();
+        for (const row of rows) {
+            const primaryRow = {};
+            for (let i = 0; i < primaryColumns.length; i++) {
+                primaryRow[primaryColumns[i]] = row[`p${i}`];
+            }
+            const identity = this.rowKey(primaryRow[identityColumn]);
+            if (Is.undefined(current) || identity !== currentIdentity) {
+                current = { key: primaryRow, joined: [] };
+                currentIdentity = identity;
+                seenJoined = new Set();
+                groups.push(current);
+            }
+            const joinedRow = {};
+            for (let i = 0; i < joinedColumns.length; i++) {
+                joinedRow[joinedColumns[i]] = row[`j${i}`];
+            }
+            // A left join with no match produces a row whose joined columns are all null, and the
+            // same joined entity appears more than once when several primary rows in a group share
+            // the same join value.
+            if (!Is.empty(joinedRow[joinedPrimaryKey])) {
+                const joinedIdentity = this.rowKey(joinedRow[joinedPrimaryKey]);
+                if (!seenJoined.has(joinedIdentity)) {
+                    seenJoined.add(joinedIdentity);
+                    current.joined.push(joinedRow);
+                }
+            }
+        }
+        return groups;
+    }
+    /**
+     * Turn a raw column value into a key which can be compared between rows.
+     * @param value The value read from the storage.
+     * @returns The key for the value.
+     * @internal
+     */
+    rowKey(value) {
+        return Is.string(value) ? value : JSON.stringify(value);
+    }
+    /**
+     * Apply to a row read by a join the same clean up a plain query applies to its entities, then
+     * remove the columns which were only read to satisfy the join.
+     * @param row The raw row read from the storage.
+     * @param schema The schema of the entity.
+     * @param internal The columns to remove.
+     * @returns The entity.
+     * @internal
+     */
+    prepareJoinEntity(row, schema, internal) {
+        const prepared = { ...row };
+        for (const prop of schema.properties ?? []) {
+            const column = String(prop.property);
+            if ((prop.type === EntitySchemaPropertyType.Object ||
+                prop.type === EntitySchemaPropertyType.Array) &&
+                Is.string(prepared[column])) {
+                try {
+                    prepared[column] = JSON.parse(String(prepared[column]));
+                }
+                catch {
+                    // Text which is not JSON is left as it was read, matching what a plain query does.
+                }
+            }
+        }
+        return EntityStorageHelper.unPrepareEntity(prepared, [
+            PostgreSqlEntityStorageConnector._PARTITION_KEY,
+            ...internal
+        ]);
+    }
+    /**
+     * Get the partition key for the current context.
+     * @returns The partition key, or undefined when the connector is not partitioned.
+     * @internal
+     */
+    async resolvePartitionKey() {
+        const contextIds = await ContextIdStore.getContextIds();
+        return ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+    }
+    /**
+     * Qualify a column with a table alias, needed when the statement reads from more than one table.
+     * @param column The column name.
+     * @param tableAlias The optional table alias.
+     * @returns The quoted column, prefixed with the alias when one was supplied.
+     * @internal
+     */
+    qualifiedColumn(column, tableAlias) {
+        return Is.stringValue(tableAlias) ? `${tableAlias}."${column}"` : `"${column}"`;
+    }
     /**
      * Check if the database exists.
      * @param adminClient The server-level connection to use for the check.
@@ -929,64 +1372,123 @@ export class PostgreSqlEntityStorageConnector {
         }
     }
     /**
-     * Ensure the secondary index for a property exists, replacing a legacy-named index if present.
+     * Ensure the secondary index for a property exists, dropping a legacy-named index if present.
+     * Every query is scoped to a single partition, so the index leads with the partition key and
+     * the property follows it, letting one index serve both the partition filter and the sort.
      * @param dbConnection The connection to query with.
+     * @param indexes The indexes already on the table, keyed by index name.
      * @param prop The indexed property.
      * @param nodeLogging Optional logging component.
      * @internal
      */
-    async ensureIndex(dbConnection, prop, nodeLogging) {
+    async ensureIndex(dbConnection, indexes, prop, nodeLogging) {
         const columnName = String(prop.property);
         const indexName = IndexHelper.generateName(this._config.tableName, columnName);
-        const indexRows = await dbConnection.unsafe(`SELECT i.relname AS "indexName", ix.indisunique AS "isUnique", ix.indnkeyatts AS "keyColumnCount"
+        const keyColumns = [PostgreSqlEntityStorageConnector._PARTITION_KEY, columnName];
+        if (!this.isIndexCovered(indexes, keyColumns)) {
+            // An index of ours under the same name but with a different shape predates the partition
+            // key leading the key columns, so it has to be replaced rather than left in place.
+            if (!Is.empty(indexes[indexName])) {
+                await dbConnection.unsafe(`DROP INDEX "${indexName}"`);
+            }
+            await dbConnection.unsafe(`CREATE INDEX IF NOT EXISTS "${indexName}" ON "${this._config.tableName}" ("${PostgreSqlEntityStorageConnector._PARTITION_KEY}", "${columnName}")`);
+        }
+        // TODO: remove the legacy index handling once every installation has bootstrapped on a release that contains it
+        const legacyName = IndexHelper.generateLegacyName(this._config.tableName, columnName, IndexHelper.DEFAULT_MAX_IDENTIFIER_LENGTH);
+        const legacyIndex = indexes[legacyName];
+        // The connector's own legacy indexes were always non-unique and single-column, anything else is an operator's
+        if (!Is.empty(legacyIndex) && legacyIndex.nonUnique && legacyIndex.columns.length === 1) {
+            await dbConnection.unsafe(`DROP INDEX "${legacyName}"`);
+            await nodeLogging?.log({
+                level: "info",
+                source: PostgreSqlEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "legacyIndexDropped",
+                data: {
+                    tableName: this._config.tableName,
+                    indexName: legacyName,
+                    newIndexName: indexName
+                }
+            });
+        }
+    }
+    /**
+     * Ensure the composite index for a schema index group exists.
+     * A group needs at least two properties to form a composite index, otherwise it is skipped.
+     * The partition key leads the index for the same reason it leads a single property index.
+     * @param dbConnection The connection to query with.
+     * @param indexes The indexes already on the table, keyed by index name.
+     * @param indexProperties The properties in the group, ordered by their index position.
+     * @internal
+     */
+    async ensureCompositeIndex(dbConnection, indexes, indexProperties) {
+        const indexName = IndexHelper.generateCompositeName(this._config.tableName, indexProperties);
+        const keyColumns = [
+            PostgreSqlEntityStorageConnector._PARTITION_KEY,
+            ...indexProperties.map(indexProperty => String(indexProperty.property.property))
+        ];
+        if (this.isIndexCovered(indexes, keyColumns)) {
+            return;
+        }
+        // An index of ours under the same name but with a different shape predates the partition
+        // key leading the key columns, so it has to be replaced rather than left in place.
+        if (!Is.empty(indexes[indexName])) {
+            await dbConnection.unsafe(`DROP INDEX "${indexName}"`);
+        }
+        const indexCols = [
+            `"${PostgreSqlEntityStorageConnector._PARTITION_KEY}" ASC`,
+            ...indexProperties.map(indexProperty => `"${String(indexProperty.property.property)}" ${indexProperty.direction === SortDirection.Descending ? "DESC" : "ASC"}`)
+        ].join(", ");
+        await dbConnection.unsafe(`CREATE INDEX IF NOT EXISTS "${indexName}" ON "${this._config.tableName}" (${indexCols})`);
+    }
+    /**
+     * Read the key columns of every usable index on the table, in key order.
+     * @param dbConnection The connection to query with.
+     * @returns The key columns and uniqueness of each index, keyed by index name.
+     * @internal
+     */
+    async readIndexes(dbConnection) {
+        const indexRows = await dbConnection.unsafe(`SELECT i.relname AS "indexName", a.attname AS "columnName", k.pos AS "position", ix.indisunique AS "isUnique"
 			FROM pg_index ix
 			JOIN pg_class t ON t.oid = ix.indrelid
 			JOIN pg_namespace n ON n.oid = t.relnamespace
-			JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ix.indkey[0]
 			JOIN pg_class i ON i.oid = ix.indexrelid
 			JOIN pg_am am ON am.oid = i.relam
+			JOIN LATERAL generate_series(0, ix.indnkeyatts - 1) AS k(pos) ON true
+			JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ix.indkey[k.pos]
 			WHERE n.nspname = 'public'
 				AND t.relname = $1
-				AND a.attname = $2
 				AND ix.indisvalid
 				AND ix.indisready
 				AND ix.indpred IS NULL
-				AND am.amname = 'btree'`, [this._config.tableName, columnName]);
-        const indexNames = indexRows.map(row => ObjectHelper.propertyGet(row, "indexName"));
-        if (!Is.arrayValue(indexNames)) {
-            await dbConnection.unsafe(`CREATE INDEX IF NOT EXISTS "${indexName}" ON "${this._config.tableName}" ("${columnName}")`);
-            return;
-        }
-        // TODO: remove the legacy index handling once every installation has bootstrapped on a release that contains it
-        const legacyName = IndexHelper.generateLegacyName(this._config.tableName, columnName, IndexHelper.DEFAULT_MAX_IDENTIFIER_LENGTH);
-        if (!indexNames.includes(legacyName)) {
-            return;
-        }
-        // The connector's own legacy indexes were always non-unique and single-column, anything else is an operator's
-        const legacyRow = indexRows.find(row => ObjectHelper.propertyGet(row, "indexName") === legacyName);
-        if (!Is.object(legacyRow) ||
-            ObjectHelper.propertyGet(legacyRow, "isUnique") !== false ||
-            Coerce.integer(ObjectHelper.propertyGet(legacyRow, "keyColumnCount")) !== 1) {
-            return;
-        }
-        const hasCurrent = indexNames.includes(indexName);
-        if (hasCurrent) {
-            await dbConnection.unsafe(`DROP INDEX "${legacyName}"`);
-        }
-        else {
-            await dbConnection.unsafe(`ALTER INDEX "${legacyName}" RENAME TO "${indexName}"`);
-        }
-        await nodeLogging?.log({
-            level: "info",
-            source: PostgreSqlEntityStorageConnector.CLASS_NAME,
-            ts: Date.now(),
-            message: hasCurrent ? "legacyIndexDropped" : "legacyIndexRenamed",
-            data: {
-                tableName: this._config.tableName,
-                indexName: legacyName,
-                newIndexName: indexName
+				AND am.amname = 'btree'
+			ORDER BY i.relname, k.pos`, [this._config.tableName]);
+        const indexes = {};
+        for (const row of indexRows) {
+            const indexName = ObjectHelper.propertyGet(row, "indexName");
+            const columnName = ObjectHelper.propertyGet(row, "columnName");
+            const position = Coerce.integer(ObjectHelper.propertyGet(row, "position"));
+            if (Is.stringValue(indexName) && Is.stringValue(columnName) && Is.integer(position)) {
+                indexes[indexName] ??= {
+                    columns: [],
+                    nonUnique: ObjectHelper.propertyGet(row, "isUnique") === false
+                };
+                // An expression key has no column to join to, leaving a hole which stops the
+                // index from matching any key column list beyond that position.
+                indexes[indexName].columns[position] = columnName;
             }
-        });
+        }
+        return indexes;
+    }
+    /**
+     * Check if any of the indexes already starts with the given key columns.
+     * @param indexes The indexes on the table, keyed by index name.
+     * @param keyColumns The leading key columns the index must have, in order.
+     * @returns True if an index already leads with the key columns.
+     * @internal
+     */
+    isIndexCovered(indexes, keyColumns) {
+        return Object.values(indexes).some(index => keyColumns.every((keyColumn, position) => index.columns[position] === keyColumn));
     }
     /**
      * Check if the table exists.
@@ -1085,10 +1587,14 @@ export class PostgreSqlEntityStorageConnector {
      * Build where clause arrays for a query, combining partition key and optional conditions.
      * @param conditions The optional entity conditions to include.
      * @param partitionKey The partition key value.
+     * @param tableAlias The optional table alias to qualify the columns with, needed when the
+     * clauses are used in a statement which reads from more than one table.
+     * @param startIndex The number the first placeholder takes, so clauses can be added to a
+     * statement which already holds values.
      * @returns The where clauses and bound values.
      * @internal
      */
-    buildWhereClause(conditions, partitionKey) {
+    buildWhereClause(conditions, partitionKey, tableAlias, startIndex = 1) {
         const whereClauses = [];
         const values = [];
         const finalConditions = {
@@ -1103,7 +1609,7 @@ export class PostgreSqlEntityStorageConnector {
         if (!Is.empty(conditions)) {
             finalConditions.conditions.push(conditions);
         }
-        this.buildQueryParameters("", finalConditions, whereClauses, values, 1);
+        this.buildQueryParameters("", finalConditions, whereClauses, values, startIndex, tableAlias);
         return { whereClauses, values };
     }
     /**
@@ -1113,9 +1619,10 @@ export class PostgreSqlEntityStorageConnector {
      * @param whereClauses The where clauses to use in the query.
      * @param values The values to use in the query.
      * @param valueIndex The current value index.
+     * @param tableAlias The optional table alias to qualify the columns with.
      * @internal
      */
-    buildQueryParameters(objectPath, condition, whereClauses, values, valueIndex) {
+    buildQueryParameters(objectPath, condition, whereClauses, values, valueIndex, tableAlias) {
         if (Is.undefined(condition)) {
             return;
         }
@@ -1126,7 +1633,7 @@ export class PostgreSqlEntityStorageConnector {
             const joinConditions = condition.conditions.map(c => {
                 const subWhereClauses = [];
                 const subValues = [];
-                this.buildQueryParameters(objectPath, c, subWhereClauses, subValues, valueIndex);
+                this.buildQueryParameters(objectPath, c, subWhereClauses, subValues, valueIndex, tableAlias);
                 values.push(...subValues);
                 valueIndex += subValues.length;
                 return subWhereClauses.join(" AND ");
@@ -1139,7 +1646,7 @@ export class PostgreSqlEntityStorageConnector {
             return;
         }
         const schemaProp = this._entitySchema.properties?.find(p => p.property === condition.property);
-        const comparison = this.mapComparisonOperator(objectPath, condition, schemaProp?.type, values, valueIndex);
+        const comparison = this.mapComparisonOperator(objectPath, condition, schemaProp?.type, values, valueIndex, tableAlias);
         whereClauses.push(comparison);
     }
     /**
@@ -1149,11 +1656,12 @@ export class PostgreSqlEntityStorageConnector {
      * @param type The type of the property.
      * @param values The values to use in the query.
      * @param valueIndex The current value index.
+     * @param tableAlias The optional table alias to qualify the columns with.
      * @returns The comparison expression.
      * @throws GeneralError if the comparison operator is not supported.
      * @internal
      */
-    mapComparisonOperator(objectPath, comparator, type, values, valueIndex) {
+    mapComparisonOperator(objectPath, comparator, type, values, valueIndex, tableAlias) {
         let prop = objectPath;
         if (prop.length > 0) {
             prop += ".";
@@ -1168,7 +1676,7 @@ export class PostgreSqlEntityStorageConnector {
             }
             values.push(...inValues.map(val => this.propertyToDbValue(val, type)));
             const placeholders = inValues.map((value, index) => `$${valueIndex + index}`).join(", ");
-            return `"${prop}" IN (${placeholders})`;
+            return `${this.qualifiedColumn(prop, tableAlias)} IN (${placeholders})`;
         }
         // null/undefined must use IS NULL / IS NOT NULL - never a parameterised placeholder.
         // Passing undefined through propertyToDbValue() coerces it to NaN for number fields
@@ -1184,10 +1692,10 @@ export class PostgreSqlEntityStorageConnector {
                     const jsonPath = nestedParts
                         .map((p, i, arr) => (i === arr.length - 1 ? `->> '${p}'` : `-> '${p}'`))
                         .join("");
-                    const jsonTextExpr = `("${rootProp}"::jsonb ${jsonPath})`;
+                    const jsonTextExpr = `(${this.qualifiedColumn(rootProp, tableAlias)}::jsonb ${jsonPath})`;
                     return `${jsonTextExpr} ${nullCheck}`;
                 }
-                return `"${prop}" ${nullCheck}`;
+                return `${this.qualifiedColumn(prop, tableAlias)} ${nullCheck}`;
             }
         }
         const dbValue = this.propertyToDbValue(comparator.value, type);
@@ -1200,29 +1708,40 @@ export class PostgreSqlEntityStorageConnector {
             const jsonPath = nestedParts
                 .map((p, i, arr) => (i === arr.length - 1 ? `->> '${p}'` : `-> '${p}'`))
                 .join("");
-            const jsonTextExpr = `("${rootProp}"::jsonb ${jsonPath})`;
+            const jsonTextExpr = `(${this.qualifiedColumn(rootProp, tableAlias)}::jsonb ${jsonPath})`;
             switch (comparator.comparison) {
                 case ComparisonOperator.Includes: {
                     values.pop();
-                    values.push(`%${String(comparator.value).toLowerCase()}%`);
+                    values.push(`%${String(comparator.value)}%`);
                     if (isArray) {
                         const elemPath = nestedParts
                             .map((p, i, arr) => (i === arr.length - 1 ? `->>'${p}'` : `->'${p}'`))
                             .join("");
-                        return `EXISTS (SELECT 1 FROM jsonb_array_elements("${rootProp}") elem WHERE LOWER(elem${elemPath}) ILIKE $${valueIndex})`;
+                        return `EXISTS (SELECT 1 FROM jsonb_array_elements(${this.qualifiedColumn(rootProp, tableAlias)}) elem WHERE elem${elemPath} LIKE $${valueIndex})`;
                     }
-                    return `LOWER(${jsonTextExpr}) ILIKE $${valueIndex}`;
+                    return `${jsonTextExpr} LIKE $${valueIndex}`;
                 }
                 case ComparisonOperator.NotIncludes: {
                     values.pop();
-                    values.push(`%${String(comparator.value).toLowerCase()}%`);
+                    values.push(`%${String(comparator.value)}%`);
+                    if (isArray) {
+                        const elemPath = nestedParts
+                            .map((p, i, arr) => (i === arr.length - 1 ? `->>'${p}'` : `->'${p}'`))
+                            .join("");
+                        return `NOT EXISTS (SELECT 1 FROM jsonb_array_elements(${this.qualifiedColumn(rootProp, tableAlias)}) elem WHERE elem${elemPath} LIKE $${valueIndex})`;
+                    }
+                    return `${jsonTextExpr} NOT LIKE $${valueIndex}`;
+                }
+                case ComparisonOperator.StartsWith: {
+                    values.pop();
+                    values.push(`${this.escapeLike(String(comparator.value))}%`);
                     if (isArray) {
                         const elemPath = nestedParts
                             .map((p, i, arr) => (i === arr.length - 1 ? `->>'${p}'` : `->'${p}'`))
                             .join("");
-                        return `NOT EXISTS (SELECT 1 FROM jsonb_array_elements("${rootProp}") elem WHERE LOWER(elem${elemPath}) ILIKE $${valueIndex})`;
+                        return `EXISTS (SELECT 1 FROM jsonb_array_elements(${this.qualifiedColumn(rootProp, tableAlias)}) elem WHERE elem${elemPath} LIKE $${valueIndex})`;
                     }
-                    return `LOWER(${jsonTextExpr}) NOT ILIKE $${valueIndex}`;
+                    return `${jsonTextExpr} LIKE $${valueIndex}`;
                 }
                 case ComparisonOperator.NotEquals:
                     return `${jsonTextExpr} <> $${valueIndex}`;
@@ -1241,28 +1760,28 @@ export class PostgreSqlEntityStorageConnector {
         switch (comparator.comparison) {
             case ComparisonOperator.Equals:
                 if (Is.object(comparator.value) || Is.array(comparator.value)) {
-                    return `"${prop}" = $${valueIndex}::jsonb`;
+                    return `${this.qualifiedColumn(prop, tableAlias)} = $${valueIndex}::jsonb`;
                 }
-                return `"${prop}" = $${valueIndex}`;
+                return `${this.qualifiedColumn(prop, tableAlias)} = $${valueIndex}`;
             case ComparisonOperator.NotEquals:
                 if (Is.object(comparator.value) || Is.array(comparator.value)) {
-                    return `"${prop}" != $${valueIndex}::jsonb`;
+                    return `${this.qualifiedColumn(prop, tableAlias)} != $${valueIndex}::jsonb`;
                 }
-                return `"${prop}" <> $${valueIndex}`;
+                return `${this.qualifiedColumn(prop, tableAlias)} <> $${valueIndex}`;
             case ComparisonOperator.GreaterThan:
-                return `"${prop}" > $${valueIndex}`;
+                return `${this.qualifiedColumn(prop, tableAlias)} > $${valueIndex}`;
             case ComparisonOperator.LessThan:
-                return `"${prop}" < $${valueIndex}`;
+                return `${this.qualifiedColumn(prop, tableAlias)} < $${valueIndex}`;
             case ComparisonOperator.GreaterThanOrEqual:
-                return `"${prop}" >= $${valueIndex}`;
+                return `${this.qualifiedColumn(prop, tableAlias)} >= $${valueIndex}`;
             case ComparisonOperator.LessThanOrEqual:
-                return `"${prop}" <= $${valueIndex}`;
+                return `${this.qualifiedColumn(prop, tableAlias)} <= $${valueIndex}`;
             case ComparisonOperator.Includes: {
                 if (type === EntitySchemaPropertyType.String) {
-                    return `"${prop}" ILIKE '%' || $${valueIndex} || '%'`;
+                    return `${this.qualifiedColumn(prop, tableAlias)} LIKE '%' || $${valueIndex} || '%'`;
                 }
                 if (type === EntitySchemaPropertyType.Array || type === EntitySchemaPropertyType.Object) {
-                    return `EXISTS (SELECT 1 FROM jsonb_array_elements("${prop}") elem WHERE elem @> $${valueIndex}::jsonb)`;
+                    return `EXISTS (SELECT 1 FROM jsonb_array_elements(${this.qualifiedColumn(prop, tableAlias)}) elem WHERE elem @> $${valueIndex}::jsonb)`;
                 }
                 throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "comparisonNotSupported", {
                     comparison: comparator.comparison,
@@ -1271,10 +1790,21 @@ export class PostgreSqlEntityStorageConnector {
             }
             case ComparisonOperator.NotIncludes: {
                 if (type === EntitySchemaPropertyType.String) {
-                    return `"${prop}" NOT ILIKE '%' || $${valueIndex} || '%'`;
+                    return `${this.qualifiedColumn(prop, tableAlias)} NOT LIKE '%' || $${valueIndex} || '%'`;
                 }
                 if (type === EntitySchemaPropertyType.Array || type === EntitySchemaPropertyType.Object) {
-                    return `NOT EXISTS (SELECT 1 FROM jsonb_array_elements("${prop}") elem WHERE elem @> $${valueIndex}::jsonb)`;
+                    return `NOT EXISTS (SELECT 1 FROM jsonb_array_elements(${this.qualifiedColumn(prop, tableAlias)}) elem WHERE elem @> $${valueIndex}::jsonb)`;
+                }
+                throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "comparisonNotSupported", {
+                    comparison: comparator.comparison,
+                    type
+                });
+            }
+            case ComparisonOperator.StartsWith: {
+                if (type === EntitySchemaPropertyType.String) {
+                    values.pop();
+                    values.push(`${this.escapeLike(String(comparator.value))}%`);
+                    return `${this.qualifiedColumn(prop, tableAlias)} LIKE $${valueIndex}`;
                 }
                 throw new GeneralError(PostgreSqlEntityStorageConnector.CLASS_NAME, "comparisonNotSupported", {
                     comparison: comparator.comparison,
@@ -1287,6 +1817,15 @@ export class PostgreSqlEntityStorageConnector {
                 });
         }
     }
+    /**
+     * Escape the LIKE wildcard characters in a value so they match literally.
+     * @param value The value to escape.
+     * @returns The escaped value.
+     * @internal
+     */
+    escapeLike(value) {
+        return value.replace(/[\\%_]/g, "\\$&");
+    }
     /**
      * Format a value to insert into DB.
      * @param value The value to format.
@@ -1372,6 +1911,7 @@ export class PostgreSqlEntityStorageConnector {
         props.unshift({
             property: PostgreSqlEntityStorageConnector._PARTITION_KEY,
             type: EntitySchemaPropertyType.String,
+            maxLength: PostgreSqlEntityStorageConnector._PARTITION_KEY_MAX_LENGTH,
             optional: false,
             isPrimary: true
         });