@twin.org/auditable-item-graph-service 0.10.0 → 0.10.1-next.1

package/dist/es/auditableItemGraphService.js

@@ -3,7 +3,7 @@
 import { HealthCategory, HealthStatus } from "@twin.org/api-models";
 import { AuditableItemGraphAuditMode, AuditableItemGraphContexts, AuditableItemGraphDataTypes, AuditableItemGraphMetricIds, AuditableItemGraphMetrics, AuditableItemGraphTopics, AuditableItemGraphTypes, VerifyDepth } from "@twin.org/auditable-item-graph-models";
 import { ContextIdHelper, ContextIdKeys, ContextIdStore } from "@twin.org/context";
-import { ArrayHelper, BaseError, Coerce, ComponentFactory, GeneralError, Guards, Is, JsonHelper, Mutex, NotFoundError, ObjectHelper, RandomHelper, StringHelper, Urn, Validation } from "@twin.org/core";
+import { ArrayHelper, BaseError, Coerce, ComponentFactory, Converter, GeneralError, Guards, Is, JsonHelper, Mutex, NotFoundError, ObjectHelper, RandomHelper, StringHelper, Urn, Validation } from "@twin.org/core";
 import { DataTypeHelper } from "@twin.org/data-core";
 import { JsonLdDataTypes, JsonLdHelper, JsonLdProcessor } from "@twin.org/data-json-ld";
 import { ComparisonOperator, LogicalOperator, SortDirection } from "@twin.org/entity";
@@ -32,6 +32,26 @@ export class AuditableItemGraphService {
      * The namespace for the service edge.
      */
     static NAMESPACE_EDGE = "edge";
+    /**
+     * The number of vertices returned by a query when the caller gives no limit.
+     * @internal
+     */
+    static _DEFAULT_PAGE_SIZE = 40;
+    /**
+     * The index type for the id of the vertex itself.
+     * @internal
+     */
+    static _INDEX_TYPE_VERTEX = "vertex";
+    /**
+     * The index type for vertex aliases.
+     * @internal
+     */
+    static _INDEX_TYPE_ALIAS = "alias";
+    /**
+     * The index type for vertex resource types.
+     * @internal
+     */
+    static _INDEX_TYPE_RESOURCE_TYPE = "resourceType";
     /**
      * The keys to pick when creating the proof for the stream.
      * @internal
@@ -53,6 +73,11 @@ export class AuditableItemGraphService {
      * @internal
      */
     _vertexStorage;
+    /**
+     * The entity storage for vertex indices.
+     * @internal
+     */
+    _vertexIndexStorage;
     /**
      * The entity storage for changesets.
      * @internal
@@ -80,6 +105,7 @@ export class AuditableItemGraphService {
     constructor(options) {
         this._immutableProofComponent = ComponentFactory.get(options?.immutableProofComponentType ?? "immutable-proof");
         this._vertexStorage = EntityStorageConnectorFactory.get(options?.vertexEntityStorageType ?? "auditable-item-graph-vertex");
+        this._vertexIndexStorage = EntityStorageConnectorFactory.get(options?.vertexIndexEntityStorageType ?? "auditable-item-graph-vertex-index");
         this._changesetStorage = EntityStorageConnectorFactory.get(options?.changesetEntityStorageType ?? "auditable-item-graph-changeset");
         this._eventBusComponent = ComponentFactory.getIfExists(options?.eventBusComponentType);
         this._telemetryComponent = ComponentFactory.getIfExists(options?.telemetryComponentType);
@@ -197,15 +223,11 @@ export class AuditableItemGraphService {
             await this.updateEdgeList(context, vertexModel, vertex.edges);
             // Bypass vertices keep no changeset history, so there is no baseline version to record.
             if (vertex.auditMode !== AuditableItemGraphAuditMode.Bypass) {
-                delete originalEntity.aliasIndex;
-                delete originalEntity.resourceTypeIndex;
                 await this.addChangeset(context, originalEntity, vertexModel, true, 0);
                 vertexModel.version = 0;
             }
-            await this._vertexStorage.set({
-                ...vertexModel,
-                ...this.buildIndexes(vertexModel)
-            });
+            await this._vertexStorage.set(vertexModel);
+            await this.syncVertexIndexes(vertexModel);
             await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.VerticesCreated);
             const fullId = new Urn(AuditableItemGraphService.NAMESPACE, id).toString();
             await this._eventBusComponent?.publish(AuditableItemGraphTopics.VertexCreated, { id: fullId });
@@ -254,7 +276,6 @@ export class AuditableItemGraphService {
                     userIdentity: contextIds?.[ContextIdKeys.User]
                 };
                 const auditModeTransition = this.resolveAuditModeTransition(vertexEntity, vertex.auditMode);
-                delete vertexEntity.aliasIndex;
                 const originalEntity = ObjectHelper.clone(vertexEntity);
                 const newEntity = ObjectHelper.clone(vertexEntity);
                 newEntity.annotationObject = vertex.annotationObject;
@@ -307,7 +328,6 @@ export class AuditableItemGraphService {
                     userIdentity: contextIds?.[ContextIdKeys.User]
                 };
                 const auditModeTransition = this.resolveAuditModeTransition(vertexEntity, partial.auditMode);
-                delete vertexEntity.aliasIndex;
                 const originalEntity = ObjectHelper.clone(vertexEntity);
                 const newEntity = ObjectHelper.clone(vertexEntity);
                 if (partial.annotationObject !== undefined) {
@@ -557,7 +577,7 @@ export class AuditableItemGraphService {
                     compactArrays: false
                 });
             }
-            const changesets = await this.internalGetChangesets(vertexId, {
+            const changesets = await this.internalGetChangesets(vertexId, ["patches"], {
                 maxVersion: version
             });
             let entityState = {
@@ -681,7 +701,8 @@ export class AuditableItemGraphService {
      * @param orderByDirection The direction for the order, defaults to desc.
      * @param properties The properties to return, if not provided defaults to id, created, aliases and object.
      * @param cursor The cursor to request the next chunk of entities.
-     * @param limit Limit the number of entities to return.
+     * @param limit The maximum number of entities to return, a page can contain fewer so follow
+     * the cursor until it is absent to read them all.
      * @returns The entities, which can be partial if a limited keys list was provided.
      */
     async query(options, conditions, orderBy, orderByDirection, properties, cursor, limit) {
@@ -693,61 +714,58 @@ export class AuditableItemGraphService {
                 "aliases",
                 "annotationObject"
             ];
-            const andGroups = [];
             const orderProperty = orderBy ?? "dateCreated";
             const orderDirection = orderByDirection ?? SortDirection.Descending;
             const idExact = options?.idExact ?? false;
-            if (!Is.empty(conditions)) {
-                andGroups.push(conditions);
-            }
             const idOrAlias = options?.id;
-            if (Is.stringValue(idOrAlias)) {
-                const idMode = options?.idMode ?? "both";
-                const idComparators = [];
+            const idMode = options?.idMode ?? "both";
+            const resourceTypes = options?.resourceTypes;
+            const hasIdFilter = Is.stringValue(idOrAlias);
+            const hasResourceTypes = Is.arrayValue(resourceTypes);
+            if (!propertiesToReturn.includes("id")) {
+                propertiesToReturn.unshift("id");
+            }
+            // Every id, alias and resource type match is held in the index storage, which carries
+            // the ordering dates so it can page its own matches by key set. Nothing is drained into
+            // memory, so a term matching a very large number of entries costs one page.
+            const indexPredicates = [];
+            if (hasIdFilter) {
+                const comparison = idExact ? ComparisonOperator.Equals : ComparisonOperator.Includes;
+                const term = idOrAlias.toLowerCase();
                 if (idMode === "id" || idMode === "both") {
-                    idComparators.push({
-                        property: "id",
-                        comparison: idExact ? ComparisonOperator.Equals : ComparisonOperator.Includes,
-                        value: idOrAlias
-                    });
+                    indexPredicates.push(this.indexTypeValueCondition(AuditableItemGraphService._INDEX_TYPE_VERTEX, comparison, term));
                 }
                 if (idMode === "alias" || idMode === "both") {
-                    idComparators.push({
-                        property: "aliasIndex",
-                        comparison: ComparisonOperator.Includes,
-                        value: idExact ? `||${idOrAlias.toLowerCase()}||` : idOrAlias.toLowerCase()
-                    });
-                }
-                if (idComparators.length === 1) {
-                    andGroups.push(idComparators[0]);
-                }
-                else if (idComparators.length > 1) {
-                    andGroups.push({ logicalOperator: LogicalOperator.Or, conditions: idComparators });
+                    indexPredicates.push(this.indexTypeValueCondition(AuditableItemGraphService._INDEX_TYPE_ALIAS, comparison, term));
                 }
             }
-            if (Is.arrayValue(options?.resourceTypes)) {
-                const resourceComparators = options.resourceTypes.map(rt => ({
-                    property: "resourceTypeIndex",
-                    comparison: ComparisonOperator.Includes,
-                    value: `||${rt.toLowerCase()}||`
-                }));
-                if (resourceComparators.length === 1) {
-                    andGroups.push(resourceComparators[0]);
-                }
-                else {
-                    andGroups.push({
-                        logicalOperator: LogicalOperator.Or,
-                        conditions: resourceComparators
-                    });
+            else if (hasResourceTypes) {
+                for (const resourceType of resourceTypes) {
+                    indexPredicates.push(this.indexTypeValueCondition(AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE, ComparisonOperator.Equals, resourceType.toLowerCase()));
                 }
             }
-            if (!propertiesToReturn.includes("id")) {
-                propertiesToReturn.unshift("id");
+            if (indexPredicates.length > 0) {
+                const indexPage = await this.queryIndexVertexIdPage(indexPredicates, orderProperty, orderDirection, this.decodeCursor("i", cursor), limit);
+                let pageVertexIds = indexPage.vertexIds;
+                // An OR over the index gives the union, so when an id or alias filter is combined
+                // with resource types the page is narrowed to the intersection here.
+                if (hasIdFilter && hasResourceTypes) {
+                    pageVertexIds = await this.filterVertexIdsByResourceTypes(pageVertexIds, resourceTypes.map(resourceType => resourceType.toLowerCase()));
+                }
+                const indexEntities = await this.fetchVerticesInOrder(pageVertexIds, propertiesToReturn, conditions);
+                return await this.buildVertexList(indexEntities, indexPage.hasMore
+                    ? this.encodeCursor({
+                        s: "i",
+                        d: indexPage.keySet?.d,
+                        i: indexPage.keySet?.i
+                    })
+                    : undefined);
             }
-            const finalConditions = {
-                logicalOperator: LogicalOperator.And,
-                conditions: andGroups
-            };
+            const finalConditions = Is.empty(conditions)
+                ? { logicalOperator: LogicalOperator.And, conditions: [] }
+                : conditions;
+            // With no id, alias or resource type filter there is nothing for the index to answer,
+            // so the vertex storage is queried directly and paged by its own cursor.
             const results = await this._vertexStorage.query(finalConditions, [
                 {
                     property: orderProperty,
@@ -757,28 +775,10 @@ export class AuditableItemGraphService {
                     property: "id",
                     sortDirection: SortDirection.Ascending
                 }
-            ], propertiesToReturn, cursor, limit);
-            const models = results.entities.map(e => this.vertexEntityToJsonLd(e));
-            const vertexList = {
-                "@context": [
-                    SchemaOrgContexts.Context,
-                    AuditableItemGraphContexts.Context,
-                    AuditableItemGraphContexts.ContextCommon
-                ],
-                type: [SchemaOrgTypes.ItemList, AuditableItemGraphTypes.VertexList],
-                [SchemaOrgTypes.ItemListElement]: models
-            };
-            const result = await JsonLdProcessor.compact(vertexList, vertexList["@context"], {
-                compactArrays: false
-            });
-            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.QueriesExecuted, {
-                resultCount: models.length,
-                hasMore: Is.stringValue(results.cursor)
-            });
-            return {
-                entries: result,
-                cursor: results.cursor
-            };
+            ], propertiesToReturn, this.decodeCursor("v", cursor)?.c, limit);
+            return await this.buildVertexList(results.entities, Is.stringValue(results.cursor)
+                ? this.encodeCursor({ s: "v", c: results.cursor })
+                : undefined);
         }
         catch (error) {
             throw new GeneralError(AuditableItemGraphService.CLASS_NAME, "queryingFailed", undefined, error);
@@ -851,7 +851,7 @@ export class AuditableItemGraphService {
      * @internal
      */
     async compactVertexHistory(vertexId) {
-        const changesets = await this.internalGetChangesets(vertexId);
+        const changesets = await this.internalGetChangesets(vertexId, ["proofId"]);
         for (const changeset of changesets) {
             await this._changesetStorage.remove(changeset.id);
             if (Is.stringValue(changeset.proofId)) {
@@ -878,17 +878,14 @@ export class AuditableItemGraphService {
             return;
         }
         const nextVersion = Is.empty(originalEntity.version)
-            ? (await this.internalGetChangesets(vertexId)).length
+            ? (await this.internalGetChangesets(vertexId, [])).length
             : originalEntity.version + 1;
         const patches = await this.addChangeset(context, originalEntity, newEntity, false, nextVersion);
         if (patches.length > 0) {
             newEntity.dateModified = context.now;
             newEntity.version = nextVersion;
-            const indexes = this.buildIndexes(newEntity);
-            await this._vertexStorage.set({
-                ...newEntity,
-                ...indexes
-            });
+            await this._vertexStorage.set(newEntity);
+            await this.syncVertexIndexes(newEntity);
             await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.VerticesUpdated, {
                 patchCount: patches.length
             });
@@ -917,11 +914,8 @@ export class AuditableItemGraphService {
             await this.compactVertexHistory(vertexId);
         }
         newEntity.dateModified = context.now;
-        const indexes = this.buildIndexes(newEntity);
-        await this._vertexStorage.set({
-            ...newEntity,
-            ...indexes
-        });
+        await this._vertexStorage.set(newEntity);
+        await this.syncVertexIndexes(newEntity);
         await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.VerticesUpdated, {
             patchCount: patches.length
         });
@@ -1061,15 +1055,24 @@ export class AuditableItemGraphService {
     /**
      * Fetch all changesets for a vertex in ascending date order.
      * @param vertexId The internal vertex id.
+     * @param properties The additional changeset properties to retrieve, the id and dateCreated sort
+     * keys are always included. Only the requested properties are populated on the returned
+     * entities, so the large patches payload is only transferred when a caller reads it.
      * @param options Optional filtering options.
      * @param options.before Only fetch changesets created strictly before this ISO 8601 timestamp.
      * @param options.maxVersion Only fetch changesets with version <= this value.
      * @returns All changeset entities sorted ascending by dateCreated.
      * @internal
      */
-    async internalGetChangesets(vertexId, options) {
+    async internalGetChangesets(vertexId, properties, options) {
         const all = [];
         let cursor;
+        const propertiesToReturn = ["id", "dateCreated"];
+        for (const property of properties) {
+            if (!propertiesToReturn.includes(property)) {
+                propertiesToReturn.push(property);
+            }
+        }
         const conditions = [
             { property: "vertexId", value: vertexId, comparison: ComparisonOperator.Equals }
         ];
@@ -1088,7 +1091,7 @@ export class AuditableItemGraphService {
             });
         }
         do {
-            const result = await this._changesetStorage.query({ conditions, logicalOperator: LogicalOperator.And }, [{ property: "dateCreated", sortDirection: SortDirection.Ascending }], undefined, cursor);
+            const result = await this._changesetStorage.query({ conditions, logicalOperator: LogicalOperator.And }, [{ property: "dateCreated", sortDirection: SortDirection.Ascending }], propertiesToReturn, cursor);
             all.push(...result.entities);
             cursor = result.cursor;
         } while (Is.stringValue(cursor));
@@ -1103,7 +1106,7 @@ export class AuditableItemGraphService {
      * @internal
      */
     async auditedVersions(vertexId, afterDate, beforeDate) {
-        const allChangesets = await this.internalGetChangesets(vertexId, {
+        const allChangesets = await this.internalGetChangesets(vertexId, ["version"], {
             before: beforeDate?.toISOString()
         });
         const versions = [];
@@ -1596,32 +1599,385 @@ export class AuditableItemGraphService {
             ObjectHelper.extractProperty(resource.resourceObject, ["id", "@id"], false));
     }
     /**
-     * Build the indexes for the vertex.
-     * @param vertex The vertex to build the indexes for.
-     * @returns The indexes.
+     * Synchronise the index storage with the current state of a vertex.
+     * @param vertex The vertex to synchronise the index entries for.
+     * @returns A promise that resolves when the index entries match the vertex.
+     * @internal
+     */
+    async syncVertexIndexes(vertex) {
+        const required = new Map();
+        // An entry always carries a modified date so a query ordered by it can be paged by the
+        // same key set as the creation date, falling back to the creation date for a vertex which
+        // has never been modified rather than leaving the column empty.
+        const indexDateModified = vertex.dateModified ?? vertex.dateCreated;
+        // Every vertex carries an entry for its own id, so a query matching on the vertex id can
+        // be answered from the index alone. Without it a vertex holding no aliases and no
+        // resources would have no entries at all and could never be matched here.
+        const vertexIdValue = vertex.id.toLowerCase();
+        required.set(`${AuditableItemGraphService._INDEX_TYPE_VERTEX}|${vertexIdValue}`, {
+            type: AuditableItemGraphService._INDEX_TYPE_VERTEX,
+            value: vertexIdValue
+        });
+        // Index values are case folded here, and every lookup folds its term to match, so the
+        // comparisons stay case insensitive without relying on the column collation or on the
+        // connector folding case itself. The vertex keeps the value exactly as supplied.
+        for (const alias of vertex.aliases ?? []) {
+            if (Is.empty(alias.dateDeleted) && Is.stringValue(alias.id)) {
+                const value = alias.id.toLowerCase();
+                required.set(`${AuditableItemGraphService._INDEX_TYPE_ALIAS}|${value}`, {
+                    type: AuditableItemGraphService._INDEX_TYPE_ALIAS,
+                    value
+                });
+            }
+        }
+        for (const resource of vertex.resources ?? []) {
+            const resourceType = ObjectHelper.extractProperty(resource.resourceObject, ["@type", "type"], false);
+            if (Is.stringValue(resourceType)) {
+                const value = resourceType.toLowerCase();
+                required.set(`${AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE}|${value}`, {
+                    type: AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE,
+                    value
+                });
+            }
+        }
+        const existing = await this.queryVertexIndexes({
+            property: "vertexId",
+            comparison: ComparisonOperator.Equals,
+            value: vertex.id
+        }, ["id", "type", "value", "dateCreated", "dateModified"]);
+        // Collapse by row id first. A paged read can return the same row on more than one page,
+        // and without this a retained row could be queued for removal by its own duplicate.
+        const existingById = new Map();
+        for (const entry of existing) {
+            if (Is.stringValue(entry.id)) {
+                existingById.set(entry.id, entry);
+            }
+        }
+        const retainedKeys = new Set();
+        const removeIds = [];
+        const writeEntries = [];
+        for (const [id, entry] of existingById) {
+            // Entries which are no longer required, and any duplicates of a retained entry,
+            // are removed so exactly one entry remains per index value.
+            const key = `${entry.type}|${entry.value}`;
+            const requiredEntry = required.get(key);
+            if (!Is.empty(requiredEntry) && !retainedKeys.has(key)) {
+                retainedKeys.add(key);
+                if (entry.dateCreated !== vertex.dateCreated || entry.dateModified !== indexDateModified) {
+                    // Keep the row id so the copied dates are corrected in place rather than duplicated.
+                    writeEntries.push({
+                        id,
+                        vertexId: vertex.id,
+                        type: requiredEntry.type,
+                        value: requiredEntry.value,
+                        dateCreated: vertex.dateCreated,
+                        dateModified: indexDateModified
+                    });
+                }
+            }
+            else {
+                removeIds.push(id);
+            }
+        }
+        for (const [key, entry] of required) {
+            if (!retainedKeys.has(key)) {
+                writeEntries.push({
+                    id: Converter.bytesToHex(RandomHelper.generate(16)),
+                    vertexId: vertex.id,
+                    type: entry.type,
+                    value: entry.value,
+                    dateCreated: vertex.dateCreated,
+                    dateModified: indexDateModified
+                });
+            }
+        }
+        if (removeIds.length > 0) {
+            await this._vertexIndexStorage.removeBatch(removeIds);
+        }
+        if (writeEntries.length > 0) {
+            await this._vertexIndexStorage.setBatch(writeEntries);
+        }
+    }
+    /**
+     * Build the JSON-LD vertex list returned by query, and record the query metric.
+     * @param entities The vertex entities for the page.
+     * @param cursor The already encoded cursor for the next page.
+     * @returns The compacted vertex list and the cursor.
+     * @internal
+     */
+    async buildVertexList(entities, cursor) {
+        const models = entities.map(e => this.vertexEntityToJsonLd(e));
+        const vertexList = {
+            "@context": [
+                SchemaOrgContexts.Context,
+                AuditableItemGraphContexts.Context,
+                AuditableItemGraphContexts.ContextCommon
+            ],
+            type: [SchemaOrgTypes.ItemList, AuditableItemGraphTypes.VertexList],
+            [SchemaOrgTypes.ItemListElement]: models
+        };
+        const result = await JsonLdProcessor.compact(vertexList, vertexList["@context"], {
+            compactArrays: false
+        });
+        await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.QueriesExecuted, {
+            resultCount: models.length,
+            hasMore: Is.stringValue(cursor)
+        });
+        return {
+            entries: result,
+            cursor
+        };
+    }
+    /**
+     * Encode a storage cursor into the opaque cursor handed back to callers.
+     * Every cursor this service returns is in this format, so a cursor which does not decode
+     * is invalid rather than a cursor from somewhere else.
+     * @param payload The position to encode.
+     * @param payload.s The strategy the position belongs to, i for index paged, v for vertex paged.
+     * @param payload.d The ordering date of the last vertex returned, index strategy only.
+     * @param payload.i The id of the last vertex returned, index strategy only.
+     * @param payload.c The storage connector cursor, vertex strategy only.
+     * @returns The encoded cursor, or undefined when there is no further page.
+     * @internal
+     */
+    encodeCursor(payload) {
+        return Converter.bytesToBase64(ObjectHelper.toBytes(payload));
+    }
+    /**
+     * Decode a cursor produced by encodeCursor.
+     * @param strategy The strategy the current query is using.
+     * @param cursor The cursor supplied by the caller.
+     * @returns The decoded payload, or undefined when no cursor was supplied.
+     * @throws GeneralError If the cursor is malformed or belongs to a different strategy.
      * @internal
      */
-    buildIndexes(vertex) {
-        const aliasIndex = vertex.aliases
-            ?.filter(a => Is.empty(a.dateDeleted))
-            .map(a => a.id)
-            .join("||")
-            .toLowerCase();
-        const resourceTypes = [];
-        if (Is.arrayValue(vertex.resources)) {
-            for (const resource of vertex.resources) {
-                const resourceType = ObjectHelper.extractProperty(resource.resourceObject, ["@type", "type"], false);
-                if (Is.stringValue(resourceType) && !resourceTypes.includes(resourceType)) {
-                    resourceTypes.push(resourceType);
+    decodeCursor(strategy, cursor) {
+        if (!Is.stringValue(cursor)) {
+            return undefined;
+        }
+        let decoded;
+        if (Is.stringBase64(cursor)) {
+            try {
+                decoded = ObjectHelper.fromBytes(Converter.base64ToBytes(cursor));
+            }
+            catch (error) {
+                // Base64 which is not an encoded cursor is still only an invalid cursor, so every
+                // malformed cursor reports the same failure rather than leaking a parse error.
+                throw new GeneralError(AuditableItemGraphService.CLASS_NAME, "invalidCursor", undefined, error);
+            }
+        }
+        // A cursor from a different strategy cannot be applied to this query, and silently
+        // restarting from the first page would loop a caller which keeps following the cursor.
+        const populated = decoded?.s === strategy &&
+            (strategy === "v"
+                ? Is.stringValue(decoded.c)
+                : Is.stringValue(decoded.d) && Is.stringValue(decoded.i));
+        if (!populated) {
+            throw new GeneralError(AuditableItemGraphService.CLASS_NAME, "invalidCursor");
+        }
+        return decoded;
+    }
+    /**
+     * Build the condition matching one index type and value.
+     * @param type The index type to match.
+     * @param comparison The comparison to apply to the value.
+     * @param value The index value to match, already lower cased.
+     * @returns The condition.
+     * @internal
+     */
+    indexTypeValueCondition(type, comparison, value) {
+        return {
+            logicalOperator: LogicalOperator.And,
+            conditions: [
+                { property: "type", comparison: ComparisonOperator.Equals, value: type },
+                { property: "value", comparison, value }
+            ]
+        };
+    }
+    /**
+     * Build the condition which resumes an index scan after a position.
+     * Every entry for a vertex carries the same date and vertex id, so a position on that pair
+     * steps past all the remaining entries of the vertex it names.
+     * @param orderProperty The index date property being ordered by.
+     * @param orderDirection The direction being ordered in.
+     * @param lastDate The date of the last vertex returned.
+     * @param lastVertexId The id of the last vertex returned.
+     * @returns The condition.
+     * @internal
+     */
+    indexKeySetCondition(orderProperty, orderDirection, lastDate, lastVertexId) {
+        const dateComparison = orderDirection === SortDirection.Ascending
+            ? ComparisonOperator.GreaterThan
+            : ComparisonOperator.LessThan;
+        return {
+            logicalOperator: LogicalOperator.Or,
+            conditions: [
+                { property: orderProperty, comparison: dateComparison, value: lastDate },
+                {
+                    logicalOperator: LogicalOperator.And,
+                    conditions: [
+                        { property: orderProperty, comparison: ComparisonOperator.Equals, value: lastDate },
+                        {
+                            property: "vertexId",
+                            comparison: ComparisonOperator.GreaterThan,
+                            value: lastVertexId
+                        }
+                    ]
+                }
+            ]
+        };
+    }
+    /**
+     * Read one page of distinct vertex ids from the index storage using a key set, so a broad
+     * match is never drained into memory however many entries it has.
+     * @param predicates The type and value conditions to match, combined with OR.
+     * @param orderProperty The index date property to order by.
+     * @param orderDirection The direction to order in.
+     * @param keySet The position returned by the previous page.
+     * @param keySet.d The ordering date of the last vertex returned.
+     * @param keySet.i The id of the last vertex returned.
+     * @param limit The maximum number of distinct vertices to return.
+     * @returns The vertex ids in order, the position to resume from, and whether more remain.
+     * @internal
+     */
+    async queryIndexVertexIdPage(predicates, orderProperty, orderDirection, keySet, limit) {
+        const pageSize = limit ?? AuditableItemGraphService._DEFAULT_PAGE_SIZE;
+        const match = predicates.length === 1
+            ? predicates[0]
+            : { logicalOperator: LogicalOperator.Or, conditions: predicates };
+        const vertexIds = [];
+        const positions = [];
+        const seen = new Set();
+        let scan = keySet;
+        let exhausted = false;
+        // One extra vertex is collected to tell whether a further page exists. Each pass either
+        // adds a vertex or exhausts the match, so this runs at most pageSize + 1 times.
+        while (vertexIds.length <= pageSize && !exhausted) {
+            const conditions = [match];
+            if (Is.stringValue(scan?.d) && Is.stringValue(scan?.i)) {
+                conditions.push(this.indexKeySetCondition(orderProperty, orderDirection, scan.d, scan.i));
+            }
+            const results = await this._vertexIndexStorage.query({ logicalOperator: LogicalOperator.And, conditions }, [
+                { property: orderProperty, sortDirection: orderDirection },
+                { property: "vertexId", sortDirection: SortDirection.Ascending }
+            ], ["vertexId", orderProperty], undefined, pageSize + 1);
+            if (results.entities.length === 0) {
+                exhausted = true;
+            }
+            for (const entity of results.entities) {
+                const entityDate = entity[orderProperty];
+                if (Is.stringValue(entity.vertexId) && Is.stringValue(entityDate)) {
+                    const position = { d: entityDate, i: entity.vertexId };
+                    scan = position;
+                    if (!seen.has(entity.vertexId) && vertexIds.length <= pageSize) {
+                        seen.add(entity.vertexId);
+                        vertexIds.push(entity.vertexId);
+                        positions.push(position);
+                    }
                 }
             }
         }
-        const resourceTypeIndex = resourceTypes.join("||").toLowerCase();
+        let hasMore = false;
+        if (vertexIds.length > pageSize) {
+            vertexIds.length = pageSize;
+            positions.length = pageSize;
+            hasMore = true;
+        }
         return {
-            aliasIndex: Is.stringValue(aliasIndex) ? `||${aliasIndex}||` : undefined,
-            resourceTypeIndex: Is.stringValue(resourceTypeIndex) ? `||${resourceTypeIndex}||` : undefined
+            vertexIds,
+            keySet: positions.length > 0 ? positions[positions.length - 1] : undefined,
+            hasMore
         };
     }
+    /**
+     * Narrow a page of vertex ids to those which also carry one of the resource types.
+     * @param vertexIds The candidate vertex ids.
+     * @param resourceTypes The resource types to match, already lower cased.
+     * @returns The matching vertex ids, keeping the incoming order.
+     * @internal
+     */
+    async filterVertexIdsByResourceTypes(vertexIds, resourceTypes) {
+        if (vertexIds.length === 0) {
+            return [];
+        }
+        const results = await this._vertexIndexStorage.query({
+            logicalOperator: LogicalOperator.And,
+            conditions: [
+                {
+                    property: "type",
+                    comparison: ComparisonOperator.Equals,
+                    value: AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE
+                },
+                { property: "value", comparison: ComparisonOperator.In, value: resourceTypes },
+                { property: "vertexId", comparison: ComparisonOperator.In, value: vertexIds }
+            ]
+        }, undefined, ["vertexId"], undefined, vertexIds.length * resourceTypes.length);
+        const matched = new Set();
+        for (const entity of results.entities) {
+            if (Is.stringValue(entity.vertexId)) {
+                matched.add(entity.vertexId);
+            }
+        }
+        return vertexIds.filter(vertexId => matched.has(vertexId));
+    }
+    /**
+     * Read the vertices for a page of ids, keeping the order the ids were given in.
+     * @param vertexIds The vertex ids to read.
+     * @param propertiesToReturn The vertex properties to return.
+     * @param conditions Additional conditions the vertices must match.
+     * @returns The vertices which exist and match, in the order of the ids.
+     * @internal
+     */
+    async fetchVerticesInOrder(vertexIds, propertiesToReturn, conditions) {
+        if (vertexIds.length === 0) {
+            return [];
+        }
+        const idCondition = {
+            property: "id",
+            comparison: ComparisonOperator.In,
+            value: vertexIds
+        };
+        const results = await this._vertexStorage.query(Is.empty(conditions)
+            ? idCondition
+            : {
+                logicalOperator: LogicalOperator.And,
+                conditions: [idCondition, conditions]
+            }, undefined, propertiesToReturn, undefined, vertexIds.length);
+        // An In lookup has no guaranteed order, so the index ordering is reapplied here.
+        const vertexById = new Map();
+        for (const entity of results.entities) {
+            if (Is.stringValue(entity.id)) {
+                vertexById.set(entity.id, entity);
+            }
+        }
+        const ordered = [];
+        for (const vertexId of vertexIds) {
+            const entity = vertexById.get(vertexId);
+            if (!Is.empty(entity)) {
+                ordered.push(entity);
+            }
+        }
+        return ordered;
+    }
+    /**
+     * Query all the pages of the vertex index storage which match the conditions.
+     * @param conditions The conditions to match.
+     * @param properties The properties to return.
+     * @returns The matching index entries.
+     * @internal
+     */
+    async queryVertexIndexes(conditions, properties) {
+        const entities = [];
+        let cursor;
+        do {
+            // Paging needs a deterministic order or a row can be repeated or skipped between
+            // pages, the entity declares no default sort so the primary key is used.
+            const results = await this._vertexIndexStorage.query(conditions, [{ property: "id", sortDirection: SortDirection.Ascending }], properties, cursor);
+            entities.push(...results.entities);
+            cursor = results.cursor;
+        } while (Is.stringValue(cursor));
+        return entities;
+    }
     /**
      * Find vertices with matching aliases.
      * @param vertexId The id of the vertex to exclude from the search.
@@ -1630,22 +1986,27 @@ export class AuditableItemGraphService {
      * @internal
      */
     async findMatchingVertices(vertexId, aliasId) {
-        const results = await this._vertexStorage.query({
+        const entities = await this.queryVertexIndexes({
+            logicalOperator: LogicalOperator.And,
             conditions: [
                 {
-                    property: "aliasIndex",
-                    comparison: ComparisonOperator.Includes,
-                    value: `||${aliasId.toLowerCase()}||`
+                    property: "type",
+                    comparison: ComparisonOperator.Equals,
+                    value: AuditableItemGraphService._INDEX_TYPE_ALIAS
                 },
                 {
-                    property: "id",
-                    value: vertexId,
-                    comparison: ComparisonOperator.NotEquals
+                    property: "value",
+                    comparison: ComparisonOperator.Equals,
+                    value: aliasId.toLowerCase()
+                },
+                {
+                    property: "vertexId",
+                    comparison: ComparisonOperator.NotEquals,
+                    value: vertexId
                 }
-            ],
-            logicalOperator: LogicalOperator.And
-        });
-        return results.entities.length > 0;
+            ]
+        }, ["vertexId"]);
+        return entities.length > 0;
     }
     /**
      * Whether an incoming edge matches a stored edge id.