@twin.org/auditable-item-graph-service 0.10.0 → 0.10.1-next.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/es/auditableItemGraphService.js +299 -120
  2. package/dist/es/auditableItemGraphService.js.map +1 -1
  3. package/dist/es/entities/auditableItemGraphAlias.js +2 -2
  4. package/dist/es/entities/auditableItemGraphAlias.js.map +1 -1
  5. package/dist/es/entities/auditableItemGraphChangeset.js +15 -5
  6. package/dist/es/entities/auditableItemGraphChangeset.js.map +1 -1
  7. package/dist/es/entities/auditableItemGraphEdge.js +2 -2
  8. package/dist/es/entities/auditableItemGraphEdge.js.map +1 -1
  9. package/dist/es/entities/auditableItemGraphPatch.js +3 -3
  10. package/dist/es/entities/auditableItemGraphPatch.js.map +1 -1
  11. package/dist/es/entities/auditableItemGraphResource.js +1 -1
  12. package/dist/es/entities/auditableItemGraphResource.js.map +1 -1
  13. package/dist/es/entities/auditableItemGraphVertex.js +4 -20
  14. package/dist/es/entities/auditableItemGraphVertex.js.map +1 -1
  15. package/dist/es/entities/auditableItemGraphVertexIndex.js +83 -0
  16. package/dist/es/entities/auditableItemGraphVertexIndex.js.map +1 -0
  17. package/dist/es/entities/auditableItemGraphVertexV1.js +113 -0
  18. package/dist/es/entities/auditableItemGraphVertexV1.js.map +1 -0
  19. package/dist/es/index.js +2 -0
  20. package/dist/es/index.js.map +1 -1
  21. package/dist/es/models/IAuditableItemGraphServiceConstructorOptions.js.map +1 -1
  22. package/dist/es/schema.js +4 -0
  23. package/dist/es/schema.js.map +1 -1
  24. package/dist/types/auditableItemGraphService.d.ts +3 -2
  25. package/dist/types/entities/auditableItemGraphVertex.d.ts +0 -8
  26. package/dist/types/entities/auditableItemGraphVertexIndex.d.ts +31 -0
  27. package/dist/types/entities/auditableItemGraphVertexV1.d.ts +58 -0
  28. package/dist/types/index.d.ts +2 -0
  29. package/dist/types/models/IAuditableItemGraphServiceConstructorOptions.d.ts +5 -0
  30. package/docs/changelog.md +86 -0
  31. package/docs/open-api/spec.json +1 -1
  32. package/docs/reference/classes/AuditableItemGraphService.md +2 -1
  33. package/docs/reference/classes/AuditableItemGraphVertex.md +0 -16
  34. package/docs/reference/classes/AuditableItemGraphVertexIndex.md +63 -0
  35. package/docs/reference/classes/AuditableItemGraphVertexV1.md +109 -0
  36. package/docs/reference/index.md +2 -0
  37. package/docs/reference/interfaces/IAuditableItemGraphServiceConstructorOptions.md +14 -0
  38. package/package.json +16 -16
@@ -3,7 +3,7 @@
3
3
  import { HealthCategory, HealthStatus } from "@twin.org/api-models";
4
4
  import { AuditableItemGraphAuditMode, AuditableItemGraphContexts, AuditableItemGraphDataTypes, AuditableItemGraphMetricIds, AuditableItemGraphMetrics, AuditableItemGraphTopics, AuditableItemGraphTypes, VerifyDepth } from "@twin.org/auditable-item-graph-models";
5
5
  import { ContextIdHelper, ContextIdKeys, ContextIdStore } from "@twin.org/context";
6
- import { ArrayHelper, BaseError, Coerce, ComponentFactory, GeneralError, Guards, Is, JsonHelper, Mutex, NotFoundError, ObjectHelper, RandomHelper, StringHelper, Urn, Validation } from "@twin.org/core";
6
+ import { ArrayHelper, BaseError, Coerce, ComponentFactory, Converter, GeneralError, Guards, Is, JsonHelper, Mutex, NotFoundError, ObjectHelper, RandomHelper, StringHelper, Urn, Validation } from "@twin.org/core";
7
7
  import { DataTypeHelper } from "@twin.org/data-core";
8
8
  import { JsonLdDataTypes, JsonLdHelper, JsonLdProcessor } from "@twin.org/data-json-ld";
9
9
  import { ComparisonOperator, LogicalOperator, SortDirection } from "@twin.org/entity";
@@ -32,6 +32,21 @@ export class AuditableItemGraphService {
32
32
  * The namespace for the service edge.
33
33
  */
34
34
  static NAMESPACE_EDGE = "edge";
35
+ /**
36
+ * The index type for the id of the vertex itself.
37
+ * @internal
38
+ */
39
+ static _INDEX_TYPE_VERTEX = "vertex";
40
+ /**
41
+ * The index type for vertex aliases.
42
+ * @internal
43
+ */
44
+ static _INDEX_TYPE_ALIAS = "alias";
45
+ /**
46
+ * The index type for vertex resource types.
47
+ * @internal
48
+ */
49
+ static _INDEX_TYPE_RESOURCE_TYPE = "resourceType";
35
50
  /**
36
51
  * The keys to pick when creating the proof for the stream.
37
52
  * @internal
@@ -53,6 +68,11 @@ export class AuditableItemGraphService {
53
68
  * @internal
54
69
  */
55
70
  _vertexStorage;
71
+ /**
72
+ * The entity storage for vertex indices.
73
+ * @internal
74
+ */
75
+ _vertexIndexStorage;
56
76
  /**
57
77
  * The entity storage for changesets.
58
78
  * @internal
@@ -80,6 +100,7 @@ export class AuditableItemGraphService {
80
100
  constructor(options) {
81
101
  this._immutableProofComponent = ComponentFactory.get(options?.immutableProofComponentType ?? "immutable-proof");
82
102
  this._vertexStorage = EntityStorageConnectorFactory.get(options?.vertexEntityStorageType ?? "auditable-item-graph-vertex");
103
+ this._vertexIndexStorage = EntityStorageConnectorFactory.get(options?.vertexIndexEntityStorageType ?? "auditable-item-graph-vertex-index");
83
104
  this._changesetStorage = EntityStorageConnectorFactory.get(options?.changesetEntityStorageType ?? "auditable-item-graph-changeset");
84
105
  this._eventBusComponent = ComponentFactory.getIfExists(options?.eventBusComponentType);
85
106
  this._telemetryComponent = ComponentFactory.getIfExists(options?.telemetryComponentType);
@@ -197,15 +218,11 @@ export class AuditableItemGraphService {
197
218
  await this.updateEdgeList(context, vertexModel, vertex.edges);
198
219
  // Bypass vertices keep no changeset history, so there is no baseline version to record.
199
220
  if (vertex.auditMode !== AuditableItemGraphAuditMode.Bypass) {
200
- delete originalEntity.aliasIndex;
201
- delete originalEntity.resourceTypeIndex;
202
221
  await this.addChangeset(context, originalEntity, vertexModel, true, 0);
203
222
  vertexModel.version = 0;
204
223
  }
205
- await this._vertexStorage.set({
206
- ...vertexModel,
207
- ...this.buildIndexes(vertexModel)
208
- });
224
+ await this._vertexStorage.set(vertexModel);
225
+ await this.syncVertexIndexes(vertexModel);
209
226
  await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.VerticesCreated);
210
227
  const fullId = new Urn(AuditableItemGraphService.NAMESPACE, id).toString();
211
228
  await this._eventBusComponent?.publish(AuditableItemGraphTopics.VertexCreated, { id: fullId });
@@ -254,7 +271,6 @@ export class AuditableItemGraphService {
254
271
  userIdentity: contextIds?.[ContextIdKeys.User]
255
272
  };
256
273
  const auditModeTransition = this.resolveAuditModeTransition(vertexEntity, vertex.auditMode);
257
- delete vertexEntity.aliasIndex;
258
274
  const originalEntity = ObjectHelper.clone(vertexEntity);
259
275
  const newEntity = ObjectHelper.clone(vertexEntity);
260
276
  newEntity.annotationObject = vertex.annotationObject;
@@ -307,7 +323,6 @@ export class AuditableItemGraphService {
307
323
  userIdentity: contextIds?.[ContextIdKeys.User]
308
324
  };
309
325
  const auditModeTransition = this.resolveAuditModeTransition(vertexEntity, partial.auditMode);
310
- delete vertexEntity.aliasIndex;
311
326
  const originalEntity = ObjectHelper.clone(vertexEntity);
312
327
  const newEntity = ObjectHelper.clone(vertexEntity);
313
328
  if (partial.annotationObject !== undefined) {
@@ -557,7 +572,7 @@ export class AuditableItemGraphService {
557
572
  compactArrays: false
558
573
  });
559
574
  }
560
- const changesets = await this.internalGetChangesets(vertexId, {
575
+ const changesets = await this.internalGetChangesets(vertexId, ["patches"], {
561
576
  maxVersion: version
562
577
  });
563
578
  let entityState = {
@@ -681,7 +696,8 @@ export class AuditableItemGraphService {
681
696
  * @param orderByDirection The direction for the order, defaults to desc.
682
697
  * @param properties The properties to return, if not provided defaults to id, created, aliases and object.
683
698
  * @param cursor The cursor to request the next chunk of entities.
684
- * @param limit Limit the number of entities to return.
699
+ * @param limit The maximum number of entities to return, a page can contain fewer so follow
700
+ * the cursor until it is absent to read them all.
685
701
  * @returns The entities, which can be partial if a limited keys list was provided.
686
702
  */
687
703
  async query(options, conditions, orderBy, orderByDirection, properties, cursor, limit) {
@@ -693,61 +709,45 @@ export class AuditableItemGraphService {
693
709
  "aliases",
694
710
  "annotationObject"
695
711
  ];
696
- const andGroups = [];
697
712
  const orderProperty = orderBy ?? "dateCreated";
698
713
  const orderDirection = orderByDirection ?? SortDirection.Descending;
699
714
  const idExact = options?.idExact ?? false;
700
- if (!Is.empty(conditions)) {
701
- andGroups.push(conditions);
702
- }
703
715
  const idOrAlias = options?.id;
704
- if (Is.stringValue(idOrAlias)) {
705
- const idMode = options?.idMode ?? "both";
706
- const idComparators = [];
716
+ const idMode = options?.idMode ?? "both";
717
+ const resourceTypes = options?.resourceTypes;
718
+ const hasIdFilter = Is.stringValue(idOrAlias);
719
+ const hasResourceTypes = Is.arrayValue(resourceTypes);
720
+ if (!propertiesToReturn.includes("id")) {
721
+ propertiesToReturn.unshift("id");
722
+ }
723
+ // Every id, alias and resource type match is held in the index storage. Each filter is
724
+ // one dimension, satisfied by any one of the entries of a vertex, and a vertex has to
725
+ // satisfy every dimension it was given.
726
+ const indexDimensions = [];
727
+ if (hasIdFilter) {
728
+ const comparison = idExact ? ComparisonOperator.Equals : ComparisonOperator.Includes;
729
+ const term = idOrAlias.toLowerCase();
730
+ const idPredicates = [];
707
731
  if (idMode === "id" || idMode === "both") {
708
- idComparators.push({
709
- property: "id",
710
- comparison: idExact ? ComparisonOperator.Equals : ComparisonOperator.Includes,
711
- value: idOrAlias
712
- });
732
+ idPredicates.push(this.indexTypeValueCondition(AuditableItemGraphService._INDEX_TYPE_VERTEX, comparison, term));
713
733
  }
714
734
  if (idMode === "alias" || idMode === "both") {
715
- idComparators.push({
716
- property: "aliasIndex",
717
- comparison: ComparisonOperator.Includes,
718
- value: idExact ? `||${idOrAlias.toLowerCase()}||` : idOrAlias.toLowerCase()
719
- });
720
- }
721
- if (idComparators.length === 1) {
722
- andGroups.push(idComparators[0]);
723
- }
724
- else if (idComparators.length > 1) {
725
- andGroups.push({ logicalOperator: LogicalOperator.Or, conditions: idComparators });
735
+ idPredicates.push(this.indexTypeValueCondition(AuditableItemGraphService._INDEX_TYPE_ALIAS, comparison, term));
726
736
  }
737
+ indexDimensions.push(this.anyOfConditions(idPredicates));
727
738
  }
728
- if (Is.arrayValue(options?.resourceTypes)) {
729
- const resourceComparators = options.resourceTypes.map(rt => ({
730
- property: "resourceTypeIndex",
731
- comparison: ComparisonOperator.Includes,
732
- value: `||${rt.toLowerCase()}||`
733
- }));
734
- if (resourceComparators.length === 1) {
735
- andGroups.push(resourceComparators[0]);
736
- }
737
- else {
738
- andGroups.push({
739
- logicalOperator: LogicalOperator.Or,
740
- conditions: resourceComparators
741
- });
742
- }
739
+ if (hasResourceTypes) {
740
+ indexDimensions.push(this.indexTypeValueCondition(AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE, ComparisonOperator.In, resourceTypes.map(resourceType => resourceType.toLowerCase())));
743
741
  }
744
- if (!propertiesToReturn.includes("id")) {
745
- propertiesToReturn.unshift("id");
742
+ if (indexDimensions.length > 0) {
743
+ const indexPage = await this.queryIndexVertices(indexDimensions, orderProperty, orderDirection, propertiesToReturn, conditions, cursor, limit);
744
+ return await this.buildVertexList(indexPage.entities, indexPage.cursor);
746
745
  }
747
- const finalConditions = {
748
- logicalOperator: LogicalOperator.And,
749
- conditions: andGroups
750
- };
746
+ const finalConditions = Is.empty(conditions)
747
+ ? { logicalOperator: LogicalOperator.And, conditions: [] }
748
+ : conditions;
749
+ // With no id, alias or resource type filter there is nothing for the index to answer,
750
+ // so the vertex storage is queried directly and paged by its own cursor.
751
751
  const results = await this._vertexStorage.query(finalConditions, [
752
752
  {
753
753
  property: orderProperty,
@@ -758,27 +758,7 @@ export class AuditableItemGraphService {
758
758
  sortDirection: SortDirection.Ascending
759
759
  }
760
760
  ], propertiesToReturn, cursor, limit);
761
- const models = results.entities.map(e => this.vertexEntityToJsonLd(e));
762
- const vertexList = {
763
- "@context": [
764
- SchemaOrgContexts.Context,
765
- AuditableItemGraphContexts.Context,
766
- AuditableItemGraphContexts.ContextCommon
767
- ],
768
- type: [SchemaOrgTypes.ItemList, AuditableItemGraphTypes.VertexList],
769
- [SchemaOrgTypes.ItemListElement]: models
770
- };
771
- const result = await JsonLdProcessor.compact(vertexList, vertexList["@context"], {
772
- compactArrays: false
773
- });
774
- await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.QueriesExecuted, {
775
- resultCount: models.length,
776
- hasMore: Is.stringValue(results.cursor)
777
- });
778
- return {
779
- entries: result,
780
- cursor: results.cursor
781
- };
761
+ return await this.buildVertexList(results.entities, results.cursor);
782
762
  }
783
763
  catch (error) {
784
764
  throw new GeneralError(AuditableItemGraphService.CLASS_NAME, "queryingFailed", undefined, error);
@@ -851,7 +831,7 @@ export class AuditableItemGraphService {
851
831
  * @internal
852
832
  */
853
833
  async compactVertexHistory(vertexId) {
854
- const changesets = await this.internalGetChangesets(vertexId);
834
+ const changesets = await this.internalGetChangesets(vertexId, ["proofId"]);
855
835
  for (const changeset of changesets) {
856
836
  await this._changesetStorage.remove(changeset.id);
857
837
  if (Is.stringValue(changeset.proofId)) {
@@ -878,17 +858,14 @@ export class AuditableItemGraphService {
878
858
  return;
879
859
  }
880
860
  const nextVersion = Is.empty(originalEntity.version)
881
- ? (await this.internalGetChangesets(vertexId)).length
861
+ ? (await this.internalGetChangesets(vertexId, [])).length
882
862
  : originalEntity.version + 1;
883
863
  const patches = await this.addChangeset(context, originalEntity, newEntity, false, nextVersion);
884
864
  if (patches.length > 0) {
885
865
  newEntity.dateModified = context.now;
886
866
  newEntity.version = nextVersion;
887
- const indexes = this.buildIndexes(newEntity);
888
- await this._vertexStorage.set({
889
- ...newEntity,
890
- ...indexes
891
- });
867
+ await this._vertexStorage.set(newEntity);
868
+ await this.syncVertexIndexes(newEntity);
892
869
  await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.VerticesUpdated, {
893
870
  patchCount: patches.length
894
871
  });
@@ -917,11 +894,8 @@ export class AuditableItemGraphService {
917
894
  await this.compactVertexHistory(vertexId);
918
895
  }
919
896
  newEntity.dateModified = context.now;
920
- const indexes = this.buildIndexes(newEntity);
921
- await this._vertexStorage.set({
922
- ...newEntity,
923
- ...indexes
924
- });
897
+ await this._vertexStorage.set(newEntity);
898
+ await this.syncVertexIndexes(newEntity);
925
899
  await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.VerticesUpdated, {
926
900
  patchCount: patches.length
927
901
  });
@@ -1061,15 +1035,24 @@ export class AuditableItemGraphService {
1061
1035
  /**
1062
1036
  * Fetch all changesets for a vertex in ascending date order.
1063
1037
  * @param vertexId The internal vertex id.
1038
+ * @param properties The additional changeset properties to retrieve, the id and dateCreated sort
1039
+ * keys are always included. Only the requested properties are populated on the returned
1040
+ * entities, so the large patches payload is only transferred when a caller reads it.
1064
1041
  * @param options Optional filtering options.
1065
1042
  * @param options.before Only fetch changesets created strictly before this ISO 8601 timestamp.
1066
1043
  * @param options.maxVersion Only fetch changesets with version <= this value.
1067
1044
  * @returns All changeset entities sorted ascending by dateCreated.
1068
1045
  * @internal
1069
1046
  */
1070
- async internalGetChangesets(vertexId, options) {
1047
+ async internalGetChangesets(vertexId, properties, options) {
1071
1048
  const all = [];
1072
1049
  let cursor;
1050
+ const propertiesToReturn = ["id", "dateCreated"];
1051
+ for (const property of properties) {
1052
+ if (!propertiesToReturn.includes(property)) {
1053
+ propertiesToReturn.push(property);
1054
+ }
1055
+ }
1073
1056
  const conditions = [
1074
1057
  { property: "vertexId", value: vertexId, comparison: ComparisonOperator.Equals }
1075
1058
  ];
@@ -1088,7 +1071,7 @@ export class AuditableItemGraphService {
1088
1071
  });
1089
1072
  }
1090
1073
  do {
1091
- const result = await this._changesetStorage.query({ conditions, logicalOperator: LogicalOperator.And }, [{ property: "dateCreated", sortDirection: SortDirection.Ascending }], undefined, cursor);
1074
+ const result = await this._changesetStorage.query({ conditions, logicalOperator: LogicalOperator.And }, [{ property: "dateCreated", sortDirection: SortDirection.Ascending }], propertiesToReturn, cursor);
1092
1075
  all.push(...result.entities);
1093
1076
  cursor = result.cursor;
1094
1077
  } while (Is.stringValue(cursor));
@@ -1103,7 +1086,7 @@ export class AuditableItemGraphService {
1103
1086
  * @internal
1104
1087
  */
1105
1088
  async auditedVersions(vertexId, afterDate, beforeDate) {
1106
- const allChangesets = await this.internalGetChangesets(vertexId, {
1089
+ const allChangesets = await this.internalGetChangesets(vertexId, ["version"], {
1107
1090
  before: beforeDate?.toISOString()
1108
1091
  });
1109
1092
  const versions = [];
@@ -1596,32 +1579,223 @@ export class AuditableItemGraphService {
1596
1579
  ObjectHelper.extractProperty(resource.resourceObject, ["id", "@id"], false));
1597
1580
  }
1598
1581
  /**
1599
- * Build the indexes for the vertex.
1600
- * @param vertex The vertex to build the indexes for.
1601
- * @returns The indexes.
1582
+ * Synchronise the index storage with the current state of a vertex.
1583
+ * @param vertex The vertex to synchronise the index entries for.
1584
+ * @returns A promise that resolves when the index entries match the vertex.
1602
1585
  * @internal
1603
1586
  */
1604
- buildIndexes(vertex) {
1605
- const aliasIndex = vertex.aliases
1606
- ?.filter(a => Is.empty(a.dateDeleted))
1607
- .map(a => a.id)
1608
- .join("||")
1609
- .toLowerCase();
1610
- const resourceTypes = [];
1611
- if (Is.arrayValue(vertex.resources)) {
1612
- for (const resource of vertex.resources) {
1613
- const resourceType = ObjectHelper.extractProperty(resource.resourceObject, ["@type", "type"], false);
1614
- if (Is.stringValue(resourceType) && !resourceTypes.includes(resourceType)) {
1615
- resourceTypes.push(resourceType);
1587
+ async syncVertexIndexes(vertex) {
1588
+ const required = new Map();
1589
+ // An entry always carries a modified date so a query ordered by it can be paged by the
1590
+ // same key set as the creation date, falling back to the creation date for a vertex which
1591
+ // has never been modified rather than leaving the column empty.
1592
+ const indexDateModified = vertex.dateModified ?? vertex.dateCreated;
1593
+ // Every vertex carries an entry for its own id, so a query matching on the vertex id can
1594
+ // be answered from the index alone. Without it a vertex holding no aliases and no
1595
+ // resources would have no entries at all and could never be matched here.
1596
+ const vertexIdValue = vertex.id.toLowerCase();
1597
+ required.set(`${AuditableItemGraphService._INDEX_TYPE_VERTEX}|${vertexIdValue}`, {
1598
+ type: AuditableItemGraphService._INDEX_TYPE_VERTEX,
1599
+ value: vertexIdValue
1600
+ });
1601
+ // Index values are case folded here, and every lookup folds its term to match, so the
1602
+ // comparisons stay case insensitive without relying on the column collation or on the
1603
+ // connector folding case itself. The vertex keeps the value exactly as supplied.
1604
+ for (const alias of vertex.aliases ?? []) {
1605
+ if (Is.empty(alias.dateDeleted) && Is.stringValue(alias.id)) {
1606
+ const value = alias.id.toLowerCase();
1607
+ required.set(`${AuditableItemGraphService._INDEX_TYPE_ALIAS}|${value}`, {
1608
+ type: AuditableItemGraphService._INDEX_TYPE_ALIAS,
1609
+ value
1610
+ });
1611
+ }
1612
+ }
1613
+ for (const resource of vertex.resources ?? []) {
1614
+ const resourceType = ObjectHelper.extractProperty(resource.resourceObject, ["@type", "type"], false);
1615
+ if (Is.stringValue(resourceType)) {
1616
+ const value = resourceType.toLowerCase();
1617
+ required.set(`${AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE}|${value}`, {
1618
+ type: AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE,
1619
+ value
1620
+ });
1621
+ }
1622
+ }
1623
+ const existing = await this.queryVertexIndexes({
1624
+ property: "vertexId",
1625
+ comparison: ComparisonOperator.Equals,
1626
+ value: vertex.id
1627
+ }, ["id", "type", "value", "dateCreated", "dateModified"]);
1628
+ // Collapse by row id first. A paged read can return the same row on more than one page,
1629
+ // and without this a retained row could be queued for removal by its own duplicate.
1630
+ const existingById = new Map();
1631
+ for (const entry of existing) {
1632
+ if (Is.stringValue(entry.id)) {
1633
+ existingById.set(entry.id, entry);
1634
+ }
1635
+ }
1636
+ const retainedKeys = new Set();
1637
+ const removeIds = [];
1638
+ const writeEntries = [];
1639
+ for (const [id, entry] of existingById) {
1640
+ // Entries which are no longer required, and any duplicates of a retained entry,
1641
+ // are removed so exactly one entry remains per index value.
1642
+ const key = `${entry.type}|${entry.value}`;
1643
+ const requiredEntry = required.get(key);
1644
+ if (!Is.empty(requiredEntry) && !retainedKeys.has(key)) {
1645
+ retainedKeys.add(key);
1646
+ if (entry.dateCreated !== vertex.dateCreated || entry.dateModified !== indexDateModified) {
1647
+ // Keep the row id so the copied dates are corrected in place rather than duplicated.
1648
+ writeEntries.push({
1649
+ id,
1650
+ vertexId: vertex.id,
1651
+ type: requiredEntry.type,
1652
+ value: requiredEntry.value,
1653
+ dateCreated: vertex.dateCreated,
1654
+ dateModified: indexDateModified
1655
+ });
1616
1656
  }
1617
1657
  }
1658
+ else {
1659
+ removeIds.push(id);
1660
+ }
1661
+ }
1662
+ for (const [key, entry] of required) {
1663
+ if (!retainedKeys.has(key)) {
1664
+ writeEntries.push({
1665
+ id: Converter.bytesToHex(RandomHelper.generate(16)),
1666
+ vertexId: vertex.id,
1667
+ type: entry.type,
1668
+ value: entry.value,
1669
+ dateCreated: vertex.dateCreated,
1670
+ dateModified: indexDateModified
1671
+ });
1672
+ }
1673
+ }
1674
+ if (removeIds.length > 0) {
1675
+ await this._vertexIndexStorage.removeBatch(removeIds);
1618
1676
  }
1619
- const resourceTypeIndex = resourceTypes.join("||").toLowerCase();
1677
+ if (writeEntries.length > 0) {
1678
+ await this._vertexIndexStorage.setBatch(writeEntries);
1679
+ }
1680
+ }
1681
+ /**
1682
+ * Build the JSON-LD vertex list returned by query, and record the query metric.
1683
+ * @param entities The vertex entities for the page.
1684
+ * @param cursor The already encoded cursor for the next page.
1685
+ * @returns The compacted vertex list and the cursor.
1686
+ * @internal
1687
+ */
1688
+ async buildVertexList(entities, cursor) {
1689
+ const models = entities.map(e => this.vertexEntityToJsonLd(e));
1690
+ const vertexList = {
1691
+ "@context": [
1692
+ SchemaOrgContexts.Context,
1693
+ AuditableItemGraphContexts.Context,
1694
+ AuditableItemGraphContexts.ContextCommon
1695
+ ],
1696
+ type: [SchemaOrgTypes.ItemList, AuditableItemGraphTypes.VertexList],
1697
+ [SchemaOrgTypes.ItemListElement]: models
1698
+ };
1699
+ const result = await JsonLdProcessor.compact(vertexList, vertexList["@context"], {
1700
+ compactArrays: false
1701
+ });
1702
+ await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.QueriesExecuted, {
1703
+ resultCount: models.length,
1704
+ hasMore: Is.stringValue(cursor)
1705
+ });
1706
+ return {
1707
+ entries: result,
1708
+ cursor
1709
+ };
1710
+ }
1711
+ /**
1712
+ * Build the condition matching one index type and value.
1713
+ * @param type The index type to match.
1714
+ * @param comparison The comparison to apply to the value.
1715
+ * @param value The index value or values to match, already lower cased.
1716
+ * @returns The condition.
1717
+ * @internal
1718
+ */
1719
+ indexTypeValueCondition(type, comparison, value) {
1720
+ return {
1721
+ logicalOperator: LogicalOperator.And,
1722
+ conditions: [
1723
+ { property: "type", comparison: ComparisonOperator.Equals, value: type },
1724
+ { property: "value", comparison, value }
1725
+ ]
1726
+ };
1727
+ }
1728
+ /**
1729
+ * Combine conditions so that matching any one of them is enough.
1730
+ * @param conditions The conditions to combine.
1731
+ * @returns The single condition when only one was given, otherwise their union.
1732
+ * @internal
1733
+ */
1734
+ anyOfConditions(conditions) {
1735
+ return conditions.length === 1
1736
+ ? conditions[0]
1737
+ : { logicalOperator: LogicalOperator.Or, conditions };
1738
+ }
1739
+ /**
1740
+ * Read one page of vertices whose index entries satisfy every dimension. The entries of a
1741
+ * vertex are grouped so it appears once however many of them match, its vertex is joined to
1742
+ * the page so no second read is needed, and the connector pages the groups with its own cursor.
1743
+ * @param dimensions The index conditions, a vertex has to satisfy every one of them.
1744
+ * @param orderProperty The index date property to order by.
1745
+ * @param orderDirection The direction to order in.
1746
+ * @param propertiesToReturn The vertex properties to return.
1747
+ * @param conditions Additional conditions the vertices must match.
1748
+ * @param cursor The cursor returned by the previous page.
1749
+ * @param limit The maximum number of vertices to return.
1750
+ * @returns The vertices in order and the cursor for the next page.
1751
+ * @internal
1752
+ */
1753
+ async queryIndexVertices(dimensions, orderProperty, orderDirection, propertiesToReturn, conditions, cursor, limit) {
1754
+ // An entry only ever satisfies one dimension, so the entries of every dimension have to
1755
+ // survive into the group for the group conditions to find them there, which is why the
1756
+ // conditions are their union and the dimensions are applied across the group.
1757
+ const results = await this._vertexIndexStorage.queryJoin(this._vertexStorage, {
1758
+ property: "vertexId",
1759
+ joinProperty: "id",
1760
+ groupProperty: "vertexId",
1761
+ conditions: this.anyOfConditions(dimensions),
1762
+ groupConditions: dimensions.length > 1 ? dimensions : undefined,
1763
+ sortProperties: [
1764
+ { property: orderProperty, sortDirection: orderDirection },
1765
+ { property: "vertexId", sortDirection: SortDirection.Ascending }
1766
+ ],
1767
+ properties: ["vertexId"],
1768
+ joinConditions: conditions,
1769
+ joinRequired: true,
1770
+ joinProperties: propertiesToReturn,
1771
+ cursor,
1772
+ limit
1773
+ });
1774
+ // The join is required, so every group which survived carries its vertex.
1620
1775
  return {
1621
- aliasIndex: Is.stringValue(aliasIndex) ? `||${aliasIndex}||` : undefined,
1622
- resourceTypeIndex: Is.stringValue(resourceTypeIndex) ? `||${resourceTypeIndex}||` : undefined
1776
+ entities: results.entities.map(entity => entity.joined[0]),
1777
+ cursor: results.cursor
1623
1778
  };
1624
1779
  }
1780
+ /**
1781
+ * Query all the pages of the vertex index storage which match the conditions.
1782
+ * @param conditions The conditions to match.
1783
+ * @param properties The properties to return.
1784
+ * @returns The matching index entries.
1785
+ * @internal
1786
+ */
1787
+ async queryVertexIndexes(conditions, properties) {
1788
+ const entities = [];
1789
+ let cursor;
1790
+ do {
1791
+ // Paging needs a deterministic order or a row can be repeated or skipped between
1792
+ // pages, the entity declares no default sort so the primary key is used.
1793
+ const results = await this._vertexIndexStorage.query(conditions, [{ property: "id", sortDirection: SortDirection.Ascending }], properties, cursor);
1794
+ entities.push(...results.entities);
1795
+ cursor = results.cursor;
1796
+ } while (Is.stringValue(cursor));
1797
+ return entities;
1798
+ }
1625
1799
  /**
1626
1800
  * Find vertices with matching aliases.
1627
1801
  * @param vertexId The id of the vertex to exclude from the search.
@@ -1630,22 +1804,27 @@ export class AuditableItemGraphService {
1630
1804
  * @internal
1631
1805
  */
1632
1806
  async findMatchingVertices(vertexId, aliasId) {
1633
- const results = await this._vertexStorage.query({
1807
+ const entities = await this.queryVertexIndexes({
1808
+ logicalOperator: LogicalOperator.And,
1634
1809
  conditions: [
1635
1810
  {
1636
- property: "aliasIndex",
1637
- comparison: ComparisonOperator.Includes,
1638
- value: `||${aliasId.toLowerCase()}||`
1811
+ property: "type",
1812
+ comparison: ComparisonOperator.Equals,
1813
+ value: AuditableItemGraphService._INDEX_TYPE_ALIAS
1639
1814
  },
1640
1815
  {
1641
- property: "id",
1642
- value: vertexId,
1643
- comparison: ComparisonOperator.NotEquals
1816
+ property: "value",
1817
+ comparison: ComparisonOperator.Equals,
1818
+ value: aliasId.toLowerCase()
1819
+ },
1820
+ {
1821
+ property: "vertexId",
1822
+ comparison: ComparisonOperator.NotEquals,
1823
+ value: vertexId
1644
1824
  }
1645
- ],
1646
- logicalOperator: LogicalOperator.And
1647
- });
1648
- return results.entities.length > 0;
1825
+ ]
1826
+ }, ["vertexId"]);
1827
+ return entities.length > 0;
1649
1828
  }
1650
1829
  /**
1651
1830
  * Whether an incoming edge matches a stored edge id.