@twin.org/auditable-item-graph-service 0.10.0 → 0.10.1-next.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/auditableItemGraphService.js +299 -120
- package/dist/es/auditableItemGraphService.js.map +1 -1
- package/dist/es/entities/auditableItemGraphAlias.js +2 -2
- package/dist/es/entities/auditableItemGraphAlias.js.map +1 -1
- package/dist/es/entities/auditableItemGraphChangeset.js +15 -5
- package/dist/es/entities/auditableItemGraphChangeset.js.map +1 -1
- package/dist/es/entities/auditableItemGraphEdge.js +2 -2
- package/dist/es/entities/auditableItemGraphEdge.js.map +1 -1
- package/dist/es/entities/auditableItemGraphPatch.js +3 -3
- package/dist/es/entities/auditableItemGraphPatch.js.map +1 -1
- package/dist/es/entities/auditableItemGraphResource.js +1 -1
- package/dist/es/entities/auditableItemGraphResource.js.map +1 -1
- package/dist/es/entities/auditableItemGraphVertex.js +4 -20
- package/dist/es/entities/auditableItemGraphVertex.js.map +1 -1
- package/dist/es/entities/auditableItemGraphVertexIndex.js +83 -0
- package/dist/es/entities/auditableItemGraphVertexIndex.js.map +1 -0
- package/dist/es/entities/auditableItemGraphVertexV1.js +113 -0
- package/dist/es/entities/auditableItemGraphVertexV1.js.map +1 -0
- package/dist/es/index.js +2 -0
- package/dist/es/index.js.map +1 -1
- package/dist/es/models/IAuditableItemGraphServiceConstructorOptions.js.map +1 -1
- package/dist/es/schema.js +4 -0
- package/dist/es/schema.js.map +1 -1
- package/dist/types/auditableItemGraphService.d.ts +3 -2
- package/dist/types/entities/auditableItemGraphVertex.d.ts +0 -8
- package/dist/types/entities/auditableItemGraphVertexIndex.d.ts +31 -0
- package/dist/types/entities/auditableItemGraphVertexV1.d.ts +58 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/models/IAuditableItemGraphServiceConstructorOptions.d.ts +5 -0
- package/docs/changelog.md +86 -0
- package/docs/open-api/spec.json +1 -1
- package/docs/reference/classes/AuditableItemGraphService.md +2 -1
- package/docs/reference/classes/AuditableItemGraphVertex.md +0 -16
- package/docs/reference/classes/AuditableItemGraphVertexIndex.md +63 -0
- package/docs/reference/classes/AuditableItemGraphVertexV1.md +109 -0
- package/docs/reference/index.md +2 -0
- package/docs/reference/interfaces/IAuditableItemGraphServiceConstructorOptions.md +14 -0
- package/package.json +16 -16
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import { HealthCategory, HealthStatus } from "@twin.org/api-models";
|
|
4
4
|
import { AuditableItemGraphAuditMode, AuditableItemGraphContexts, AuditableItemGraphDataTypes, AuditableItemGraphMetricIds, AuditableItemGraphMetrics, AuditableItemGraphTopics, AuditableItemGraphTypes, VerifyDepth } from "@twin.org/auditable-item-graph-models";
|
|
5
5
|
import { ContextIdHelper, ContextIdKeys, ContextIdStore } from "@twin.org/context";
|
|
6
|
-
import { ArrayHelper, BaseError, Coerce, ComponentFactory, GeneralError, Guards, Is, JsonHelper, Mutex, NotFoundError, ObjectHelper, RandomHelper, StringHelper, Urn, Validation } from "@twin.org/core";
|
|
6
|
+
import { ArrayHelper, BaseError, Coerce, ComponentFactory, Converter, GeneralError, Guards, Is, JsonHelper, Mutex, NotFoundError, ObjectHelper, RandomHelper, StringHelper, Urn, Validation } from "@twin.org/core";
|
|
7
7
|
import { DataTypeHelper } from "@twin.org/data-core";
|
|
8
8
|
import { JsonLdDataTypes, JsonLdHelper, JsonLdProcessor } from "@twin.org/data-json-ld";
|
|
9
9
|
import { ComparisonOperator, LogicalOperator, SortDirection } from "@twin.org/entity";
|
|
@@ -32,6 +32,21 @@ export class AuditableItemGraphService {
|
|
|
32
32
|
* The namespace for the service edge.
|
|
33
33
|
*/
|
|
34
34
|
static NAMESPACE_EDGE = "edge";
|
|
35
|
+
/**
|
|
36
|
+
* The index type for the id of the vertex itself.
|
|
37
|
+
* @internal
|
|
38
|
+
*/
|
|
39
|
+
static _INDEX_TYPE_VERTEX = "vertex";
|
|
40
|
+
/**
|
|
41
|
+
* The index type for vertex aliases.
|
|
42
|
+
* @internal
|
|
43
|
+
*/
|
|
44
|
+
static _INDEX_TYPE_ALIAS = "alias";
|
|
45
|
+
/**
|
|
46
|
+
* The index type for vertex resource types.
|
|
47
|
+
* @internal
|
|
48
|
+
*/
|
|
49
|
+
static _INDEX_TYPE_RESOURCE_TYPE = "resourceType";
|
|
35
50
|
/**
|
|
36
51
|
* The keys to pick when creating the proof for the stream.
|
|
37
52
|
* @internal
|
|
@@ -53,6 +68,11 @@ export class AuditableItemGraphService {
|
|
|
53
68
|
* @internal
|
|
54
69
|
*/
|
|
55
70
|
_vertexStorage;
|
|
71
|
+
/**
|
|
72
|
+
* The entity storage for vertex indices.
|
|
73
|
+
* @internal
|
|
74
|
+
*/
|
|
75
|
+
_vertexIndexStorage;
|
|
56
76
|
/**
|
|
57
77
|
* The entity storage for changesets.
|
|
58
78
|
* @internal
|
|
@@ -80,6 +100,7 @@ export class AuditableItemGraphService {
|
|
|
80
100
|
constructor(options) {
|
|
81
101
|
this._immutableProofComponent = ComponentFactory.get(options?.immutableProofComponentType ?? "immutable-proof");
|
|
82
102
|
this._vertexStorage = EntityStorageConnectorFactory.get(options?.vertexEntityStorageType ?? "auditable-item-graph-vertex");
|
|
103
|
+
this._vertexIndexStorage = EntityStorageConnectorFactory.get(options?.vertexIndexEntityStorageType ?? "auditable-item-graph-vertex-index");
|
|
83
104
|
this._changesetStorage = EntityStorageConnectorFactory.get(options?.changesetEntityStorageType ?? "auditable-item-graph-changeset");
|
|
84
105
|
this._eventBusComponent = ComponentFactory.getIfExists(options?.eventBusComponentType);
|
|
85
106
|
this._telemetryComponent = ComponentFactory.getIfExists(options?.telemetryComponentType);
|
|
@@ -197,15 +218,11 @@ export class AuditableItemGraphService {
|
|
|
197
218
|
await this.updateEdgeList(context, vertexModel, vertex.edges);
|
|
198
219
|
// Bypass vertices keep no changeset history, so there is no baseline version to record.
|
|
199
220
|
if (vertex.auditMode !== AuditableItemGraphAuditMode.Bypass) {
|
|
200
|
-
delete originalEntity.aliasIndex;
|
|
201
|
-
delete originalEntity.resourceTypeIndex;
|
|
202
221
|
await this.addChangeset(context, originalEntity, vertexModel, true, 0);
|
|
203
222
|
vertexModel.version = 0;
|
|
204
223
|
}
|
|
205
|
-
await this._vertexStorage.set(
|
|
206
|
-
|
|
207
|
-
...this.buildIndexes(vertexModel)
|
|
208
|
-
});
|
|
224
|
+
await this._vertexStorage.set(vertexModel);
|
|
225
|
+
await this.syncVertexIndexes(vertexModel);
|
|
209
226
|
await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.VerticesCreated);
|
|
210
227
|
const fullId = new Urn(AuditableItemGraphService.NAMESPACE, id).toString();
|
|
211
228
|
await this._eventBusComponent?.publish(AuditableItemGraphTopics.VertexCreated, { id: fullId });
|
|
@@ -254,7 +271,6 @@ export class AuditableItemGraphService {
|
|
|
254
271
|
userIdentity: contextIds?.[ContextIdKeys.User]
|
|
255
272
|
};
|
|
256
273
|
const auditModeTransition = this.resolveAuditModeTransition(vertexEntity, vertex.auditMode);
|
|
257
|
-
delete vertexEntity.aliasIndex;
|
|
258
274
|
const originalEntity = ObjectHelper.clone(vertexEntity);
|
|
259
275
|
const newEntity = ObjectHelper.clone(vertexEntity);
|
|
260
276
|
newEntity.annotationObject = vertex.annotationObject;
|
|
@@ -307,7 +323,6 @@ export class AuditableItemGraphService {
|
|
|
307
323
|
userIdentity: contextIds?.[ContextIdKeys.User]
|
|
308
324
|
};
|
|
309
325
|
const auditModeTransition = this.resolveAuditModeTransition(vertexEntity, partial.auditMode);
|
|
310
|
-
delete vertexEntity.aliasIndex;
|
|
311
326
|
const originalEntity = ObjectHelper.clone(vertexEntity);
|
|
312
327
|
const newEntity = ObjectHelper.clone(vertexEntity);
|
|
313
328
|
if (partial.annotationObject !== undefined) {
|
|
@@ -557,7 +572,7 @@ export class AuditableItemGraphService {
|
|
|
557
572
|
compactArrays: false
|
|
558
573
|
});
|
|
559
574
|
}
|
|
560
|
-
const changesets = await this.internalGetChangesets(vertexId, {
|
|
575
|
+
const changesets = await this.internalGetChangesets(vertexId, ["patches"], {
|
|
561
576
|
maxVersion: version
|
|
562
577
|
});
|
|
563
578
|
let entityState = {
|
|
@@ -681,7 +696,8 @@ export class AuditableItemGraphService {
|
|
|
681
696
|
* @param orderByDirection The direction for the order, defaults to desc.
|
|
682
697
|
* @param properties The properties to return, if not provided defaults to id, created, aliases and object.
|
|
683
698
|
* @param cursor The cursor to request the next chunk of entities.
|
|
684
|
-
* @param limit
|
|
699
|
+
* @param limit The maximum number of entities to return, a page can contain fewer so follow
|
|
700
|
+
* the cursor until it is absent to read them all.
|
|
685
701
|
* @returns The entities, which can be partial if a limited keys list was provided.
|
|
686
702
|
*/
|
|
687
703
|
async query(options, conditions, orderBy, orderByDirection, properties, cursor, limit) {
|
|
@@ -693,61 +709,45 @@ export class AuditableItemGraphService {
|
|
|
693
709
|
"aliases",
|
|
694
710
|
"annotationObject"
|
|
695
711
|
];
|
|
696
|
-
const andGroups = [];
|
|
697
712
|
const orderProperty = orderBy ?? "dateCreated";
|
|
698
713
|
const orderDirection = orderByDirection ?? SortDirection.Descending;
|
|
699
714
|
const idExact = options?.idExact ?? false;
|
|
700
|
-
if (!Is.empty(conditions)) {
|
|
701
|
-
andGroups.push(conditions);
|
|
702
|
-
}
|
|
703
715
|
const idOrAlias = options?.id;
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
716
|
+
const idMode = options?.idMode ?? "both";
|
|
717
|
+
const resourceTypes = options?.resourceTypes;
|
|
718
|
+
const hasIdFilter = Is.stringValue(idOrAlias);
|
|
719
|
+
const hasResourceTypes = Is.arrayValue(resourceTypes);
|
|
720
|
+
if (!propertiesToReturn.includes("id")) {
|
|
721
|
+
propertiesToReturn.unshift("id");
|
|
722
|
+
}
|
|
723
|
+
// Every id, alias and resource type match is held in the index storage. Each filter is
|
|
724
|
+
// one dimension, satisfied by any one of the entries of a vertex, and a vertex has to
|
|
725
|
+
// satisfy every dimension it was given.
|
|
726
|
+
const indexDimensions = [];
|
|
727
|
+
if (hasIdFilter) {
|
|
728
|
+
const comparison = idExact ? ComparisonOperator.Equals : ComparisonOperator.Includes;
|
|
729
|
+
const term = idOrAlias.toLowerCase();
|
|
730
|
+
const idPredicates = [];
|
|
707
731
|
if (idMode === "id" || idMode === "both") {
|
|
708
|
-
|
|
709
|
-
property: "id",
|
|
710
|
-
comparison: idExact ? ComparisonOperator.Equals : ComparisonOperator.Includes,
|
|
711
|
-
value: idOrAlias
|
|
712
|
-
});
|
|
732
|
+
idPredicates.push(this.indexTypeValueCondition(AuditableItemGraphService._INDEX_TYPE_VERTEX, comparison, term));
|
|
713
733
|
}
|
|
714
734
|
if (idMode === "alias" || idMode === "both") {
|
|
715
|
-
|
|
716
|
-
property: "aliasIndex",
|
|
717
|
-
comparison: ComparisonOperator.Includes,
|
|
718
|
-
value: idExact ? `||${idOrAlias.toLowerCase()}||` : idOrAlias.toLowerCase()
|
|
719
|
-
});
|
|
720
|
-
}
|
|
721
|
-
if (idComparators.length === 1) {
|
|
722
|
-
andGroups.push(idComparators[0]);
|
|
723
|
-
}
|
|
724
|
-
else if (idComparators.length > 1) {
|
|
725
|
-
andGroups.push({ logicalOperator: LogicalOperator.Or, conditions: idComparators });
|
|
735
|
+
idPredicates.push(this.indexTypeValueCondition(AuditableItemGraphService._INDEX_TYPE_ALIAS, comparison, term));
|
|
726
736
|
}
|
|
737
|
+
indexDimensions.push(this.anyOfConditions(idPredicates));
|
|
727
738
|
}
|
|
728
|
-
if (
|
|
729
|
-
|
|
730
|
-
property: "resourceTypeIndex",
|
|
731
|
-
comparison: ComparisonOperator.Includes,
|
|
732
|
-
value: `||${rt.toLowerCase()}||`
|
|
733
|
-
}));
|
|
734
|
-
if (resourceComparators.length === 1) {
|
|
735
|
-
andGroups.push(resourceComparators[0]);
|
|
736
|
-
}
|
|
737
|
-
else {
|
|
738
|
-
andGroups.push({
|
|
739
|
-
logicalOperator: LogicalOperator.Or,
|
|
740
|
-
conditions: resourceComparators
|
|
741
|
-
});
|
|
742
|
-
}
|
|
739
|
+
if (hasResourceTypes) {
|
|
740
|
+
indexDimensions.push(this.indexTypeValueCondition(AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE, ComparisonOperator.In, resourceTypes.map(resourceType => resourceType.toLowerCase())));
|
|
743
741
|
}
|
|
744
|
-
if (
|
|
745
|
-
|
|
742
|
+
if (indexDimensions.length > 0) {
|
|
743
|
+
const indexPage = await this.queryIndexVertices(indexDimensions, orderProperty, orderDirection, propertiesToReturn, conditions, cursor, limit);
|
|
744
|
+
return await this.buildVertexList(indexPage.entities, indexPage.cursor);
|
|
746
745
|
}
|
|
747
|
-
const finalConditions =
|
|
748
|
-
logicalOperator: LogicalOperator.And,
|
|
749
|
-
|
|
750
|
-
|
|
746
|
+
const finalConditions = Is.empty(conditions)
|
|
747
|
+
? { logicalOperator: LogicalOperator.And, conditions: [] }
|
|
748
|
+
: conditions;
|
|
749
|
+
// With no id, alias or resource type filter there is nothing for the index to answer,
|
|
750
|
+
// so the vertex storage is queried directly and paged by its own cursor.
|
|
751
751
|
const results = await this._vertexStorage.query(finalConditions, [
|
|
752
752
|
{
|
|
753
753
|
property: orderProperty,
|
|
@@ -758,27 +758,7 @@ export class AuditableItemGraphService {
|
|
|
758
758
|
sortDirection: SortDirection.Ascending
|
|
759
759
|
}
|
|
760
760
|
], propertiesToReturn, cursor, limit);
|
|
761
|
-
|
|
762
|
-
const vertexList = {
|
|
763
|
-
"@context": [
|
|
764
|
-
SchemaOrgContexts.Context,
|
|
765
|
-
AuditableItemGraphContexts.Context,
|
|
766
|
-
AuditableItemGraphContexts.ContextCommon
|
|
767
|
-
],
|
|
768
|
-
type: [SchemaOrgTypes.ItemList, AuditableItemGraphTypes.VertexList],
|
|
769
|
-
[SchemaOrgTypes.ItemListElement]: models
|
|
770
|
-
};
|
|
771
|
-
const result = await JsonLdProcessor.compact(vertexList, vertexList["@context"], {
|
|
772
|
-
compactArrays: false
|
|
773
|
-
});
|
|
774
|
-
await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.QueriesExecuted, {
|
|
775
|
-
resultCount: models.length,
|
|
776
|
-
hasMore: Is.stringValue(results.cursor)
|
|
777
|
-
});
|
|
778
|
-
return {
|
|
779
|
-
entries: result,
|
|
780
|
-
cursor: results.cursor
|
|
781
|
-
};
|
|
761
|
+
return await this.buildVertexList(results.entities, results.cursor);
|
|
782
762
|
}
|
|
783
763
|
catch (error) {
|
|
784
764
|
throw new GeneralError(AuditableItemGraphService.CLASS_NAME, "queryingFailed", undefined, error);
|
|
@@ -851,7 +831,7 @@ export class AuditableItemGraphService {
|
|
|
851
831
|
* @internal
|
|
852
832
|
*/
|
|
853
833
|
async compactVertexHistory(vertexId) {
|
|
854
|
-
const changesets = await this.internalGetChangesets(vertexId);
|
|
834
|
+
const changesets = await this.internalGetChangesets(vertexId, ["proofId"]);
|
|
855
835
|
for (const changeset of changesets) {
|
|
856
836
|
await this._changesetStorage.remove(changeset.id);
|
|
857
837
|
if (Is.stringValue(changeset.proofId)) {
|
|
@@ -878,17 +858,14 @@ export class AuditableItemGraphService {
|
|
|
878
858
|
return;
|
|
879
859
|
}
|
|
880
860
|
const nextVersion = Is.empty(originalEntity.version)
|
|
881
|
-
? (await this.internalGetChangesets(vertexId)).length
|
|
861
|
+
? (await this.internalGetChangesets(vertexId, [])).length
|
|
882
862
|
: originalEntity.version + 1;
|
|
883
863
|
const patches = await this.addChangeset(context, originalEntity, newEntity, false, nextVersion);
|
|
884
864
|
if (patches.length > 0) {
|
|
885
865
|
newEntity.dateModified = context.now;
|
|
886
866
|
newEntity.version = nextVersion;
|
|
887
|
-
|
|
888
|
-
await this.
|
|
889
|
-
...newEntity,
|
|
890
|
-
...indexes
|
|
891
|
-
});
|
|
867
|
+
await this._vertexStorage.set(newEntity);
|
|
868
|
+
await this.syncVertexIndexes(newEntity);
|
|
892
869
|
await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.VerticesUpdated, {
|
|
893
870
|
patchCount: patches.length
|
|
894
871
|
});
|
|
@@ -917,11 +894,8 @@ export class AuditableItemGraphService {
|
|
|
917
894
|
await this.compactVertexHistory(vertexId);
|
|
918
895
|
}
|
|
919
896
|
newEntity.dateModified = context.now;
|
|
920
|
-
|
|
921
|
-
await this.
|
|
922
|
-
...newEntity,
|
|
923
|
-
...indexes
|
|
924
|
-
});
|
|
897
|
+
await this._vertexStorage.set(newEntity);
|
|
898
|
+
await this.syncVertexIndexes(newEntity);
|
|
925
899
|
await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.VerticesUpdated, {
|
|
926
900
|
patchCount: patches.length
|
|
927
901
|
});
|
|
@@ -1061,15 +1035,24 @@ export class AuditableItemGraphService {
|
|
|
1061
1035
|
/**
|
|
1062
1036
|
* Fetch all changesets for a vertex in ascending date order.
|
|
1063
1037
|
* @param vertexId The internal vertex id.
|
|
1038
|
+
* @param properties The additional changeset properties to retrieve, the id and dateCreated sort
|
|
1039
|
+
* keys are always included. Only the requested properties are populated on the returned
|
|
1040
|
+
* entities, so the large patches payload is only transferred when a caller reads it.
|
|
1064
1041
|
* @param options Optional filtering options.
|
|
1065
1042
|
* @param options.before Only fetch changesets created strictly before this ISO 8601 timestamp.
|
|
1066
1043
|
* @param options.maxVersion Only fetch changesets with version <= this value.
|
|
1067
1044
|
* @returns All changeset entities sorted ascending by dateCreated.
|
|
1068
1045
|
* @internal
|
|
1069
1046
|
*/
|
|
1070
|
-
async internalGetChangesets(vertexId, options) {
|
|
1047
|
+
async internalGetChangesets(vertexId, properties, options) {
|
|
1071
1048
|
const all = [];
|
|
1072
1049
|
let cursor;
|
|
1050
|
+
const propertiesToReturn = ["id", "dateCreated"];
|
|
1051
|
+
for (const property of properties) {
|
|
1052
|
+
if (!propertiesToReturn.includes(property)) {
|
|
1053
|
+
propertiesToReturn.push(property);
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1073
1056
|
const conditions = [
|
|
1074
1057
|
{ property: "vertexId", value: vertexId, comparison: ComparisonOperator.Equals }
|
|
1075
1058
|
];
|
|
@@ -1088,7 +1071,7 @@ export class AuditableItemGraphService {
|
|
|
1088
1071
|
});
|
|
1089
1072
|
}
|
|
1090
1073
|
do {
|
|
1091
|
-
const result = await this._changesetStorage.query({ conditions, logicalOperator: LogicalOperator.And }, [{ property: "dateCreated", sortDirection: SortDirection.Ascending }],
|
|
1074
|
+
const result = await this._changesetStorage.query({ conditions, logicalOperator: LogicalOperator.And }, [{ property: "dateCreated", sortDirection: SortDirection.Ascending }], propertiesToReturn, cursor);
|
|
1092
1075
|
all.push(...result.entities);
|
|
1093
1076
|
cursor = result.cursor;
|
|
1094
1077
|
} while (Is.stringValue(cursor));
|
|
@@ -1103,7 +1086,7 @@ export class AuditableItemGraphService {
|
|
|
1103
1086
|
* @internal
|
|
1104
1087
|
*/
|
|
1105
1088
|
async auditedVersions(vertexId, afterDate, beforeDate) {
|
|
1106
|
-
const allChangesets = await this.internalGetChangesets(vertexId, {
|
|
1089
|
+
const allChangesets = await this.internalGetChangesets(vertexId, ["version"], {
|
|
1107
1090
|
before: beforeDate?.toISOString()
|
|
1108
1091
|
});
|
|
1109
1092
|
const versions = [];
|
|
@@ -1596,32 +1579,223 @@ export class AuditableItemGraphService {
|
|
|
1596
1579
|
ObjectHelper.extractProperty(resource.resourceObject, ["id", "@id"], false));
|
|
1597
1580
|
}
|
|
1598
1581
|
/**
|
|
1599
|
-
*
|
|
1600
|
-
* @param vertex The vertex to
|
|
1601
|
-
* @returns
|
|
1582
|
+
* Synchronise the index storage with the current state of a vertex.
|
|
1583
|
+
* @param vertex The vertex to synchronise the index entries for.
|
|
1584
|
+
* @returns A promise that resolves when the index entries match the vertex.
|
|
1602
1585
|
* @internal
|
|
1603
1586
|
*/
|
|
1604
|
-
|
|
1605
|
-
const
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1587
|
+
async syncVertexIndexes(vertex) {
|
|
1588
|
+
const required = new Map();
|
|
1589
|
+
// An entry always carries a modified date so a query ordered by it can be paged by the
|
|
1590
|
+
// same key set as the creation date, falling back to the creation date for a vertex which
|
|
1591
|
+
// has never been modified rather than leaving the column empty.
|
|
1592
|
+
const indexDateModified = vertex.dateModified ?? vertex.dateCreated;
|
|
1593
|
+
// Every vertex carries an entry for its own id, so a query matching on the vertex id can
|
|
1594
|
+
// be answered from the index alone. Without it a vertex holding no aliases and no
|
|
1595
|
+
// resources would have no entries at all and could never be matched here.
|
|
1596
|
+
const vertexIdValue = vertex.id.toLowerCase();
|
|
1597
|
+
required.set(`${AuditableItemGraphService._INDEX_TYPE_VERTEX}|${vertexIdValue}`, {
|
|
1598
|
+
type: AuditableItemGraphService._INDEX_TYPE_VERTEX,
|
|
1599
|
+
value: vertexIdValue
|
|
1600
|
+
});
|
|
1601
|
+
// Index values are case folded here, and every lookup folds its term to match, so the
|
|
1602
|
+
// comparisons stay case insensitive without relying on the column collation or on the
|
|
1603
|
+
// connector folding case itself. The vertex keeps the value exactly as supplied.
|
|
1604
|
+
for (const alias of vertex.aliases ?? []) {
|
|
1605
|
+
if (Is.empty(alias.dateDeleted) && Is.stringValue(alias.id)) {
|
|
1606
|
+
const value = alias.id.toLowerCase();
|
|
1607
|
+
required.set(`${AuditableItemGraphService._INDEX_TYPE_ALIAS}|${value}`, {
|
|
1608
|
+
type: AuditableItemGraphService._INDEX_TYPE_ALIAS,
|
|
1609
|
+
value
|
|
1610
|
+
});
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
for (const resource of vertex.resources ?? []) {
|
|
1614
|
+
const resourceType = ObjectHelper.extractProperty(resource.resourceObject, ["@type", "type"], false);
|
|
1615
|
+
if (Is.stringValue(resourceType)) {
|
|
1616
|
+
const value = resourceType.toLowerCase();
|
|
1617
|
+
required.set(`${AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE}|${value}`, {
|
|
1618
|
+
type: AuditableItemGraphService._INDEX_TYPE_RESOURCE_TYPE,
|
|
1619
|
+
value
|
|
1620
|
+
});
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
const existing = await this.queryVertexIndexes({
|
|
1624
|
+
property: "vertexId",
|
|
1625
|
+
comparison: ComparisonOperator.Equals,
|
|
1626
|
+
value: vertex.id
|
|
1627
|
+
}, ["id", "type", "value", "dateCreated", "dateModified"]);
|
|
1628
|
+
// Collapse by row id first. A paged read can return the same row on more than one page,
|
|
1629
|
+
// and without this a retained row could be queued for removal by its own duplicate.
|
|
1630
|
+
const existingById = new Map();
|
|
1631
|
+
for (const entry of existing) {
|
|
1632
|
+
if (Is.stringValue(entry.id)) {
|
|
1633
|
+
existingById.set(entry.id, entry);
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
const retainedKeys = new Set();
|
|
1637
|
+
const removeIds = [];
|
|
1638
|
+
const writeEntries = [];
|
|
1639
|
+
for (const [id, entry] of existingById) {
|
|
1640
|
+
// Entries which are no longer required, and any duplicates of a retained entry,
|
|
1641
|
+
// are removed so exactly one entry remains per index value.
|
|
1642
|
+
const key = `${entry.type}|${entry.value}`;
|
|
1643
|
+
const requiredEntry = required.get(key);
|
|
1644
|
+
if (!Is.empty(requiredEntry) && !retainedKeys.has(key)) {
|
|
1645
|
+
retainedKeys.add(key);
|
|
1646
|
+
if (entry.dateCreated !== vertex.dateCreated || entry.dateModified !== indexDateModified) {
|
|
1647
|
+
// Keep the row id so the copied dates are corrected in place rather than duplicated.
|
|
1648
|
+
writeEntries.push({
|
|
1649
|
+
id,
|
|
1650
|
+
vertexId: vertex.id,
|
|
1651
|
+
type: requiredEntry.type,
|
|
1652
|
+
value: requiredEntry.value,
|
|
1653
|
+
dateCreated: vertex.dateCreated,
|
|
1654
|
+
dateModified: indexDateModified
|
|
1655
|
+
});
|
|
1616
1656
|
}
|
|
1617
1657
|
}
|
|
1658
|
+
else {
|
|
1659
|
+
removeIds.push(id);
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
for (const [key, entry] of required) {
|
|
1663
|
+
if (!retainedKeys.has(key)) {
|
|
1664
|
+
writeEntries.push({
|
|
1665
|
+
id: Converter.bytesToHex(RandomHelper.generate(16)),
|
|
1666
|
+
vertexId: vertex.id,
|
|
1667
|
+
type: entry.type,
|
|
1668
|
+
value: entry.value,
|
|
1669
|
+
dateCreated: vertex.dateCreated,
|
|
1670
|
+
dateModified: indexDateModified
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
if (removeIds.length > 0) {
|
|
1675
|
+
await this._vertexIndexStorage.removeBatch(removeIds);
|
|
1618
1676
|
}
|
|
1619
|
-
|
|
1677
|
+
if (writeEntries.length > 0) {
|
|
1678
|
+
await this._vertexIndexStorage.setBatch(writeEntries);
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
/**
|
|
1682
|
+
* Build the JSON-LD vertex list returned by query, and record the query metric.
|
|
1683
|
+
* @param entities The vertex entities for the page.
|
|
1684
|
+
* @param cursor The already encoded cursor for the next page.
|
|
1685
|
+
* @returns The compacted vertex list and the cursor.
|
|
1686
|
+
* @internal
|
|
1687
|
+
*/
|
|
1688
|
+
async buildVertexList(entities, cursor) {
|
|
1689
|
+
const models = entities.map(e => this.vertexEntityToJsonLd(e));
|
|
1690
|
+
const vertexList = {
|
|
1691
|
+
"@context": [
|
|
1692
|
+
SchemaOrgContexts.Context,
|
|
1693
|
+
AuditableItemGraphContexts.Context,
|
|
1694
|
+
AuditableItemGraphContexts.ContextCommon
|
|
1695
|
+
],
|
|
1696
|
+
type: [SchemaOrgTypes.ItemList, AuditableItemGraphTypes.VertexList],
|
|
1697
|
+
[SchemaOrgTypes.ItemListElement]: models
|
|
1698
|
+
};
|
|
1699
|
+
const result = await JsonLdProcessor.compact(vertexList, vertexList["@context"], {
|
|
1700
|
+
compactArrays: false
|
|
1701
|
+
});
|
|
1702
|
+
await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemGraphMetricIds.QueriesExecuted, {
|
|
1703
|
+
resultCount: models.length,
|
|
1704
|
+
hasMore: Is.stringValue(cursor)
|
|
1705
|
+
});
|
|
1706
|
+
return {
|
|
1707
|
+
entries: result,
|
|
1708
|
+
cursor
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
/**
|
|
1712
|
+
* Build the condition matching one index type and value.
|
|
1713
|
+
* @param type The index type to match.
|
|
1714
|
+
* @param comparison The comparison to apply to the value.
|
|
1715
|
+
* @param value The index value or values to match, already lower cased.
|
|
1716
|
+
* @returns The condition.
|
|
1717
|
+
* @internal
|
|
1718
|
+
*/
|
|
1719
|
+
indexTypeValueCondition(type, comparison, value) {
|
|
1720
|
+
return {
|
|
1721
|
+
logicalOperator: LogicalOperator.And,
|
|
1722
|
+
conditions: [
|
|
1723
|
+
{ property: "type", comparison: ComparisonOperator.Equals, value: type },
|
|
1724
|
+
{ property: "value", comparison, value }
|
|
1725
|
+
]
|
|
1726
|
+
};
|
|
1727
|
+
}
|
|
1728
|
+
/**
|
|
1729
|
+
* Combine conditions so that matching any one of them is enough.
|
|
1730
|
+
* @param conditions The conditions to combine.
|
|
1731
|
+
* @returns The single condition when only one was given, otherwise their union.
|
|
1732
|
+
* @internal
|
|
1733
|
+
*/
|
|
1734
|
+
anyOfConditions(conditions) {
|
|
1735
|
+
return conditions.length === 1
|
|
1736
|
+
? conditions[0]
|
|
1737
|
+
: { logicalOperator: LogicalOperator.Or, conditions };
|
|
1738
|
+
}
|
|
1739
|
+
/**
|
|
1740
|
+
* Read one page of vertices whose index entries satisfy every dimension. The entries of a
|
|
1741
|
+
* vertex are grouped so it appears once however many of them match, its vertex is joined to
|
|
1742
|
+
* the page so no second read is needed, and the connector pages the groups with its own cursor.
|
|
1743
|
+
* @param dimensions The index conditions, a vertex has to satisfy every one of them.
|
|
1744
|
+
* @param orderProperty The index date property to order by.
|
|
1745
|
+
* @param orderDirection The direction to order in.
|
|
1746
|
+
* @param propertiesToReturn The vertex properties to return.
|
|
1747
|
+
* @param conditions Additional conditions the vertices must match.
|
|
1748
|
+
* @param cursor The cursor returned by the previous page.
|
|
1749
|
+
* @param limit The maximum number of vertices to return.
|
|
1750
|
+
* @returns The vertices in order and the cursor for the next page.
|
|
1751
|
+
* @internal
|
|
1752
|
+
*/
|
|
1753
|
+
async queryIndexVertices(dimensions, orderProperty, orderDirection, propertiesToReturn, conditions, cursor, limit) {
|
|
1754
|
+
// An entry only ever satisfies one dimension, so the entries of every dimension have to
|
|
1755
|
+
// survive into the group for the group conditions to find them there, which is why the
|
|
1756
|
+
// conditions are their union and the dimensions are applied across the group.
|
|
1757
|
+
const results = await this._vertexIndexStorage.queryJoin(this._vertexStorage, {
|
|
1758
|
+
property: "vertexId",
|
|
1759
|
+
joinProperty: "id",
|
|
1760
|
+
groupProperty: "vertexId",
|
|
1761
|
+
conditions: this.anyOfConditions(dimensions),
|
|
1762
|
+
groupConditions: dimensions.length > 1 ? dimensions : undefined,
|
|
1763
|
+
sortProperties: [
|
|
1764
|
+
{ property: orderProperty, sortDirection: orderDirection },
|
|
1765
|
+
{ property: "vertexId", sortDirection: SortDirection.Ascending }
|
|
1766
|
+
],
|
|
1767
|
+
properties: ["vertexId"],
|
|
1768
|
+
joinConditions: conditions,
|
|
1769
|
+
joinRequired: true,
|
|
1770
|
+
joinProperties: propertiesToReturn,
|
|
1771
|
+
cursor,
|
|
1772
|
+
limit
|
|
1773
|
+
});
|
|
1774
|
+
// The join is required, so every group which survived carries its vertex.
|
|
1620
1775
|
return {
|
|
1621
|
-
|
|
1622
|
-
|
|
1776
|
+
entities: results.entities.map(entity => entity.joined[0]),
|
|
1777
|
+
cursor: results.cursor
|
|
1623
1778
|
};
|
|
1624
1779
|
}
|
|
1780
|
+
/**
|
|
1781
|
+
* Query all the pages of the vertex index storage which match the conditions.
|
|
1782
|
+
* @param conditions The conditions to match.
|
|
1783
|
+
* @param properties The properties to return.
|
|
1784
|
+
* @returns The matching index entries.
|
|
1785
|
+
* @internal
|
|
1786
|
+
*/
|
|
1787
|
+
async queryVertexIndexes(conditions, properties) {
|
|
1788
|
+
const entities = [];
|
|
1789
|
+
let cursor;
|
|
1790
|
+
do {
|
|
1791
|
+
// Paging needs a deterministic order or a row can be repeated or skipped between
|
|
1792
|
+
// pages, the entity declares no default sort so the primary key is used.
|
|
1793
|
+
const results = await this._vertexIndexStorage.query(conditions, [{ property: "id", sortDirection: SortDirection.Ascending }], properties, cursor);
|
|
1794
|
+
entities.push(...results.entities);
|
|
1795
|
+
cursor = results.cursor;
|
|
1796
|
+
} while (Is.stringValue(cursor));
|
|
1797
|
+
return entities;
|
|
1798
|
+
}
|
|
1625
1799
|
/**
|
|
1626
1800
|
* Find vertices with matching aliases.
|
|
1627
1801
|
* @param vertexId The id of the vertex to exclude from the search.
|
|
@@ -1630,22 +1804,27 @@ export class AuditableItemGraphService {
|
|
|
1630
1804
|
* @internal
|
|
1631
1805
|
*/
|
|
1632
1806
|
async findMatchingVertices(vertexId, aliasId) {
|
|
1633
|
-
const
|
|
1807
|
+
const entities = await this.queryVertexIndexes({
|
|
1808
|
+
logicalOperator: LogicalOperator.And,
|
|
1634
1809
|
conditions: [
|
|
1635
1810
|
{
|
|
1636
|
-
property: "
|
|
1637
|
-
comparison: ComparisonOperator.
|
|
1638
|
-
value:
|
|
1811
|
+
property: "type",
|
|
1812
|
+
comparison: ComparisonOperator.Equals,
|
|
1813
|
+
value: AuditableItemGraphService._INDEX_TYPE_ALIAS
|
|
1639
1814
|
},
|
|
1640
1815
|
{
|
|
1641
|
-
property: "
|
|
1642
|
-
|
|
1643
|
-
|
|
1816
|
+
property: "value",
|
|
1817
|
+
comparison: ComparisonOperator.Equals,
|
|
1818
|
+
value: aliasId.toLowerCase()
|
|
1819
|
+
},
|
|
1820
|
+
{
|
|
1821
|
+
property: "vertexId",
|
|
1822
|
+
comparison: ComparisonOperator.NotEquals,
|
|
1823
|
+
value: vertexId
|
|
1644
1824
|
}
|
|
1645
|
-
]
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
return results.entities.length > 0;
|
|
1825
|
+
]
|
|
1826
|
+
}, ["vertexId"]);
|
|
1827
|
+
return entities.length > 0;
|
|
1649
1828
|
}
|
|
1650
1829
|
/**
|
|
1651
1830
|
* Whether an incoming edge matches a stored edge id.
|