@twin.org/dataspace-data-plane-service 0.0.3-next.29 → 0.0.3-next.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/dataspaceDataPlaneRoutes.js +13 -4
- package/dist/es/dataspaceDataPlaneRoutes.js.map +1 -1
- package/dist/es/dataspaceDataPlaneService.js +555 -6
- package/dist/es/dataspaceDataPlaneService.js.map +1 -1
- package/dist/es/entities/pushSubscription.js +83 -0
- package/dist/es/entities/pushSubscription.js.map +1 -0
- package/dist/es/index.js +1 -0
- package/dist/es/index.js.map +1 -1
- package/dist/es/models/IDataspaceDataPlaneServiceConfig.js.map +1 -1
- package/dist/es/models/IDataspaceDataPlaneServiceConstructorOptions.js.map +1 -1
- package/dist/es/schema.js +2 -0
- package/dist/es/schema.js.map +1 -1
- package/dist/types/dataspaceDataPlaneService.d.ts +37 -2
- package/dist/types/entities/pushSubscription.d.ts +35 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/models/IDataspaceDataPlaneServiceConfig.d.ts +21 -0
- package/dist/types/models/IDataspaceDataPlaneServiceConstructorOptions.d.ts +5 -0
- package/docs/changelog.md +34 -0
- package/docs/open-api/spec.json +9 -21
- package/docs/reference/classes/DataspaceDataPlaneService.md +140 -1
- package/docs/reference/classes/PushSubscription.md +89 -0
- package/docs/reference/index.md +1 -0
- package/docs/reference/interfaces/IDataspaceDataPlaneServiceConfig.md +57 -0
- package/docs/reference/interfaces/IDataspaceDataPlaneServiceConstructorOptions.md +14 -0
- package/locales/en.json +23 -3
- package/package.json +3 -3
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
import { TaskStatus } from "@twin.org/background-task-models";
|
|
2
2
|
import { ContextIdKeys, ContextIdStore } from "@twin.org/context";
|
|
3
|
-
import { ArrayHelper, BaseError, ComponentFactory, ConflictError, Converter, GeneralError, GuardError, Guards, Is, JsonHelper, NotFoundError, RandomHelper, UnprocessableError, Validation } from "@twin.org/core";
|
|
3
|
+
import { ArrayHelper, BaseError, ComponentFactory, ConflictError, Converter, GeneralError, GuardError, Guards, Is, JsonHelper, NotFoundError, RandomHelper, UnauthorizedError, UnprocessableError, Validation } from "@twin.org/core";
|
|
4
4
|
import { Blake2b } from "@twin.org/crypto";
|
|
5
5
|
import { DataTypeHelper, JsonSchemaHelper } from "@twin.org/data-core";
|
|
6
6
|
import { JsonLdDataTypes, JsonLdHelper, JsonLdProcessor } from "@twin.org/data-json-ld";
|
|
7
|
-
import { ActivityProcessingStatus, ActivityTaskStatus, DataRequestType, DataspaceAppFactory, DataspaceContexts, DataspaceDataTypes, DataspaceTypes } from "@twin.org/dataspace-models";
|
|
7
|
+
import { ActivityProcessingStatus, ActivityTaskStatus, DataRequestType, DataspaceAppFactory, DataspaceContexts, DataspaceDataTypes, DataspaceTypes, getJsonLdType } from "@twin.org/dataspace-models";
|
|
8
8
|
import { EngineCoreFactory } from "@twin.org/engine-models";
|
|
9
9
|
import { ComparisonOperator, LogicalOperator } from "@twin.org/entity";
|
|
10
10
|
import { EntityStorageConnectorFactory } from "@twin.org/entity-storage-models";
|
|
11
11
|
import { DataspaceProtocolDataTypes, DataspaceProtocolTransferProcessStateType } from "@twin.org/standards-dataspace-protocol";
|
|
12
12
|
import { SchemaOrgContexts, SchemaOrgDataTypes, SchemaOrgTypes } from "@twin.org/standards-schema-org";
|
|
13
|
-
import {
|
|
13
|
+
import { ActivityStreamsContexts, ActivityStreamsTypes } from "@twin.org/standards-w3c-activity-streams";
|
|
14
|
+
import { OdrlContexts, OdrlTypes } from "@twin.org/standards-w3c-odrl";
|
|
14
15
|
import { TrustHelper } from "@twin.org/trust-models";
|
|
16
|
+
const FOLLOW_ACTIVITY_URN_PREFIX = "urn:x-follow:";
|
|
17
|
+
const TRANSFER_URN_PREFIX = "urn:x-transfer:";
|
|
18
|
+
const UNDO_ACTIVITY_URN_PREFIX = "urn:x-undo:";
|
|
19
|
+
const ACTIVITY_LOG_URN_PREFIX = "urn:x-activity-log:";
|
|
15
20
|
/**
|
|
16
21
|
* Dataspace Data Plane Service.
|
|
17
22
|
*/
|
|
@@ -20,6 +25,10 @@ export class DataspaceDataPlaneService {
|
|
|
20
25
|
* Runtime name for the class.
|
|
21
26
|
*/
|
|
22
27
|
static CLASS_NAME = "DataspaceDataPlaneService";
|
|
28
|
+
/**
|
|
29
|
+
* Background task type identifier for push delivery tasks.
|
|
30
|
+
*/
|
|
31
|
+
static PUSH_DELIVERY_TASK_TYPE = "push-delivery";
|
|
23
32
|
/**
|
|
24
33
|
* Milliseconds per minute (60 * 1000).
|
|
25
34
|
* @internal
|
|
@@ -90,11 +99,31 @@ export class DataspaceDataPlaneService {
|
|
|
90
99
|
* @internal
|
|
91
100
|
*/
|
|
92
101
|
_retryCount;
|
|
102
|
+
/**
|
|
103
|
+
* Max retry count for push delivery HTTP requests.
|
|
104
|
+
* @internal
|
|
105
|
+
*/
|
|
106
|
+
_pushRetryCount;
|
|
107
|
+
/**
|
|
108
|
+
* Base retry delay (ms) for push delivery HTTP requests.
|
|
109
|
+
* @internal
|
|
110
|
+
*/
|
|
111
|
+
_pushRetryBaseDelayMs;
|
|
112
|
+
/**
|
|
113
|
+
* Timeout (ms) for each push delivery HTTP POST request.
|
|
114
|
+
* @internal
|
|
115
|
+
*/
|
|
116
|
+
_pushTimeoutMs;
|
|
93
117
|
/**
|
|
94
118
|
* Clean up interval for activity logs.
|
|
95
119
|
* @internal
|
|
96
120
|
*/
|
|
97
121
|
_activityLogCleanUpInterval;
|
|
122
|
+
/**
|
|
123
|
+
* Interval in minutes between orphaned PushSubscription cleanup scans.
|
|
124
|
+
* @internal
|
|
125
|
+
*/
|
|
126
|
+
_pushSubscriptionCleanupIntervalMinutes;
|
|
98
127
|
/**
|
|
99
128
|
* Whether there is an ongoing clean up process.
|
|
100
129
|
* @internal
|
|
@@ -131,6 +160,12 @@ export class DataspaceDataPlaneService {
|
|
|
131
160
|
* @internal
|
|
132
161
|
*/
|
|
133
162
|
_transferProcessStorage;
|
|
163
|
+
/**
|
|
164
|
+
* Factory key used to look up the push-subscription storage. Resolved lazily at call time
|
|
165
|
+
* (not at construction) so the storage may register after the data plane is built.
|
|
166
|
+
* @internal
|
|
167
|
+
*/
|
|
168
|
+
_pushSubscriptionStorageType;
|
|
134
169
|
/**
|
|
135
170
|
* Entity storage for tenant-supplied Dataspace App Dataset entities.
|
|
136
171
|
* @internal
|
|
@@ -153,6 +188,11 @@ export class DataspaceDataPlaneService {
|
|
|
153
188
|
// Entity storage for Transfer Process state lookup
|
|
154
189
|
// Used to read transfer state from shared storage (written by Control Plane)
|
|
155
190
|
this._transferProcessStorage = EntityStorageConnectorFactory.get(options?.transferProcessEntityStorageType ?? "transfer-process");
|
|
191
|
+
// Push-subscription storage is optional and resolved lazily. The data plane is typically
|
|
192
|
+
// constructed before its dependent storages register, so caching the connector here would
|
|
193
|
+
// permanently miss it. We only need the factory key — every push call site re-resolves.
|
|
194
|
+
this._pushSubscriptionStorageType =
|
|
195
|
+
options?.pushSubscriptionEntityStorageType ?? "push-subscription";
|
|
156
196
|
this._dataspaceAppDatasetStorage = EntityStorageConnectorFactory.get(options?.dataspaceAppDatasetEntityStorageType ?? "dataspace-app-dataset");
|
|
157
197
|
JsonLdDataTypes.registerTypes();
|
|
158
198
|
DataspaceDataTypes.registerTypes();
|
|
@@ -167,7 +207,11 @@ export class DataspaceDataPlaneService {
|
|
|
167
207
|
this._retainActivityLogsFor =
|
|
168
208
|
DataspaceDataPlaneService._DEFAULT_RETAIN_INTERVAL * DataspaceDataPlaneService._MS_PER_MINUTE;
|
|
169
209
|
this._retryCount = options?.config?.retryCount;
|
|
210
|
+
this._pushRetryCount = options?.config?.pushRetryCount ?? 3;
|
|
211
|
+
this._pushRetryBaseDelayMs = options?.config?.pushRetryBaseDelayMs ?? 1000;
|
|
212
|
+
this._pushTimeoutMs = options?.config?.pushTimeoutMs ?? 30000;
|
|
170
213
|
this._activityLogCleanUpInterval = DataspaceDataPlaneService._DEFAULT_CLEANUP_INTERVAL;
|
|
214
|
+
this._pushSubscriptionCleanupIntervalMinutes = Math.max(1, Math.round((options?.config?.pushSubscriptionCleanupIntervalMs ?? 3_600_000) / 60_000));
|
|
171
215
|
this._cleanUpProcessOngoing = false;
|
|
172
216
|
const validationErrors = [];
|
|
173
217
|
if (!Is.empty(options?.config?.retainActivityLogsFor)) {
|
|
@@ -191,6 +235,11 @@ export class DataspaceDataPlaneService {
|
|
|
191
235
|
Validation.integer("options.config.activityLogsCleanUpInterval", options.config.activityLogsCleanUpInterval, validationErrors, undefined, { minValue: 1 });
|
|
192
236
|
this._activityLogCleanUpInterval = options.config.activityLogsCleanUpInterval;
|
|
193
237
|
}
|
|
238
|
+
if (!Is.empty(options?.config?.pushTimeoutMs)) {
|
|
239
|
+
Guards.integer(DataspaceDataPlaneService.CLASS_NAME, "options.config.pushTimeoutMs", options.config.pushTimeoutMs);
|
|
240
|
+
Validation.integer("options.config.pushTimeoutMs", options.config.pushTimeoutMs, validationErrors, undefined, { minValue: 1 });
|
|
241
|
+
this._pushTimeoutMs = options.config.pushTimeoutMs;
|
|
242
|
+
}
|
|
194
243
|
Validation.asValidationError(DataspaceDataPlaneService.CLASS_NAME, "options.config", validationErrors);
|
|
195
244
|
}
|
|
196
245
|
/**
|
|
@@ -205,6 +254,11 @@ export class DataspaceDataPlaneService {
|
|
|
205
254
|
* @param nodeLoggingComponentType The node logging component type.
|
|
206
255
|
*/
|
|
207
256
|
async start(nodeLoggingComponentType) {
|
|
257
|
+
await this._backgroundTaskComponent.registerHandler(DataspaceDataPlaneService.PUSH_DELIVERY_TASK_TYPE, "@twin.org/dataspace-app-runner", "pushDeliveryRunner", undefined, {
|
|
258
|
+
initialiseMethod: "pushDeliveryRunnerStart",
|
|
259
|
+
shutdownMethod: "pushDeliveryRunnerEnd",
|
|
260
|
+
idleShutdownTimeout: -1
|
|
261
|
+
});
|
|
208
262
|
const engine = EngineCoreFactory.getIfExists("engine");
|
|
209
263
|
if (Is.empty(engine) || engine.isClone()) {
|
|
210
264
|
await this._logging?.log({
|
|
@@ -239,14 +293,72 @@ export class DataspaceDataPlaneService {
|
|
|
239
293
|
}
|
|
240
294
|
});
|
|
241
295
|
}
|
|
296
|
+
const pushCleanupTaskTime = [
|
|
297
|
+
{
|
|
298
|
+
nextTriggerTime: Date.now() + 10_000,
|
|
299
|
+
...this.calculateCleaningTaskSchedule(this._pushSubscriptionCleanupIntervalMinutes)
|
|
300
|
+
}
|
|
301
|
+
];
|
|
302
|
+
await this._taskScheduler.addTask("dataspace-push-subscription-cleanup", pushCleanupTaskTime, async () => {
|
|
303
|
+
await this.cleanupOrphanedPushSubscriptions();
|
|
304
|
+
});
|
|
242
305
|
}
|
|
243
306
|
/**
|
|
244
307
|
* Notify an Activity.
|
|
245
308
|
* @param activity The Activity notified.
|
|
309
|
+
* @param trustPayload Optional trust payload to verify the requester's identity.
|
|
246
310
|
* @returns The activity's id or entry.
|
|
247
311
|
*/
|
|
248
|
-
async notifyActivity(activity) {
|
|
312
|
+
async notifyActivity(activity, trustPayload) {
|
|
249
313
|
Guards.object(DataspaceDataPlaneService.CLASS_NAME, "activity", activity);
|
|
314
|
+
// For cross-node push deliveries the caller presents a JWT. Verify it,
|
|
315
|
+
// confirm the referenced transfer is still in STARTED state, and assert
|
|
316
|
+
// the verified identity is one of the two parties on that transfer.
|
|
317
|
+
if (Is.stringValue(trustPayload)) {
|
|
318
|
+
const trustInfo = await TrustHelper.verifyTrust(this._trustComponent, trustPayload, "notifyActivity");
|
|
319
|
+
const generatorPid = this.calculateActivityGeneratorIdentity(activity);
|
|
320
|
+
// Primary lookup: by consumerPid (the entity's primary key). If this hits, the
|
|
321
|
+
// generator's PID equals consumerPid — the generator is the consumer side.
|
|
322
|
+
let transferProcess = await this._transferProcessStorage.get(generatorPid);
|
|
323
|
+
const generatorIsConsumer = Boolean(transferProcess);
|
|
324
|
+
if (!transferProcess) {
|
|
325
|
+
// Fallback: generatorPid === providerPid. providerPid is a UUIDv7 so it's
|
|
326
|
+
// unique per transfer, but defensively reject any case where the secondary
|
|
327
|
+
// index returns more than one match — silent first-match would risk
|
|
328
|
+
// authorising the wrong transfer if the invariant ever breaks.
|
|
329
|
+
const result = await this._transferProcessStorage.query({
|
|
330
|
+
conditions: [
|
|
331
|
+
{
|
|
332
|
+
property: "providerPid",
|
|
333
|
+
value: generatorPid,
|
|
334
|
+
comparison: ComparisonOperator.Equals
|
|
335
|
+
}
|
|
336
|
+
]
|
|
337
|
+
});
|
|
338
|
+
if (result.entities.length > 1) {
|
|
339
|
+
throw new UnauthorizedError(DataspaceDataPlaneService.CLASS_NAME, "pushActivityNotAuthorized");
|
|
340
|
+
}
|
|
341
|
+
transferProcess = result.entities[0];
|
|
342
|
+
// generatorIsConsumer stays false → generator is the provider side.
|
|
343
|
+
}
|
|
344
|
+
if (transferProcess?.state !== DataspaceProtocolTransferProcessStateType.STARTED) {
|
|
345
|
+
throw new UnauthorizedError(DataspaceDataPlaneService.CLASS_NAME, "pushActivityNotAuthorized");
|
|
346
|
+
}
|
|
347
|
+
// Bind the verified identity to the side of the transfer matching the claimed
|
|
348
|
+
// generator. Without this, a party with a valid token for transfer X can post
|
|
349
|
+
// an activity claiming to be the other party on the same transfer.
|
|
350
|
+
// Stored consumer/provider identities are composites
|
|
351
|
+
// (`nodeDid:hash(tenantId)`)
|
|
352
|
+
const expectedIdentity = generatorIsConsumer
|
|
353
|
+
? transferProcess.consumerIdentity
|
|
354
|
+
: transferProcess.providerIdentity;
|
|
355
|
+
const callerComposite = Is.stringValue(trustInfo.tenantId)
|
|
356
|
+
? `${trustInfo.identity}:${trustInfo.tenantId}`
|
|
357
|
+
: trustInfo.identity;
|
|
358
|
+
if (!Is.stringValue(expectedIdentity) || callerComposite !== expectedIdentity) {
|
|
359
|
+
throw new UnauthorizedError(DataspaceDataPlaneService.CLASS_NAME, "pushActivityNotAuthorized");
|
|
360
|
+
}
|
|
361
|
+
}
|
|
250
362
|
await this._logging?.log({
|
|
251
363
|
level: "debug",
|
|
252
364
|
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
@@ -271,7 +383,7 @@ export class DataspaceDataPlaneService {
|
|
|
271
383
|
const canonical = JsonHelper.canonicalize(activity);
|
|
272
384
|
const canonicalBytes = Converter.utf8ToBytes(canonical);
|
|
273
385
|
const activityLogId = Converter.bytesToHex(Blake2b.sum256(canonicalBytes));
|
|
274
|
-
const activityLogEntryId =
|
|
386
|
+
const activityLogEntryId = `${ACTIVITY_LOG_URN_PREFIX}${activityLogId}`;
|
|
275
387
|
// Check if entry already exists
|
|
276
388
|
let logEntry = await this._entityStorageActivityLogs.get(activityLogEntryId);
|
|
277
389
|
let existingSuccessfulApps = [];
|
|
@@ -550,6 +662,313 @@ export class DataspaceDataPlaneService {
|
|
|
550
662
|
}
|
|
551
663
|
return this.buildTransferContext(transferProcess);
|
|
552
664
|
}
|
|
665
|
+
/**
|
|
666
|
+
* Set up a push subscription after a transfer enters STARTED from REQUESTED.
|
|
667
|
+
* Reads the TransferProcess, builds an IFollowActivity, calls the app's
|
|
668
|
+
* subscribeToData, and persists a PushSubscription entity.
|
|
669
|
+
* @param consumerPid The consumer process ID identifying the transfer.
|
|
670
|
+
*/
|
|
671
|
+
async setupPushSubscription(consumerPid) {
|
|
672
|
+
Guards.stringValue(DataspaceDataPlaneService.CLASS_NAME, "consumerPid", consumerPid);
|
|
673
|
+
const transferProcess = await this._transferProcessStorage.get(consumerPid);
|
|
674
|
+
if (!transferProcess) {
|
|
675
|
+
throw new NotFoundError(DataspaceDataPlaneService.CLASS_NAME, "transferProcessNotFound", consumerPid);
|
|
676
|
+
}
|
|
677
|
+
if (transferProcess.state !== DataspaceProtocolTransferProcessStateType.STARTED) {
|
|
678
|
+
throw new GeneralError(DataspaceDataPlaneService.CLASS_NAME, "transferNotInStartedState", {
|
|
679
|
+
currentState: transferProcess.state
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
if (!Is.stringValue(transferProcess.dataAddress?.endpoint)) {
|
|
683
|
+
throw new GeneralError(DataspaceDataPlaneService.CLASS_NAME, "transferMissingDataAddress", {
|
|
684
|
+
consumerPid
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
// On multi-tenant publishers, the consumer must have baked its tenant token into the
|
|
688
|
+
// callback URL so the consumer's TenantProcessor can route inbound push deliveries.
|
|
689
|
+
// Reject at setup time rather than silently 401 every push.
|
|
690
|
+
// Parse as a real URL so we match on the actual query parameter, not a substring
|
|
691
|
+
// in a path segment / value position.
|
|
692
|
+
if (this._partitionContextIds?.includes(ContextIdKeys.Tenant)) {
|
|
693
|
+
let hasTenantToken = false;
|
|
694
|
+
try {
|
|
695
|
+
hasTenantToken = new URL(transferProcess.dataAddress.endpoint).searchParams.has("x-enc-tenant-token");
|
|
696
|
+
}
|
|
697
|
+
catch {
|
|
698
|
+
// Malformed URL — treat as missing token (will fail closed below).
|
|
699
|
+
}
|
|
700
|
+
if (!hasTenantToken) {
|
|
701
|
+
throw new GeneralError(DataspaceDataPlaneService.CLASS_NAME, "pushSubscriptionMissingTenantToken", { consumerPid, endpoint: transferProcess.dataAddress.endpoint });
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
const followActivityId = `${FOLLOW_ACTIVITY_URN_PREFIX}${RandomHelper.generateUuidV7("compact")}`;
|
|
705
|
+
const followActivity = {
|
|
706
|
+
"@context": ActivityStreamsContexts.Context,
|
|
707
|
+
id: followActivityId,
|
|
708
|
+
type: ActivityStreamsTypes.Follow,
|
|
709
|
+
generator: transferProcess.consumerPid,
|
|
710
|
+
actor: transferProcess.consumerIdentity ?? "",
|
|
711
|
+
object: { id: `${TRANSFER_URN_PREFIX}${transferProcess.consumerPid}` }
|
|
712
|
+
};
|
|
713
|
+
let appForCompensation;
|
|
714
|
+
if (Is.stringValue(transferProcess.datasetId)) {
|
|
715
|
+
const appDataset = await this._dataspaceAppDatasetStorage.get(transferProcess.datasetId);
|
|
716
|
+
if (appDataset) {
|
|
717
|
+
const app = DataspaceAppFactory.get(appDataset.appId);
|
|
718
|
+
await app.subscribeToData?.(followActivity);
|
|
719
|
+
appForCompensation = app;
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
const setupContextIds = await ContextIdStore.getContextIds();
|
|
723
|
+
const setupTenantId = setupContextIds?.[ContextIdKeys.Tenant];
|
|
724
|
+
const subscription = {
|
|
725
|
+
consumerPid: transferProcess.consumerPid,
|
|
726
|
+
providerPid: transferProcess.providerPid,
|
|
727
|
+
followActivityId,
|
|
728
|
+
datasetId: transferProcess.datasetId,
|
|
729
|
+
tenantId: Is.stringValue(setupTenantId) ? setupTenantId : undefined,
|
|
730
|
+
consumerEndpoint: transferProcess.dataAddress.endpoint,
|
|
731
|
+
consumerAuthToken: transferProcess.dataAddress.endpointProperties?.find(p => p.name === "authorization")?.value,
|
|
732
|
+
paused: false,
|
|
733
|
+
dateCreated: new Date().toISOString(),
|
|
734
|
+
dateModified: new Date().toISOString()
|
|
735
|
+
};
|
|
736
|
+
try {
|
|
737
|
+
await this.requirePushSubscriptionStorage().set(subscription);
|
|
738
|
+
}
|
|
739
|
+
catch (storageError) {
|
|
740
|
+
// Compensating Undo: the app's subscribeToData succeeded but the row didn't
|
|
741
|
+
// persist. Without compensation, a retry generates a new followActivityId and
|
|
742
|
+
// registers a second Follow on the app — with no persisted id to drive an Undo.
|
|
743
|
+
if (appForCompensation) {
|
|
744
|
+
const compensatingUndo = {
|
|
745
|
+
"@context": ActivityStreamsContexts.Context,
|
|
746
|
+
id: `${UNDO_ACTIVITY_URN_PREFIX}${RandomHelper.generateUuidV7("compact")}`,
|
|
747
|
+
type: ActivityStreamsTypes.Undo,
|
|
748
|
+
generator: transferProcess.consumerPid,
|
|
749
|
+
actor: transferProcess.consumerIdentity ?? "",
|
|
750
|
+
object: followActivityId
|
|
751
|
+
};
|
|
752
|
+
try {
|
|
753
|
+
await appForCompensation.unsubscribeToData?.(compensatingUndo);
|
|
754
|
+
}
|
|
755
|
+
catch (compensationError) {
|
|
756
|
+
await this._logging?.log({
|
|
757
|
+
level: "error",
|
|
758
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
759
|
+
ts: Date.now(),
|
|
760
|
+
message: "pushSubscriptionCompensationFailed",
|
|
761
|
+
data: { consumerPid, providerPid: transferProcess.providerPid },
|
|
762
|
+
error: BaseError.fromError(compensationError)
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
throw storageError;
|
|
767
|
+
}
|
|
768
|
+
await this._logging?.log({
|
|
769
|
+
level: "info",
|
|
770
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
771
|
+
ts: Date.now(),
|
|
772
|
+
message: "pushSubscriptionCreated",
|
|
773
|
+
data: { consumerPid, providerPid: transferProcess.providerPid }
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
/**
|
|
777
|
+
* Pause deliveries for a push subscription. The subscription entity stays
|
|
778
|
+
* alive with status=Paused. No app unsubscribe call.
|
|
779
|
+
* @param consumerPid The consumer process ID identifying the transfer.
|
|
780
|
+
*/
|
|
781
|
+
async suspendPushSubscription(consumerPid) {
|
|
782
|
+
Guards.stringValue(DataspaceDataPlaneService.CLASS_NAME, "consumerPid", consumerPid);
|
|
783
|
+
const subscription = await this.requirePushSubscriptionStorage().get(consumerPid);
|
|
784
|
+
if (!subscription) {
|
|
785
|
+
throw new NotFoundError(DataspaceDataPlaneService.CLASS_NAME, "pushSubscriptionNotFound", consumerPid);
|
|
786
|
+
}
|
|
787
|
+
if (subscription.paused) {
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
790
|
+
subscription.paused = true;
|
|
791
|
+
subscription.dateModified = new Date().toISOString();
|
|
792
|
+
await this.requirePushSubscriptionStorage().set(subscription);
|
|
793
|
+
await this._logging?.log({
|
|
794
|
+
level: "info",
|
|
795
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
796
|
+
ts: Date.now(),
|
|
797
|
+
message: "pushSubscriptionSuspended",
|
|
798
|
+
data: { consumerPid }
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* Resume deliveries after a SUSPENDED → STARTED transition. Flips status
|
|
803
|
+
* back to Active. No app subscribeToData call.
|
|
804
|
+
* @param consumerPid The consumer process ID identifying the transfer.
|
|
805
|
+
*/
|
|
806
|
+
async resumePushSubscription(consumerPid) {
|
|
807
|
+
Guards.stringValue(DataspaceDataPlaneService.CLASS_NAME, "consumerPid", consumerPid);
|
|
808
|
+
const subscription = await this.requirePushSubscriptionStorage().get(consumerPid);
|
|
809
|
+
if (!subscription) {
|
|
810
|
+
throw new NotFoundError(DataspaceDataPlaneService.CLASS_NAME, "pushSubscriptionNotFound", consumerPid);
|
|
811
|
+
}
|
|
812
|
+
if (!subscription.paused) {
|
|
813
|
+
return;
|
|
814
|
+
}
|
|
815
|
+
subscription.paused = false;
|
|
816
|
+
subscription.dateModified = new Date().toISOString();
|
|
817
|
+
await this.requirePushSubscriptionStorage().set(subscription);
|
|
818
|
+
await this._logging?.log({
|
|
819
|
+
level: "info",
|
|
820
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
821
|
+
ts: Date.now(),
|
|
822
|
+
message: "pushSubscriptionResumed",
|
|
823
|
+
data: { consumerPid }
|
|
824
|
+
});
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* Tear down a push subscription. Builds an IUndoActivity, calls the app's
|
|
828
|
+
* unsubscribeToData, and deletes the PushSubscription entity.
|
|
829
|
+
* @param consumerPid The consumer process ID identifying the transfer.
|
|
830
|
+
*/
|
|
831
|
+
async teardownPushSubscription(consumerPid) {
|
|
832
|
+
Guards.stringValue(DataspaceDataPlaneService.CLASS_NAME, "consumerPid", consumerPid);
|
|
833
|
+
const subscription = await this.requirePushSubscriptionStorage().get(consumerPid);
|
|
834
|
+
if (!subscription) {
|
|
835
|
+
await this._logging?.log({
|
|
836
|
+
level: "warn",
|
|
837
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
838
|
+
ts: Date.now(),
|
|
839
|
+
message: "pushSubscriptionNotFoundOnTeardown",
|
|
840
|
+
data: { consumerPid }
|
|
841
|
+
});
|
|
842
|
+
return;
|
|
843
|
+
}
|
|
844
|
+
const transferProcess = await this._transferProcessStorage.get(consumerPid);
|
|
845
|
+
const undoActivity = {
|
|
846
|
+
"@context": ActivityStreamsContexts.Context,
|
|
847
|
+
id: `${UNDO_ACTIVITY_URN_PREFIX}${RandomHelper.generateUuidV7("compact")}`,
|
|
848
|
+
type: ActivityStreamsTypes.Undo,
|
|
849
|
+
generator: subscription.consumerPid,
|
|
850
|
+
actor: transferProcess?.consumerIdentity ?? "",
|
|
851
|
+
object: subscription.followActivityId
|
|
852
|
+
};
|
|
853
|
+
if (Is.stringValue(subscription.datasetId)) {
|
|
854
|
+
const appDataset = await this._dataspaceAppDatasetStorage.get(subscription.datasetId);
|
|
855
|
+
if (appDataset) {
|
|
856
|
+
const app = DataspaceAppFactory.get(appDataset.appId);
|
|
857
|
+
await app.unsubscribeToData?.(undoActivity);
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
await this.requirePushSubscriptionStorage().remove(consumerPid);
|
|
861
|
+
await this._logging?.log({
|
|
862
|
+
level: "info",
|
|
863
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
864
|
+
ts: Date.now(),
|
|
865
|
+
message: "pushSubscriptionTornDown",
|
|
866
|
+
data: { consumerPid, providerPid: subscription.providerPid }
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
/**
|
|
870
|
+
* Schedule a push delivery when the app has new outbound data.
|
|
871
|
+
* @param activity The outbound activity carrying the data payload.
|
|
872
|
+
*/
|
|
873
|
+
async processOutboxActivity(activity) {
|
|
874
|
+
Guards.object(DataspaceDataPlaneService.CLASS_NAME, "activity", activity);
|
|
875
|
+
// Extract consumerPid from activity.to
|
|
876
|
+
let consumerPid;
|
|
877
|
+
if (Is.stringValue(activity.to)) {
|
|
878
|
+
consumerPid = activity.to;
|
|
879
|
+
}
|
|
880
|
+
else if (Is.array(activity.to)) {
|
|
881
|
+
if (activity.to.length > 1) {
|
|
882
|
+
throw new GeneralError(DataspaceDataPlaneService.CLASS_NAME, "processOutboxActivityMultipleTo", { count: activity.to.length });
|
|
883
|
+
}
|
|
884
|
+
consumerPid = activity.to[0];
|
|
885
|
+
}
|
|
886
|
+
if (!Is.stringValue(consumerPid)) {
|
|
887
|
+
await this._logging?.log({
|
|
888
|
+
level: "warn",
|
|
889
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
890
|
+
message: "pushSubscriptionNotFoundForActivity",
|
|
891
|
+
data: { consumerPid }
|
|
892
|
+
});
|
|
893
|
+
return;
|
|
894
|
+
}
|
|
895
|
+
// Load PushSubscription
|
|
896
|
+
const subscription = await this.requirePushSubscriptionStorage().get(consumerPid);
|
|
897
|
+
if (!subscription) {
|
|
898
|
+
await this._logging?.log({
|
|
899
|
+
level: "warn",
|
|
900
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
901
|
+
message: "pushSubscriptionNotFoundForActivity",
|
|
902
|
+
data: { consumerPid }
|
|
903
|
+
});
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
// Skip delivery if subscription is paused
|
|
907
|
+
if (subscription.paused) {
|
|
908
|
+
await this._logging?.log({
|
|
909
|
+
level: "debug",
|
|
910
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
911
|
+
message: "pushDeliverySkippedPaused",
|
|
912
|
+
data: { consumerPid }
|
|
913
|
+
});
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
// Load TransferProcess and validate it is STARTED
|
|
917
|
+
const transferProcess = await this._transferProcessStorage.get(consumerPid);
|
|
918
|
+
if (transferProcess?.state !== DataspaceProtocolTransferProcessStateType.STARTED) {
|
|
919
|
+
await this._logging?.log({
|
|
920
|
+
level: "warn",
|
|
921
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
922
|
+
message: "pushSubscriptionNotFoundForActivity",
|
|
923
|
+
data: { consumerPid }
|
|
924
|
+
});
|
|
925
|
+
return;
|
|
926
|
+
}
|
|
927
|
+
// Build agreement from stored transfer context
|
|
928
|
+
const { agreement } = this.buildTransferContext(transferProcess);
|
|
929
|
+
// Extract data object from activity; a string value is an IRI reference — wrap it so the IRI is preserved.
|
|
930
|
+
let data;
|
|
931
|
+
if (Is.object(activity.object)) {
|
|
932
|
+
data = activity.object;
|
|
933
|
+
}
|
|
934
|
+
else if (Is.stringValue(activity.object)) {
|
|
935
|
+
data = { "@id": activity.object };
|
|
936
|
+
}
|
|
937
|
+
else {
|
|
938
|
+
data = {};
|
|
939
|
+
}
|
|
940
|
+
const entityType = Is.object(activity.object) ? (getJsonLdType(activity.object) ?? "") : "";
|
|
941
|
+
// Capture the subscription's tenantId so the background-task runner can re-enter the
|
|
942
|
+
// owning tenant's context before any tenant-scoped operation (PEP, trust signing).
|
|
943
|
+
// Falls back to the current request context if the subscription pre-dates tenantId capture.
|
|
944
|
+
let payloadTenantId = subscription.tenantId;
|
|
945
|
+
if (!Is.stringValue(payloadTenantId)) {
|
|
946
|
+
const ctxIds = await ContextIdStore.getContextIds();
|
|
947
|
+
const ctxTenant = ctxIds?.[ContextIdKeys.Tenant];
|
|
948
|
+
payloadTenantId = Is.stringValue(ctxTenant) ? ctxTenant : undefined;
|
|
949
|
+
}
|
|
950
|
+
const payload = {
|
|
951
|
+
consumerPid,
|
|
952
|
+
providerPid: transferProcess.providerPid,
|
|
953
|
+
generatorPid: transferProcess.providerPid,
|
|
954
|
+
consumerEndpoint: subscription.consumerEndpoint,
|
|
955
|
+
consumerAuthToken: subscription.consumerAuthToken,
|
|
956
|
+
agreement,
|
|
957
|
+
data,
|
|
958
|
+
entityType,
|
|
959
|
+
tenantId: payloadTenantId,
|
|
960
|
+
pushTimeoutMs: this._pushTimeoutMs,
|
|
961
|
+
pushRetryCount: this._pushRetryCount,
|
|
962
|
+
pushRetryBaseDelayMs: this._pushRetryBaseDelayMs
|
|
963
|
+
};
|
|
964
|
+
const taskId = await this._backgroundTaskComponent.create(DataspaceDataPlaneService.PUSH_DELIVERY_TASK_TYPE, payload, { retainFor: this._retainTasksFor, retryCount: this._retryCount });
|
|
965
|
+
await this._logging?.log({
|
|
966
|
+
level: "info",
|
|
967
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
968
|
+
message: "pushDeliveryTaskScheduled",
|
|
969
|
+
data: { taskId, consumerPid }
|
|
970
|
+
});
|
|
971
|
+
}
|
|
553
972
|
// ============================================================================
|
|
554
973
|
// PRIVATE HELPER METHODS
|
|
555
974
|
// ============================================================================
|
|
@@ -776,6 +1195,136 @@ export class DataspaceDataPlaneService {
|
|
|
776
1195
|
}
|
|
777
1196
|
return result;
|
|
778
1197
|
}
|
|
1198
|
+
/**
|
|
1199
|
+
* Resolve the push-subscription storage or throw if it isn't registered. Pull-only
|
|
1200
|
+
* deployments may run the data plane without it.
|
|
1201
|
+
* @returns The push-subscription storage connector.
|
|
1202
|
+
* @throws GeneralError if the storage isn't registered.
|
|
1203
|
+
* @internal
|
|
1204
|
+
*/
|
|
1205
|
+
requirePushSubscriptionStorage() {
|
|
1206
|
+
const storage = EntityStorageConnectorFactory.getIfExists(this._pushSubscriptionStorageType);
|
|
1207
|
+
if (!storage) {
|
|
1208
|
+
throw new GeneralError(DataspaceDataPlaneService.CLASS_NAME, "pushSubscriptionStorageNotRegistered");
|
|
1209
|
+
}
|
|
1210
|
+
return storage;
|
|
1211
|
+
}
|
|
1212
|
+
/**
|
|
1213
|
+
* Deletes PushSubscription entities whose TransferProcess is absent, COMPLETED, or TERMINATED.
|
|
1214
|
+
* On multi-tenant nodes (`partitionContextIds` includes `Tenant`), iterates each registered tenant
|
|
1215
|
+
* and runs the cleanup inside that tenant's context so partitioned storage queries scope correctly.
|
|
1216
|
+
* @internal
|
|
1217
|
+
*/
|
|
1218
|
+
async cleanupOrphanedPushSubscriptions() {
|
|
1219
|
+
// Pull-only deployments may run the data plane without push-subscription storage —
|
|
1220
|
+
// the scheduled cleanup task fires regardless, so silent no-op is the right behaviour.
|
|
1221
|
+
const storage = EntityStorageConnectorFactory.getIfExists(this._pushSubscriptionStorageType);
|
|
1222
|
+
if (!storage) {
|
|
1223
|
+
return;
|
|
1224
|
+
}
|
|
1225
|
+
let numDeleted = 0;
|
|
1226
|
+
if (this._partitionContextIds?.includes(ContextIdKeys.Tenant)) {
|
|
1227
|
+
// Per-tenant try/catch — a transient failure on one tenant must not poison the rest
|
|
1228
|
+
// of the cleanup pass. Each tenant gets its own attempt; failures are logged with the
|
|
1229
|
+
// offending tenantId so operators can triage.
|
|
1230
|
+
let cursor;
|
|
1231
|
+
do {
|
|
1232
|
+
let result;
|
|
1233
|
+
try {
|
|
1234
|
+
result = await this._tenantAdmin?.query(undefined, cursor);
|
|
1235
|
+
}
|
|
1236
|
+
catch (error) {
|
|
1237
|
+
await this._logging?.log({
|
|
1238
|
+
level: "error",
|
|
1239
|
+
message: "cleanupFailed",
|
|
1240
|
+
ts: Date.now(),
|
|
1241
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
1242
|
+
error: BaseError.fromError(error)
|
|
1243
|
+
});
|
|
1244
|
+
break;
|
|
1245
|
+
}
|
|
1246
|
+
cursor = result?.cursor;
|
|
1247
|
+
if (!Is.empty(result)) {
|
|
1248
|
+
for (const tenantId of result.tenants.map(t => t.id)) {
|
|
1249
|
+
try {
|
|
1250
|
+
const localContextIds = (await ContextIdStore.getContextIds()) ?? {};
|
|
1251
|
+
localContextIds[ContextIdKeys.Tenant] = tenantId;
|
|
1252
|
+
await ContextIdStore.run(localContextIds, async () => {
|
|
1253
|
+
numDeleted += await this.cleanupOrphanedPushSubscriptionsPartition();
|
|
1254
|
+
});
|
|
1255
|
+
}
|
|
1256
|
+
catch (error) {
|
|
1257
|
+
await this._logging?.log({
|
|
1258
|
+
level: "error",
|
|
1259
|
+
message: "cleanupFailed",
|
|
1260
|
+
ts: Date.now(),
|
|
1261
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
1262
|
+
data: { tenantId },
|
|
1263
|
+
error: BaseError.fromError(error)
|
|
1264
|
+
});
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
} while (Is.stringValue(cursor));
|
|
1269
|
+
}
|
|
1270
|
+
else {
|
|
1271
|
+
numDeleted += await this.cleanupOrphanedPushSubscriptionsPartition();
|
|
1272
|
+
}
|
|
1273
|
+
await this._logging?.log({
|
|
1274
|
+
level: "debug",
|
|
1275
|
+
message: "pushSubscriptionsCleanedUp",
|
|
1276
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
1277
|
+
data: { numDeleted }
|
|
1278
|
+
});
|
|
1279
|
+
}
|
|
1280
|
+
/**
|
|
1281
|
+
* Per-partition cleanup body for orphaned PushSubscriptions. Must run inside the
|
|
1282
|
+
* target tenant's context on multi-tenant nodes (caller wraps `ContextIdStore.run`).
|
|
1283
|
+
* @returns The number of subscriptions deleted in this partition.
|
|
1284
|
+
* @internal
|
|
1285
|
+
*/
|
|
1286
|
+
async cleanupOrphanedPushSubscriptionsPartition() {
|
|
1287
|
+
let numDeleted = 0;
|
|
1288
|
+
try {
|
|
1289
|
+
// First pass: read all pages without modifying storage to avoid cursor drift
|
|
1290
|
+
const toDelete = [];
|
|
1291
|
+
let cursor;
|
|
1292
|
+
do {
|
|
1293
|
+
const result = await this.requirePushSubscriptionStorage().query(undefined, undefined, undefined, cursor);
|
|
1294
|
+
cursor = result.cursor;
|
|
1295
|
+
const pids = result.entities.map(s => s.consumerPid);
|
|
1296
|
+
const tpResult = await this._transferProcessStorage.query({
|
|
1297
|
+
property: "consumerPid",
|
|
1298
|
+
value: pids,
|
|
1299
|
+
comparison: ComparisonOperator.In
|
|
1300
|
+
});
|
|
1301
|
+
const tpMap = new Map(tpResult.entities.map(tp => [tp.consumerPid, tp.state]));
|
|
1302
|
+
for (const sub of result.entities) {
|
|
1303
|
+
const state = tpMap.get(sub.consumerPid);
|
|
1304
|
+
if (!state ||
|
|
1305
|
+
state === DataspaceProtocolTransferProcessStateType.COMPLETED ||
|
|
1306
|
+
state === DataspaceProtocolTransferProcessStateType.TERMINATED) {
|
|
1307
|
+
toDelete.push(sub.consumerPid);
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
} while (Is.stringValue(cursor));
|
|
1311
|
+
// Second pass: delete after all reads are complete
|
|
1312
|
+
for (const pid of toDelete) {
|
|
1313
|
+
await this.teardownPushSubscription(pid);
|
|
1314
|
+
numDeleted++;
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
catch (error) {
|
|
1318
|
+
await this._logging?.log({
|
|
1319
|
+
level: "error",
|
|
1320
|
+
message: "cleanupFailed",
|
|
1321
|
+
ts: Date.now(),
|
|
1322
|
+
source: DataspaceDataPlaneService.CLASS_NAME,
|
|
1323
|
+
error: BaseError.fromError(error)
|
|
1324
|
+
});
|
|
1325
|
+
}
|
|
1326
|
+
return numDeleted;
|
|
1327
|
+
}
|
|
779
1328
|
/**
|
|
780
1329
|
* Calculates the cleaning task schedule.
|
|
781
1330
|
* @param minutes The period in minutes.
|
|
@@ -939,7 +1488,7 @@ export class DataspaceDataPlaneService {
|
|
|
939
1488
|
// This would ensure policies are always up-to-date and support dynamic policy updates.
|
|
940
1489
|
const agreement = {
|
|
941
1490
|
"@context": OdrlContexts.Context,
|
|
942
|
-
"@type":
|
|
1491
|
+
"@type": OdrlTypes.Agreement,
|
|
943
1492
|
"@id": transferProcess.agreementId,
|
|
944
1493
|
target: transferProcess.datasetId,
|
|
945
1494
|
// Provider is the assigner, consumer is the assignee
|