@twin.org/dataspace-control-plane-service 0.9.1-next.9 → 0.9.2-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/es/dataspaceControlPlaneRoutes.js +6 -2
- package/dist/es/dataspaceControlPlaneRoutes.js.map +1 -1
- package/dist/es/dataspaceControlPlaneService.js +346 -29
- package/dist/es/dataspaceControlPlaneService.js.map +1 -1
- package/dist/es/models/IDataspaceControlPlaneServiceConfig.js.map +1 -1
- package/dist/es/models/IDataspaceControlPlaneServiceConstructorOptions.js.map +1 -1
- package/dist/es/models/INegotiationState.js.map +1 -1
- package/dist/es/schema.js +3 -1
- package/dist/es/schema.js.map +1 -1
- package/dist/types/dataspaceControlPlaneService.d.ts +28 -6
- package/dist/types/models/IDataspaceControlPlaneServiceConfig.d.ts +13 -0
- package/dist/types/models/IDataspaceControlPlaneServiceConstructorOptions.d.ts +6 -0
- package/dist/types/models/INegotiationState.d.ts +3 -2
- package/docs/changelog.md +168 -0
- package/docs/reference/classes/DataspaceControlPlaneService.md +73 -4
- package/docs/reference/interfaces/IDataspaceControlPlaneServiceConfig.md +21 -0
- package/docs/reference/interfaces/IDataspaceControlPlaneServiceConstructorOptions.md +15 -0
- package/docs/reference/interfaces/INegotiationState.md +1 -1
- package/locales/en.json +8 -2
- package/package.json +3 -4
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0.
|
|
3
3
|
import { HttpContextIdKeys, HttpUrlHelper } from "@twin.org/api-models";
|
|
4
4
|
import { ContextIdHelper, ContextIdKeys, ContextIdStore } from "@twin.org/context";
|
|
5
|
-
import { AlreadyExistsError, ArrayHelper, BaseError, ComponentFactory, Converter, GeneralError, Guards, Is, NotFoundError, ObjectHelper, RandomHelper, StringHelper, UnauthorizedError, Url, Urn, ValidationError } from "@twin.org/core";
|
|
5
|
+
import { AlreadyExistsError, ArrayHelper, BaseError, Coerce, ComponentFactory, Converter, Factory, GeneralError, Guards, Is, NotFoundError, ObjectHelper, RandomHelper, StringHelper, UnauthorizedError, Url, Urn, ValidationError } from "@twin.org/core";
|
|
6
6
|
import { JsonLdHelper } from "@twin.org/data-json-ld";
|
|
7
|
-
import { DataspaceControlPlaneMetricIds, DataspaceControlPlaneMetrics, DataspaceTransferFormat, TransferProcessRole, getJsonLdId, getJsonLdType } from "@twin.org/dataspace-models";
|
|
8
|
-
import {
|
|
9
|
-
import { ComparisonOperator } from "@twin.org/entity";
|
|
7
|
+
import { DataspaceControlPlaneMetricIds, DataspaceControlPlaneMetrics, DataspaceTransferFormat, TransferProcessRole, TransferTerminationCode, getJsonLdId, getJsonLdType } from "@twin.org/dataspace-models";
|
|
8
|
+
import { ComparisonOperator, LogicalOperator } from "@twin.org/entity";
|
|
10
9
|
import { EntityStorageConnectorFactory } from "@twin.org/entity-storage-models";
|
|
11
10
|
import { OdrlPolicyHelper, PolicyRequesterFactory } from "@twin.org/rights-management-models";
|
|
12
11
|
import { DataspaceProtocolContexts, DataspaceProtocolContractNegotiationTypes, DataspaceProtocolHelper, DataspaceProtocolTransferProcessStateType, DataspaceProtocolTransferProcessTypes, DataspaceProtocolVersionBindingType } from "@twin.org/standards-dataspace-protocol";
|
|
@@ -49,6 +48,11 @@ export class DataspaceControlPlaneService {
|
|
|
49
48
|
* @internal
|
|
50
49
|
*/
|
|
51
50
|
static _STALLED_TRANSFER_THRESHOLD_MS = 30 * 60 * 1000;
|
|
51
|
+
/**
|
|
52
|
+
* Default provider transfer policy sweep interval in milliseconds (5 minutes).
|
|
53
|
+
* @internal
|
|
54
|
+
*/
|
|
55
|
+
static _PROVIDER_TRANSFER_POLICY_SWEEP_INTERVAL_MS = 5 * 60 * 1000;
|
|
52
56
|
/**
|
|
53
57
|
* The logging component.
|
|
54
58
|
* @internal
|
|
@@ -171,6 +175,22 @@ export class DataspaceControlPlaneService {
|
|
|
171
175
|
* @internal
|
|
172
176
|
*/
|
|
173
177
|
_stalledTransferTimeoutMs;
|
|
178
|
+
/**
|
|
179
|
+
* Idle window (ms) for Provider-side STARTED PULL transfers; undefined disables the policy
|
|
180
|
+
* node-wide.
|
|
181
|
+
* @internal
|
|
182
|
+
*/
|
|
183
|
+
_providerTransferIdleTimeoutMs;
|
|
184
|
+
/**
|
|
185
|
+
* Interval (ms) at which the provider transfer policy sweep runs.
|
|
186
|
+
* @internal
|
|
187
|
+
*/
|
|
188
|
+
_providerTransferPolicySweepIntervalMs;
|
|
189
|
+
/**
|
|
190
|
+
* Storage for transfer retrievals; undefined when the hosting engine does not register it.
|
|
191
|
+
* @internal
|
|
192
|
+
*/
|
|
193
|
+
_transferRetrievalStorage;
|
|
174
194
|
/**
|
|
175
195
|
* Create a new instance of DataspaceControlPlaneService.
|
|
176
196
|
* @param options The options for the service.
|
|
@@ -204,6 +224,11 @@ export class DataspaceControlPlaneService {
|
|
|
204
224
|
this._stalledTransferTimeoutMs =
|
|
205
225
|
options?.config?.stalledTransferTimeoutMs ??
|
|
206
226
|
DataspaceControlPlaneService._STALLED_TRANSFER_THRESHOLD_MS;
|
|
227
|
+
this._providerTransferIdleTimeoutMs = options?.config?.providerTransferIdleTimeoutMs;
|
|
228
|
+
this._providerTransferPolicySweepIntervalMs =
|
|
229
|
+
options?.config?.providerTransferPolicySweepIntervalMs ??
|
|
230
|
+
DataspaceControlPlaneService._PROVIDER_TRANSFER_POLICY_SWEEP_INTERVAL_MS;
|
|
231
|
+
this._transferRetrievalStorage = EntityStorageConnectorFactory.getIfExists(options?.transferRetrievalEntityStorageType ?? "transfer-retrieval");
|
|
207
232
|
this._taskScheduler = ComponentFactory.getIfExists(options?.taskSchedulerComponentType ?? "task-scheduler");
|
|
208
233
|
this._platformComponent = ComponentFactory.get(options?.platformComponentType ?? "platform");
|
|
209
234
|
this._telemetryComponent = ComponentFactory.getIfExists(options?.telemetryComponentType);
|
|
@@ -269,10 +294,10 @@ export class DataspaceControlPlaneService {
|
|
|
269
294
|
* @returns A promise that resolves when the federated catalogue is populated and the cleanup task is scheduled.
|
|
270
295
|
*/
|
|
271
296
|
async start(nodeLoggingComponentType) {
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
if (
|
|
297
|
+
const isCloneOrNoEngine = Factory.getFactory("engine-core")
|
|
298
|
+
?.getIfExists("engine")
|
|
299
|
+
?.isClone() ?? true;
|
|
300
|
+
if (isCloneOrNoEngine) {
|
|
276
301
|
await this._loggingComponent?.log({
|
|
277
302
|
level: "debug",
|
|
278
303
|
ts: Date.now(),
|
|
@@ -281,6 +306,7 @@ export class DataspaceControlPlaneService {
|
|
|
281
306
|
});
|
|
282
307
|
return;
|
|
283
308
|
}
|
|
309
|
+
await MetricHelper.createMetrics(this._telemetryComponent, DataspaceControlPlaneMetrics);
|
|
284
310
|
if (this._taskScheduler) {
|
|
285
311
|
await this._taskScheduler.addTask("control-plane-negotiation-cleanup", [
|
|
286
312
|
{
|
|
@@ -298,6 +324,15 @@ export class DataspaceControlPlaneService {
|
|
|
298
324
|
], async () => {
|
|
299
325
|
await this.cleanupStalledTransfers();
|
|
300
326
|
});
|
|
327
|
+
await this._taskScheduler.addTask("control-plane-transfer-policy", [
|
|
328
|
+
{
|
|
329
|
+
nextTriggerTime: Date.now(),
|
|
330
|
+
// The scheduler has minute granularity; floor at one minute.
|
|
331
|
+
intervalMinutes: Math.max(1, Math.round(this._providerTransferPolicySweepIntervalMs / 60000))
|
|
332
|
+
}
|
|
333
|
+
], async () => {
|
|
334
|
+
await this.applyProviderTransferPolicies();
|
|
335
|
+
});
|
|
301
336
|
}
|
|
302
337
|
}
|
|
303
338
|
/**
|
|
@@ -310,6 +345,7 @@ export class DataspaceControlPlaneService {
|
|
|
310
345
|
if (this._taskScheduler) {
|
|
311
346
|
await this._taskScheduler.removeTask("control-plane-negotiation-cleanup");
|
|
312
347
|
await this._taskScheduler.removeTask("control-plane-transfer-cleanup");
|
|
348
|
+
await this._taskScheduler.removeTask("control-plane-transfer-policy");
|
|
313
349
|
}
|
|
314
350
|
}
|
|
315
351
|
// ----------------------------------------------------------------------------
|
|
@@ -445,13 +481,14 @@ export class DataspaceControlPlaneService {
|
|
|
445
481
|
* @param trustPayload Trust payload for authenticating this call.
|
|
446
482
|
* @returns The consumerPid of the newly created TransferProcess.
|
|
447
483
|
*
|
|
448
|
-
* **Engine configuration requirement
|
|
484
|
+
* **Engine configuration requirement (remote transfers only):** when `providerEndpoint` is a
|
|
485
|
+
* remote origin the outbound call uses
|
|
449
486
|
* `ComponentFactory.create(remoteControlPlaneComponentType, { endpoint, pathPrefix })`.
|
|
450
487
|
* For the runtime `providerEndpoint` to be forwarded correctly, the engine **must** register
|
|
451
488
|
* the component type (default: `dataspace-control-plane-rest-client`) as a
|
|
452
489
|
* **multi-instance** component (`isMultiInstance: true` in engine config). A singleton
|
|
453
490
|
* registration ignores the runtime `endpoint` arg and silently POSTs to its
|
|
454
|
-
* static endpoint instead.
|
|
491
|
+
* static endpoint instead. Local-origin (same-node) transfers run in-process and are unaffected.
|
|
455
492
|
*/
|
|
456
493
|
async prepareTransfer(agreementId, providerEndpoint, format, trustPayload) {
|
|
457
494
|
Guards.stringValue(DataspaceControlPlaneService.CLASS_NAME, "agreementId", agreementId);
|
|
@@ -514,9 +551,10 @@ export class DataspaceControlPlaneService {
|
|
|
514
551
|
});
|
|
515
552
|
// Generate outbound trust token to authenticate this node to the provider.
|
|
516
553
|
const outboundToken = await this._trustComponent.generate(organizationIdentity, this._overrideTrustGeneratorType, { subject: { consumerPid, agreementId } });
|
|
517
|
-
//
|
|
518
|
-
|
|
519
|
-
|
|
554
|
+
// Call the provider's requestTransfer. When providerEndpoint resolves to a local origin this
|
|
555
|
+
// runs in-process (no loopback HTTP, no multi-instance client required); otherwise it uses a
|
|
556
|
+
// remote REST client. Mirrors how state-change deliveries route via withControlPlaneComponent.
|
|
557
|
+
const result = await this.withControlPlaneComponent(providerEndpoint, async (component) => component.requestTransfer(transferRequestMessage, outboundToken));
|
|
520
558
|
if (getJsonLdType(result) === DataspaceProtocolTransferProcessTypes.TransferError) {
|
|
521
559
|
const transferError = result;
|
|
522
560
|
throw new GeneralError(DataspaceControlPlaneService.CLASS_NAME, "transferRequestRejectedByProvider", {
|
|
@@ -614,6 +652,12 @@ export class DataspaceControlPlaneService {
|
|
|
614
652
|
// CONSUMER-RECEIVE: the provider POSTed a TransferStart to our callback. Use the dataAddress
|
|
615
653
|
// from the message (rebuilding it is provider work), mark our record STARTED, and notify.
|
|
616
654
|
if (role === TransferProcessRole.Consumer) {
|
|
655
|
+
// PUSH keeps the consumer's request-time inbox; format-less legacy records need dataAddress absent for format inference.
|
|
656
|
+
if (!Is.empty(message.dataAddress) &&
|
|
657
|
+
(entity.format === DataspaceTransferFormat.HttpDataPull ||
|
|
658
|
+
entity.format === DataspaceTransferFormat.HttpDataPost)) {
|
|
659
|
+
entity.dataAddress = message.dataAddress;
|
|
660
|
+
}
|
|
617
661
|
entity.state = DataspaceProtocolTransferProcessStateType.STARTED;
|
|
618
662
|
entity.dateModified = new Date();
|
|
619
663
|
await this._transferProcessStorage.set(this.modelToStorageEntity(entity));
|
|
@@ -736,7 +780,7 @@ export class DataspaceControlPlaneService {
|
|
|
736
780
|
}
|
|
737
781
|
}
|
|
738
782
|
// ----------------------------------------------------------------------------
|
|
739
|
-
// SHARED STATE MANAGEMENT OPERATIONS (
|
|
783
|
+
// SHARED STATE MANAGEMENT OPERATIONS (either side, except completeTransfer: consumer-initiated only)
|
|
740
784
|
// ----------------------------------------------------------------------------
|
|
741
785
|
/**
|
|
742
786
|
* Complete a Transfer Process.
|
|
@@ -814,6 +858,7 @@ export class DataspaceControlPlaneService {
|
|
|
814
858
|
await this._transferProcessStorage.set(this.modelToStorageEntity(entity));
|
|
815
859
|
throw teardownError;
|
|
816
860
|
}
|
|
861
|
+
await this._transferRetrievalStorage?.remove(entity.consumerPid);
|
|
817
862
|
// Completion is consumer-initiated (the auth above accepts only the consumer): a consumer→provider
|
|
818
863
|
// notification, so no provider→consumer delivery here — unlike suspend/terminate (either party).
|
|
819
864
|
if (role === TransferProcessRole.Consumer) {
|
|
@@ -1005,6 +1050,7 @@ export class DataspaceControlPlaneService {
|
|
|
1005
1050
|
await this._transferProcessStorage.set(this.modelToStorageEntity(entity));
|
|
1006
1051
|
throw teardownError;
|
|
1007
1052
|
}
|
|
1053
|
+
await this._transferRetrievalStorage?.remove(entity.consumerPid);
|
|
1008
1054
|
if (role === TransferProcessRole.Consumer) {
|
|
1009
1055
|
await this._internalTransferCallback.onStateChanged(entity.consumerPid, DataspaceProtocolTransferProcessStateType.TERMINATED);
|
|
1010
1056
|
// DSP reason is typed any[] — forward only the first entry since the callback
|
|
@@ -1068,6 +1114,67 @@ export class DataspaceControlPlaneService {
|
|
|
1068
1114
|
return transformToTransferError(error, { consumerPid: pid, providerPid: pid });
|
|
1069
1115
|
}
|
|
1070
1116
|
}
|
|
1117
|
+
/**
|
|
1118
|
+
* Query Transfer Processes by agreement id.
|
|
1119
|
+
* Results are limited to transfers where the authenticated caller is a party
|
|
1120
|
+
* (consumer or provider identity).
|
|
1121
|
+
* @param agreementId The agreement id to look up transfer processes for.
|
|
1122
|
+
* @param state Optional filter to a single transfer process state.
|
|
1123
|
+
* @param cursor Optional pagination cursor from a previous result page.
|
|
1124
|
+
* @param trustPayload Trust payload containing authorization information.
|
|
1125
|
+
* @returns The matching transfer processes and a pagination cursor when more pages exist, empty when none match.
|
|
1126
|
+
*/
|
|
1127
|
+
async queryDataTransfer(agreementId, state, cursor, trustPayload) {
|
|
1128
|
+
Guards.stringValue(DataspaceControlPlaneService.CLASS_NAME, "agreementId", agreementId);
|
|
1129
|
+
const trustInfo = await TrustHelper.verifyTrust(this._trustComponent, trustPayload, "queryDataTransfer");
|
|
1130
|
+
const conditions = [
|
|
1131
|
+
{
|
|
1132
|
+
property: "agreementId",
|
|
1133
|
+
value: agreementId,
|
|
1134
|
+
comparison: ComparisonOperator.Equals
|
|
1135
|
+
},
|
|
1136
|
+
{
|
|
1137
|
+
conditions: [
|
|
1138
|
+
{
|
|
1139
|
+
property: "consumerIdentity",
|
|
1140
|
+
value: trustInfo.identity,
|
|
1141
|
+
comparison: ComparisonOperator.Equals
|
|
1142
|
+
},
|
|
1143
|
+
{
|
|
1144
|
+
property: "providerIdentity",
|
|
1145
|
+
value: trustInfo.identity,
|
|
1146
|
+
comparison: ComparisonOperator.Equals
|
|
1147
|
+
}
|
|
1148
|
+
],
|
|
1149
|
+
logicalOperator: LogicalOperator.Or
|
|
1150
|
+
}
|
|
1151
|
+
];
|
|
1152
|
+
if (Is.stringValue(state)) {
|
|
1153
|
+
conditions.push({
|
|
1154
|
+
property: "state",
|
|
1155
|
+
value: state,
|
|
1156
|
+
comparison: ComparisonOperator.Equals
|
|
1157
|
+
});
|
|
1158
|
+
}
|
|
1159
|
+
const page = await this._transferProcessStorage.query({ conditions, logicalOperator: LogicalOperator.And }, undefined, undefined, cursor);
|
|
1160
|
+
const transfers = page.entities.map(pageEntity => this.storageEntityToModel(pageEntity));
|
|
1161
|
+
await this._loggingComponent?.log({
|
|
1162
|
+
level: "info",
|
|
1163
|
+
source: DataspaceControlPlaneService.CLASS_NAME,
|
|
1164
|
+
ts: Date.now(),
|
|
1165
|
+
message: "dataTransfersQueried",
|
|
1166
|
+
data: {
|
|
1167
|
+
agreementId,
|
|
1168
|
+
state,
|
|
1169
|
+
count: transfers.length,
|
|
1170
|
+
hasCursor: Boolean(page.cursor)
|
|
1171
|
+
}
|
|
1172
|
+
});
|
|
1173
|
+
return {
|
|
1174
|
+
transfers,
|
|
1175
|
+
cursor: page.cursor
|
|
1176
|
+
};
|
|
1177
|
+
}
|
|
1071
1178
|
// ============================================================================
|
|
1072
1179
|
// CONTRACT NEGOTIATION
|
|
1073
1180
|
// ============================================================================
|
|
@@ -1396,9 +1503,12 @@ export class DataspaceControlPlaneService {
|
|
|
1396
1503
|
* or generated.
|
|
1397
1504
|
* @param appId The dataspace app this dataset belongs to.
|
|
1398
1505
|
* @param dataset The dataset payload.
|
|
1506
|
+
* @param options Optional dataset settings.
|
|
1507
|
+
* @param options.transferIdleTimeoutMs Optional idle window (ms) overriding the node-level idle
|
|
1508
|
+
* policy for this dataset's PULL transfers; 0 disables it for this dataset.
|
|
1399
1509
|
* @returns The resolved dataset id.
|
|
1400
1510
|
*/
|
|
1401
|
-
async createAppDataset(id, appId, dataset) {
|
|
1511
|
+
async createAppDataset(id, appId, dataset, options) {
|
|
1402
1512
|
Guards.stringValue(DataspaceControlPlaneService.CLASS_NAME, "appId", appId);
|
|
1403
1513
|
Guards.object(DataspaceControlPlaneService.CLASS_NAME, "dataset", dataset);
|
|
1404
1514
|
let resolvedId;
|
|
@@ -1429,6 +1539,7 @@ export class DataspaceControlPlaneService {
|
|
|
1429
1539
|
tenantId: await this.resolveContextTenantId(),
|
|
1430
1540
|
appId,
|
|
1431
1541
|
dataset: ObjectHelper.omit(dataset, ["@id"]),
|
|
1542
|
+
transferIdleTimeoutMs: Coerce.integer(options?.transferIdleTimeoutMs),
|
|
1432
1543
|
dateCreated: now,
|
|
1433
1544
|
dateModified: now
|
|
1434
1545
|
};
|
|
@@ -1457,6 +1568,7 @@ export class DataspaceControlPlaneService {
|
|
|
1457
1568
|
id: entity.id,
|
|
1458
1569
|
appId: entity.appId,
|
|
1459
1570
|
dataset: this.restampDatasetId(entity.dataset, entity.id),
|
|
1571
|
+
transferIdleTimeoutMs: entity.transferIdleTimeoutMs,
|
|
1460
1572
|
dateCreated: entity.dateCreated,
|
|
1461
1573
|
dateModified: entity.dateModified
|
|
1462
1574
|
};
|
|
@@ -1478,6 +1590,7 @@ export class DataspaceControlPlaneService {
|
|
|
1478
1590
|
id: entity.id,
|
|
1479
1591
|
appId: entity.appId,
|
|
1480
1592
|
dataset: this.restampDatasetId(entity.dataset ?? {}, entity.id ?? ""),
|
|
1593
|
+
transferIdleTimeoutMs: entity.transferIdleTimeoutMs,
|
|
1481
1594
|
dateCreated: entity.dateCreated,
|
|
1482
1595
|
dateModified: entity.dateModified
|
|
1483
1596
|
}));
|
|
@@ -1491,9 +1604,12 @@ export class DataspaceControlPlaneService {
|
|
|
1491
1604
|
* @param id The stored dataset id.
|
|
1492
1605
|
* @param appId The dataspace app this dataset belongs to.
|
|
1493
1606
|
* @param dataset The dataset payload.
|
|
1607
|
+
* @param options Optional dataset settings.
|
|
1608
|
+
* @param options.transferIdleTimeoutMs Optional idle window (ms) overriding the node-level idle
|
|
1609
|
+
* policy for this dataset's PULL transfers; 0 disables it for this dataset.
|
|
1494
1610
|
* @returns A promise that resolves when the dataset has been updated in storage and the catalogue.
|
|
1495
1611
|
*/
|
|
1496
|
-
async updateAppDataset(id, appId, dataset) {
|
|
1612
|
+
async updateAppDataset(id, appId, dataset, options) {
|
|
1497
1613
|
Guards.stringValue(DataspaceControlPlaneService.CLASS_NAME, "id", id);
|
|
1498
1614
|
Guards.stringValue(DataspaceControlPlaneService.CLASS_NAME, "appId", appId);
|
|
1499
1615
|
Guards.object(DataspaceControlPlaneService.CLASS_NAME, "dataset", dataset);
|
|
@@ -1509,6 +1625,7 @@ export class DataspaceControlPlaneService {
|
|
|
1509
1625
|
...existing,
|
|
1510
1626
|
appId,
|
|
1511
1627
|
dataset: ObjectHelper.omit(dataset, ["@id"]),
|
|
1628
|
+
transferIdleTimeoutMs: Coerce.integer(options?.transferIdleTimeoutMs),
|
|
1512
1629
|
dateModified: new Date().toISOString()
|
|
1513
1630
|
};
|
|
1514
1631
|
// Side effect first, primary storage last
|
|
@@ -1538,7 +1655,7 @@ export class DataspaceControlPlaneService {
|
|
|
1538
1655
|
datasetId: id,
|
|
1539
1656
|
tenantId: existing.tenantId ?? "",
|
|
1540
1657
|
catalogErrorCode: removeResult.code
|
|
1541
|
-
});
|
|
1658
|
+
}, BaseError.expand(removeResult.reason));
|
|
1542
1659
|
}
|
|
1543
1660
|
await this._dataspaceAppDatasetStorage.remove(id);
|
|
1544
1661
|
await MetricHelper.metricIncrement(this._telemetryComponent, DataspaceControlPlaneMetricIds.AppDatasetsDeleted);
|
|
@@ -1698,6 +1815,209 @@ export class DataspaceControlPlaneService {
|
|
|
1698
1815
|
}
|
|
1699
1816
|
});
|
|
1700
1817
|
}
|
|
1818
|
+
/**
|
|
1819
|
+
* Apply the idle lifecycle policy to Provider-role STARTED transfers, transitioning through
|
|
1820
|
+
* the standard terminateTransfer path.
|
|
1821
|
+
* @returns A promise that resolves when all matching transfers have been processed.
|
|
1822
|
+
* @internal
|
|
1823
|
+
*/
|
|
1824
|
+
async applyProviderTransferPolicies() {
|
|
1825
|
+
const now = Date.now();
|
|
1826
|
+
// Runs per tenant so the storage access inherits the correct [Node, Tenant] partition.
|
|
1827
|
+
await this._platformComponent.execute(async () => {
|
|
1828
|
+
if (Is.integer(this._providerTransferIdleTimeoutMs) &&
|
|
1829
|
+
Is.empty(this._transferRetrievalStorage)) {
|
|
1830
|
+
await this._loggingComponent?.log({
|
|
1831
|
+
level: "warn",
|
|
1832
|
+
source: DataspaceControlPlaneService.CLASS_NAME,
|
|
1833
|
+
ts: Date.now(),
|
|
1834
|
+
message: "providerTransferIdleStorageMissing"
|
|
1835
|
+
});
|
|
1836
|
+
}
|
|
1837
|
+
const providerStarted = [];
|
|
1838
|
+
let cursor;
|
|
1839
|
+
do {
|
|
1840
|
+
// The localRole condition also skips legacy records without a persisted localRole.
|
|
1841
|
+
const page = await this._transferProcessStorage.query({
|
|
1842
|
+
conditions: [
|
|
1843
|
+
{
|
|
1844
|
+
property: "state",
|
|
1845
|
+
value: DataspaceProtocolTransferProcessStateType.STARTED,
|
|
1846
|
+
comparison: ComparisonOperator.Equals
|
|
1847
|
+
},
|
|
1848
|
+
{
|
|
1849
|
+
property: "localRole",
|
|
1850
|
+
value: TransferProcessRole.Provider,
|
|
1851
|
+
comparison: ComparisonOperator.Equals
|
|
1852
|
+
}
|
|
1853
|
+
],
|
|
1854
|
+
logicalOperator: LogicalOperator.And
|
|
1855
|
+
}, undefined, undefined, cursor);
|
|
1856
|
+
providerStarted.push(...page.entities);
|
|
1857
|
+
cursor = page.cursor;
|
|
1858
|
+
} while (Is.stringValue(cursor));
|
|
1859
|
+
const datasetIdleCache = new Map();
|
|
1860
|
+
let terminated = 0;
|
|
1861
|
+
for (const transfer of providerStarted) {
|
|
1862
|
+
if (!Is.stringValue(transfer.providerIdentity)) {
|
|
1863
|
+
// Cannot be transitioned without a provider identity, so remove it directly.
|
|
1864
|
+
await this._transferProcessStorage.remove(transfer.consumerPid);
|
|
1865
|
+
await this._loggingComponent?.log({
|
|
1866
|
+
level: "warn",
|
|
1867
|
+
source: DataspaceControlPlaneService.CLASS_NAME,
|
|
1868
|
+
ts: Date.now(),
|
|
1869
|
+
message: "providerTransferPolicyNoProviderIdentity",
|
|
1870
|
+
data: { consumerPid: transfer.consumerPid }
|
|
1871
|
+
});
|
|
1872
|
+
}
|
|
1873
|
+
else if (await this.terminateIdleProviderTransfer(transfer, now, datasetIdleCache)) {
|
|
1874
|
+
terminated++;
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
if (terminated > 0) {
|
|
1878
|
+
await this._loggingComponent?.log({
|
|
1879
|
+
level: "info",
|
|
1880
|
+
source: DataspaceControlPlaneService.CLASS_NAME,
|
|
1881
|
+
ts: Date.now(),
|
|
1882
|
+
message: "providerTransferPolicySweepComplete",
|
|
1883
|
+
data: { terminated }
|
|
1884
|
+
});
|
|
1885
|
+
}
|
|
1886
|
+
});
|
|
1887
|
+
}
|
|
1888
|
+
/**
|
|
1889
|
+
* Terminate a Provider-side PULL transfer idle beyond its effective window (dataset override,
|
|
1890
|
+
* falling back to the node config; 0 disables). Activity is the later of the last state change
|
|
1891
|
+
* and the last successful retrieval. Notifies the registered callbacks (onTimeout, falling
|
|
1892
|
+
* back to onFailed).
|
|
1893
|
+
* @param transfer The transfer process entity.
|
|
1894
|
+
* @param now The sweep timestamp.
|
|
1895
|
+
* @param datasetIdleCache Per-sweep cache of dataset overrides.
|
|
1896
|
+
* @returns True when the transfer was terminated.
|
|
1897
|
+
* @internal
|
|
1898
|
+
*/
|
|
1899
|
+
async terminateIdleProviderTransfer(transfer, now, datasetIdleCache) {
|
|
1900
|
+
// Retrieval activity is invisible without the storage, so skip rather than cut off
|
|
1901
|
+
// consumers that are actively pulling.
|
|
1902
|
+
if (this.inferTransferFormat(transfer) !== DataspaceTransferFormat.HttpDataPull ||
|
|
1903
|
+
Is.empty(this._transferRetrievalStorage)) {
|
|
1904
|
+
return false;
|
|
1905
|
+
}
|
|
1906
|
+
let datasetIdleMs;
|
|
1907
|
+
if (datasetIdleCache.has(transfer.datasetId)) {
|
|
1908
|
+
datasetIdleMs = datasetIdleCache.get(transfer.datasetId);
|
|
1909
|
+
}
|
|
1910
|
+
else {
|
|
1911
|
+
const dataset = await this._dataspaceAppDatasetStorage.get(transfer.datasetId);
|
|
1912
|
+
datasetIdleMs = dataset?.transferIdleTimeoutMs;
|
|
1913
|
+
datasetIdleCache.set(transfer.datasetId, datasetIdleMs);
|
|
1914
|
+
}
|
|
1915
|
+
const idleMs = datasetIdleMs ?? this._providerTransferIdleTimeoutMs;
|
|
1916
|
+
if (!Is.integer(idleMs) || idleMs === 0) {
|
|
1917
|
+
return false;
|
|
1918
|
+
}
|
|
1919
|
+
const retrieval = await this._transferRetrievalStorage.get(transfer.consumerPid);
|
|
1920
|
+
const lastRetrieved = retrieval?.dateLastRetrieved;
|
|
1921
|
+
const lastActivity = Math.max(new Date(transfer.dateModified).getTime(), Is.stringValue(lastRetrieved) ? new Date(lastRetrieved).getTime() : 0);
|
|
1922
|
+
if (now - lastActivity <= idleMs) {
|
|
1923
|
+
return false;
|
|
1924
|
+
}
|
|
1925
|
+
const consumerPid = transfer.consumerPid;
|
|
1926
|
+
await this._loggingComponent?.log({
|
|
1927
|
+
level: "warn",
|
|
1928
|
+
source: DataspaceControlPlaneService.CLASS_NAME,
|
|
1929
|
+
ts: Date.now(),
|
|
1930
|
+
message: "providerTransferIdleTimedOut",
|
|
1931
|
+
data: {
|
|
1932
|
+
consumerPid,
|
|
1933
|
+
providerPid: transfer.providerPid,
|
|
1934
|
+
idleTimeoutMs: idleMs
|
|
1935
|
+
}
|
|
1936
|
+
});
|
|
1937
|
+
for (const [key, cb] of this._transferCallbacks.entries()) {
|
|
1938
|
+
const onTimeout = cb.onTimeout?.bind(cb);
|
|
1939
|
+
const onFailed = cb.onFailed?.bind(cb);
|
|
1940
|
+
try {
|
|
1941
|
+
if (Is.function(onTimeout)) {
|
|
1942
|
+
await onTimeout(consumerPid);
|
|
1943
|
+
}
|
|
1944
|
+
else if (Is.function(onFailed)) {
|
|
1945
|
+
await onFailed(consumerPid, TransferTerminationCode.IdleTimeout);
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
catch (error) {
|
|
1949
|
+
await this._loggingComponent?.log({
|
|
1950
|
+
level: "error",
|
|
1951
|
+
source: DataspaceControlPlaneService.CLASS_NAME,
|
|
1952
|
+
ts: Date.now(),
|
|
1953
|
+
message: "transferCallbackError",
|
|
1954
|
+
data: {
|
|
1955
|
+
key,
|
|
1956
|
+
consumerPid,
|
|
1957
|
+
method: Is.function(onTimeout) ? "onTimeout" : "onFailed",
|
|
1958
|
+
error
|
|
1959
|
+
}
|
|
1960
|
+
});
|
|
1961
|
+
}
|
|
1962
|
+
}
|
|
1963
|
+
return this.terminateProviderPolicyTransfer(transfer, TransferTerminationCode.IdleTimeout);
|
|
1964
|
+
}
|
|
1965
|
+
/**
|
|
1966
|
+
* Terminate a Provider-side transfer for a lifecycle policy: a self-issued provider token and
|
|
1967
|
+
* the owning organization context (the per-tenant execute establishes only the tenant), then
|
|
1968
|
+
* the standard terminateTransfer path. Failures are logged, never thrown.
|
|
1969
|
+
* @param transfer The transfer process entity.
|
|
1970
|
+
* @param code The termination code, also forwarded as the reason.
|
|
1971
|
+
* @returns True when the transfer was terminated.
|
|
1972
|
+
* @internal
|
|
1973
|
+
*/
|
|
1974
|
+
async terminateProviderPolicyTransfer(transfer, code) {
|
|
1975
|
+
const consumerPid = transfer.consumerPid;
|
|
1976
|
+
if (!Is.stringValue(transfer.providerIdentity)) {
|
|
1977
|
+
return false;
|
|
1978
|
+
}
|
|
1979
|
+
try {
|
|
1980
|
+
const selfToken = await this._trustComponent.generate(transfer.providerIdentity, this._overrideTrustGeneratorType, {
|
|
1981
|
+
subject: {
|
|
1982
|
+
consumerPid,
|
|
1983
|
+
providerPid: transfer.providerPid,
|
|
1984
|
+
agreementId: transfer.agreementId
|
|
1985
|
+
}
|
|
1986
|
+
});
|
|
1987
|
+
const terminationMessage = {
|
|
1988
|
+
"@context": [DataspaceProtocolContexts.Context],
|
|
1989
|
+
"@type": DataspaceProtocolTransferProcessTypes.TransferTerminationMessage,
|
|
1990
|
+
consumerPid,
|
|
1991
|
+
providerPid: transfer.providerPid,
|
|
1992
|
+
code,
|
|
1993
|
+
reason: [code]
|
|
1994
|
+
};
|
|
1995
|
+
const contextIds = await ContextIdStore.getContextIds();
|
|
1996
|
+
const result = await ContextIdStore.run({ ...contextIds, [ContextIdKeys.Organization]: transfer.organizationIdentity }, async () => this.terminateTransfer(terminationMessage, selfToken));
|
|
1997
|
+
if (getJsonLdType(result) === DataspaceProtocolTransferProcessTypes.TransferError) {
|
|
1998
|
+
await this._loggingComponent?.log({
|
|
1999
|
+
level: "error",
|
|
2000
|
+
source: DataspaceControlPlaneService.CLASS_NAME,
|
|
2001
|
+
ts: Date.now(),
|
|
2002
|
+
message: "providerTransferTerminateFailed",
|
|
2003
|
+
data: { consumerPid, code: result.code }
|
|
2004
|
+
});
|
|
2005
|
+
return false;
|
|
2006
|
+
}
|
|
2007
|
+
return true;
|
|
2008
|
+
}
|
|
2009
|
+
catch (error) {
|
|
2010
|
+
await this._loggingComponent?.log({
|
|
2011
|
+
level: "error",
|
|
2012
|
+
source: DataspaceControlPlaneService.CLASS_NAME,
|
|
2013
|
+
ts: Date.now(),
|
|
2014
|
+
message: "providerTransferTerminateFailed",
|
|
2015
|
+
error: BaseError.fromError(error),
|
|
2016
|
+
data: { consumerPid }
|
|
2017
|
+
});
|
|
2018
|
+
return false;
|
|
2019
|
+
}
|
|
2020
|
+
}
|
|
1701
2021
|
/**
|
|
1702
2022
|
* Perform the provider-side start of a just-requested transfer (build dataAddress, transition to
|
|
1703
2023
|
* STARTED, deliver to the consumer). Only invoked when the request opted into auto-start, so there is no
|
|
@@ -2348,18 +2668,15 @@ export class DataspaceControlPlaneService {
|
|
|
2348
2668
|
const tenantContextIds = { ...contextIds, [ContextIdKeys.Tenant]: appDataset.tenantId };
|
|
2349
2669
|
await ContextIdStore.run(tenantContextIds, async () => {
|
|
2350
2670
|
const datasetPayload = this.restampDatasetId(appDataset.dataset, appDataset.id);
|
|
2351
|
-
const
|
|
2352
|
-
const
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
catalogErrorCode: publishResult.code
|
|
2361
|
-
});
|
|
2362
|
-
}
|
|
2671
|
+
const dataset = await this.populateDefaults(datasetPayload);
|
|
2672
|
+
const publishResult = await this._federatedCatalogueComponent.set(dataset, localTrustPayload);
|
|
2673
|
+
if (isCatalogError(publishResult)) {
|
|
2674
|
+
throw new GeneralError(DataspaceControlPlaneService.CLASS_NAME, "datasetPublishFailed", {
|
|
2675
|
+
datasetId: appDataset.id,
|
|
2676
|
+
appId: appDataset.appId,
|
|
2677
|
+
tenantId: appDataset.tenantId ?? "",
|
|
2678
|
+
catalogErrorCode: publishResult.code
|
|
2679
|
+
}, BaseError.expand(publishResult.reason));
|
|
2363
2680
|
}
|
|
2364
2681
|
});
|
|
2365
2682
|
}
|