@twin.org/dataspace-control-plane-service 0.9.1 → 0.9.2-next.1

package/dist/es/dataspaceControlPlaneService.js

@@ -2,11 +2,10 @@
 // SPDX-License-Identifier: Apache-2.0.
 import { HttpContextIdKeys, HttpUrlHelper } from "@twin.org/api-models";
 import { ContextIdHelper, ContextIdKeys, ContextIdStore } from "@twin.org/context";
-import { AlreadyExistsError, ArrayHelper, BaseError, ComponentFactory, Converter, GeneralError, Guards, Is, NotFoundError, ObjectHelper, RandomHelper, StringHelper, UnauthorizedError, Url, Urn, ValidationError } from "@twin.org/core";
+import { AlreadyExistsError, ArrayHelper, BaseError, Coerce, ComponentFactory, Converter, Factory, GeneralError, Guards, Is, NotFoundError, ObjectHelper, RandomHelper, StringHelper, UnauthorizedError, Url, Urn, ValidationError } from "@twin.org/core";
 import { JsonLdHelper } from "@twin.org/data-json-ld";
-import { DataspaceControlPlaneMetricIds, DataspaceControlPlaneMetrics, DataspaceTransferFormat, TransferProcessRole, getJsonLdId, getJsonLdType } from "@twin.org/dataspace-models";
-import { EngineCoreFactory } from "@twin.org/engine-models";
-import { ComparisonOperator } from "@twin.org/entity";
+import { DataspaceControlPlaneMetricIds, DataspaceControlPlaneMetrics, DataspaceTransferFormat, TransferProcessRole, TransferTerminationCode, getJsonLdId, getJsonLdType } from "@twin.org/dataspace-models";
+import { ComparisonOperator, LogicalOperator } from "@twin.org/entity";
 import { EntityStorageConnectorFactory } from "@twin.org/entity-storage-models";
 import { OdrlPolicyHelper, PolicyRequesterFactory } from "@twin.org/rights-management-models";
 import { DataspaceProtocolContexts, DataspaceProtocolContractNegotiationTypes, DataspaceProtocolHelper, DataspaceProtocolTransferProcessStateType, DataspaceProtocolTransferProcessTypes, DataspaceProtocolVersionBindingType } from "@twin.org/standards-dataspace-protocol";
@@ -49,6 +48,11 @@ export class DataspaceControlPlaneService {
      * @internal
      */
     static _STALLED_TRANSFER_THRESHOLD_MS = 30 * 60 * 1000;
+    /**
+     * Default provider transfer policy sweep interval in milliseconds (5 minutes).
+     * @internal
+     */
+    static _PROVIDER_TRANSFER_POLICY_SWEEP_INTERVAL_MS = 5 * 60 * 1000;
     /**
      * The logging component.
      * @internal
@@ -171,6 +175,22 @@ export class DataspaceControlPlaneService {
      * @internal
      */
     _stalledTransferTimeoutMs;
+    /**
+     * Idle window (ms) for Provider-side STARTED PULL transfers; undefined disables the policy
+     * node-wide.
+     * @internal
+     */
+    _providerTransferIdleTimeoutMs;
+    /**
+     * Interval (ms) at which the provider transfer policy sweep runs.
+     * @internal
+     */
+    _providerTransferPolicySweepIntervalMs;
+    /**
+     * Storage for transfer retrievals; undefined when the hosting engine does not register it.
+     * @internal
+     */
+    _transferRetrievalStorage;
     /**
      * Create a new instance of DataspaceControlPlaneService.
      * @param options The options for the service.
@@ -204,6 +224,11 @@ export class DataspaceControlPlaneService {
         this._stalledTransferTimeoutMs =
             options?.config?.stalledTransferTimeoutMs ??
                 DataspaceControlPlaneService._STALLED_TRANSFER_THRESHOLD_MS;
+        this._providerTransferIdleTimeoutMs = options?.config?.providerTransferIdleTimeoutMs;
+        this._providerTransferPolicySweepIntervalMs =
+            options?.config?.providerTransferPolicySweepIntervalMs ??
+                DataspaceControlPlaneService._PROVIDER_TRANSFER_POLICY_SWEEP_INTERVAL_MS;
+        this._transferRetrievalStorage = EntityStorageConnectorFactory.getIfExists(options?.transferRetrievalEntityStorageType ?? "transfer-retrieval");
         this._taskScheduler = ComponentFactory.getIfExists(options?.taskSchedulerComponentType ?? "task-scheduler");
         this._platformComponent = ComponentFactory.get(options?.platformComponentType ?? "platform");
         this._telemetryComponent = ComponentFactory.getIfExists(options?.telemetryComponentType);
@@ -269,10 +294,10 @@ export class DataspaceControlPlaneService {
      * @returns A promise that resolves when the federated catalogue is populated and the cleanup task is scheduled.
      */
     async start(nodeLoggingComponentType) {
-        await MetricHelper.createMetrics(this._telemetryComponent, DataspaceControlPlaneMetrics);
-        const engine = EngineCoreFactory.getIfExists("engine");
-        // Skip if no engine exists OR if this is a clone instance
-        if (Is.empty(engine) || engine.isClone()) {
+        const isCloneOrNoEngine = Factory.getFactory("engine-core")
+            ?.getIfExists("engine")
+            ?.isClone() ?? true;
+        if (isCloneOrNoEngine) {
             await this._loggingComponent?.log({
                 level: "debug",
                 ts: Date.now(),
@@ -281,6 +306,7 @@ export class DataspaceControlPlaneService {
             });
             return;
         }
+        await MetricHelper.createMetrics(this._telemetryComponent, DataspaceControlPlaneMetrics);
         if (this._taskScheduler) {
             await this._taskScheduler.addTask("control-plane-negotiation-cleanup", [
                 {
@@ -298,6 +324,15 @@ export class DataspaceControlPlaneService {
             ], async () => {
                 await this.cleanupStalledTransfers();
             });
+            await this._taskScheduler.addTask("control-plane-transfer-policy", [
+                {
+                    nextTriggerTime: Date.now(),
+                    // The scheduler has minute granularity; floor at one minute.
+                    intervalMinutes: Math.max(1, Math.round(this._providerTransferPolicySweepIntervalMs / 60000))
+                }
+            ], async () => {
+                await this.applyProviderTransferPolicies();
+            });
         }
     }
     /**
@@ -310,6 +345,7 @@ export class DataspaceControlPlaneService {
         if (this._taskScheduler) {
             await this._taskScheduler.removeTask("control-plane-negotiation-cleanup");
             await this._taskScheduler.removeTask("control-plane-transfer-cleanup");
+            await this._taskScheduler.removeTask("control-plane-transfer-policy");
         }
     }
     // ----------------------------------------------------------------------------
@@ -616,6 +652,12 @@ export class DataspaceControlPlaneService {
             // CONSUMER-RECEIVE: the provider POSTed a TransferStart to our callback. Use the dataAddress
             // from the message (rebuilding it is provider work), mark our record STARTED, and notify.
             if (role === TransferProcessRole.Consumer) {
+                // PUSH keeps the consumer's request-time inbox; format-less legacy records need dataAddress absent for format inference.
+                if (!Is.empty(message.dataAddress) &&
+                    (entity.format === DataspaceTransferFormat.HttpDataPull ||
+                        entity.format === DataspaceTransferFormat.HttpDataPost)) {
+                    entity.dataAddress = message.dataAddress;
+                }
                 entity.state = DataspaceProtocolTransferProcessStateType.STARTED;
                 entity.dateModified = new Date();
                 await this._transferProcessStorage.set(this.modelToStorageEntity(entity));
@@ -738,7 +780,7 @@ export class DataspaceControlPlaneService {
         }
     }
     // ----------------------------------------------------------------------------
-    // SHARED STATE MANAGEMENT OPERATIONS (Either Side)
+    // SHARED STATE MANAGEMENT OPERATIONS (either side, except completeTransfer: consumer-initiated only)
     // ----------------------------------------------------------------------------
     /**
      * Complete a Transfer Process.
@@ -816,6 +858,7 @@ export class DataspaceControlPlaneService {
                 await this._transferProcessStorage.set(this.modelToStorageEntity(entity));
                 throw teardownError;
             }
+            await this._transferRetrievalStorage?.remove(entity.consumerPid);
             // Completion is consumer-initiated (the auth above accepts only the consumer): a consumer→provider
             // notification, so no provider→consumer delivery here — unlike suspend/terminate (either party).
             if (role === TransferProcessRole.Consumer) {
@@ -1007,6 +1050,7 @@ export class DataspaceControlPlaneService {
                 await this._transferProcessStorage.set(this.modelToStorageEntity(entity));
                 throw teardownError;
             }
+            await this._transferRetrievalStorage?.remove(entity.consumerPid);
             if (role === TransferProcessRole.Consumer) {
                 await this._internalTransferCallback.onStateChanged(entity.consumerPid, DataspaceProtocolTransferProcessStateType.TERMINATED);
                 // DSP reason is typed any[] — forward only the first entry since the callback
@@ -1070,6 +1114,67 @@ export class DataspaceControlPlaneService {
             return transformToTransferError(error, { consumerPid: pid, providerPid: pid });
         }
     }
+    /**
+     * Query Transfer Processes by agreement id.
+     * Results are limited to transfers where the authenticated caller is a party
+     * (consumer or provider identity).
+     * @param agreementId The agreement id to look up transfer processes for.
+     * @param state Optional filter to a single transfer process state.
+     * @param cursor Optional pagination cursor from a previous result page.
+     * @param trustPayload Trust payload containing authorization information.
+     * @returns The matching transfer processes and a pagination cursor when more pages exist, empty when none match.
+     */
+    async queryDataTransfer(agreementId, state, cursor, trustPayload) {
+        Guards.stringValue(DataspaceControlPlaneService.CLASS_NAME, "agreementId", agreementId);
+        const trustInfo = await TrustHelper.verifyTrust(this._trustComponent, trustPayload, "queryDataTransfer");
+        const conditions = [
+            {
+                property: "agreementId",
+                value: agreementId,
+                comparison: ComparisonOperator.Equals
+            },
+            {
+                conditions: [
+                    {
+                        property: "consumerIdentity",
+                        value: trustInfo.identity,
+                        comparison: ComparisonOperator.Equals
+                    },
+                    {
+                        property: "providerIdentity",
+                        value: trustInfo.identity,
+                        comparison: ComparisonOperator.Equals
+                    }
+                ],
+                logicalOperator: LogicalOperator.Or
+            }
+        ];
+        if (Is.stringValue(state)) {
+            conditions.push({
+                property: "state",
+                value: state,
+                comparison: ComparisonOperator.Equals
+            });
+        }
+        const page = await this._transferProcessStorage.query({ conditions, logicalOperator: LogicalOperator.And }, undefined, undefined, cursor);
+        const transfers = page.entities.map(pageEntity => this.storageEntityToModel(pageEntity));
+        await this._loggingComponent?.log({
+            level: "info",
+            source: DataspaceControlPlaneService.CLASS_NAME,
+            ts: Date.now(),
+            message: "dataTransfersQueried",
+            data: {
+                agreementId,
+                state,
+                count: transfers.length,
+                hasCursor: Boolean(page.cursor)
+            }
+        });
+        return {
+            transfers,
+            cursor: page.cursor
+        };
+    }
     // ============================================================================
     // CONTRACT NEGOTIATION
     // ============================================================================
@@ -1398,9 +1503,12 @@ export class DataspaceControlPlaneService {
      * or generated.
      * @param appId The dataspace app this dataset belongs to.
      * @param dataset The dataset payload.
+     * @param options Optional dataset settings.
+     * @param options.transferIdleTimeoutMs Optional idle window (ms) overriding the node-level idle
+     * policy for this dataset's PULL transfers; 0 disables it for this dataset.
      * @returns The resolved dataset id.
      */
-    async createAppDataset(id, appId, dataset) {
+    async createAppDataset(id, appId, dataset, options) {
         Guards.stringValue(DataspaceControlPlaneService.CLASS_NAME, "appId", appId);
         Guards.object(DataspaceControlPlaneService.CLASS_NAME, "dataset", dataset);
         let resolvedId;
@@ -1431,6 +1539,7 @@ export class DataspaceControlPlaneService {
             tenantId: await this.resolveContextTenantId(),
             appId,
             dataset: ObjectHelper.omit(dataset, ["@id"]),
+            transferIdleTimeoutMs: Coerce.integer(options?.transferIdleTimeoutMs),
             dateCreated: now,
             dateModified: now
         };
@@ -1459,6 +1568,7 @@ export class DataspaceControlPlaneService {
             id: entity.id,
             appId: entity.appId,
             dataset: this.restampDatasetId(entity.dataset, entity.id),
+            transferIdleTimeoutMs: entity.transferIdleTimeoutMs,
             dateCreated: entity.dateCreated,
             dateModified: entity.dateModified
         };
@@ -1480,6 +1590,7 @@ export class DataspaceControlPlaneService {
             id: entity.id,
             appId: entity.appId,
             dataset: this.restampDatasetId(entity.dataset ?? {}, entity.id ?? ""),
+            transferIdleTimeoutMs: entity.transferIdleTimeoutMs,
             dateCreated: entity.dateCreated,
             dateModified: entity.dateModified
         }));
@@ -1493,9 +1604,12 @@ export class DataspaceControlPlaneService {
      * @param id The stored dataset id.
      * @param appId The dataspace app this dataset belongs to.
      * @param dataset The dataset payload.
+     * @param options Optional dataset settings.
+     * @param options.transferIdleTimeoutMs Optional idle window (ms) overriding the node-level idle
+     * policy for this dataset's PULL transfers; 0 disables it for this dataset.
      * @returns A promise that resolves when the dataset has been updated in storage and the catalogue.
      */
-    async updateAppDataset(id, appId, dataset) {
+    async updateAppDataset(id, appId, dataset, options) {
         Guards.stringValue(DataspaceControlPlaneService.CLASS_NAME, "id", id);
         Guards.stringValue(DataspaceControlPlaneService.CLASS_NAME, "appId", appId);
         Guards.object(DataspaceControlPlaneService.CLASS_NAME, "dataset", dataset);
@@ -1511,6 +1625,7 @@ export class DataspaceControlPlaneService {
             ...existing,
             appId,
             dataset: ObjectHelper.omit(dataset, ["@id"]),
+            transferIdleTimeoutMs: Coerce.integer(options?.transferIdleTimeoutMs),
             dateModified: new Date().toISOString()
         };
         // Side effect first, primary storage last
@@ -1700,6 +1815,209 @@ export class DataspaceControlPlaneService {
             }
         });
     }
+    /**
+     * Apply the idle lifecycle policy to Provider-role STARTED transfers, transitioning through
+     * the standard terminateTransfer path.
+     * @returns A promise that resolves when all matching transfers have been processed.
+     * @internal
+     */
+    async applyProviderTransferPolicies() {
+        const now = Date.now();
+        // Runs per tenant so the storage access inherits the correct [Node, Tenant] partition.
+        await this._platformComponent.execute(async () => {
+            if (Is.integer(this._providerTransferIdleTimeoutMs) &&
+                Is.empty(this._transferRetrievalStorage)) {
+                await this._loggingComponent?.log({
+                    level: "warn",
+                    source: DataspaceControlPlaneService.CLASS_NAME,
+                    ts: Date.now(),
+                    message: "providerTransferIdleStorageMissing"
+                });
+            }
+            const providerStarted = [];
+            let cursor;
+            do {
+                // The localRole condition also skips legacy records without a persisted localRole.
+                const page = await this._transferProcessStorage.query({
+                    conditions: [
+                        {
+                            property: "state",
+                            value: DataspaceProtocolTransferProcessStateType.STARTED,
+                            comparison: ComparisonOperator.Equals
+                        },
+                        {
+                            property: "localRole",
+                            value: TransferProcessRole.Provider,
+                            comparison: ComparisonOperator.Equals
+                        }
+                    ],
+                    logicalOperator: LogicalOperator.And
+                }, undefined, undefined, cursor);
+                providerStarted.push(...page.entities);
+                cursor = page.cursor;
+            } while (Is.stringValue(cursor));
+            const datasetIdleCache = new Map();
+            let terminated = 0;
+            for (const transfer of providerStarted) {
+                if (!Is.stringValue(transfer.providerIdentity)) {
+                    // Cannot be transitioned without a provider identity, so remove it directly.
+                    await this._transferProcessStorage.remove(transfer.consumerPid);
+                    await this._loggingComponent?.log({
+                        level: "warn",
+                        source: DataspaceControlPlaneService.CLASS_NAME,
+                        ts: Date.now(),
+                        message: "providerTransferPolicyNoProviderIdentity",
+                        data: { consumerPid: transfer.consumerPid }
+                    });
+                }
+                else if (await this.terminateIdleProviderTransfer(transfer, now, datasetIdleCache)) {
+                    terminated++;
+                }
+            }
+            if (terminated > 0) {
+                await this._loggingComponent?.log({
+                    level: "info",
+                    source: DataspaceControlPlaneService.CLASS_NAME,
+                    ts: Date.now(),
+                    message: "providerTransferPolicySweepComplete",
+                    data: { terminated }
+                });
+            }
+        });
+    }
+    /**
+     * Terminate a Provider-side PULL transfer idle beyond its effective window (dataset override,
+     * falling back to the node config; 0 disables). Activity is the later of the last state change
+     * and the last successful retrieval. Notifies the registered callbacks (onTimeout, falling
+     * back to onFailed).
+     * @param transfer The transfer process entity.
+     * @param now The sweep timestamp.
+     * @param datasetIdleCache Per-sweep cache of dataset overrides.
+     * @returns True when the transfer was terminated.
+     * @internal
+     */
+    async terminateIdleProviderTransfer(transfer, now, datasetIdleCache) {
+        // Retrieval activity is invisible without the storage, so skip rather than cut off
+        // consumers that are actively pulling.
+        if (this.inferTransferFormat(transfer) !== DataspaceTransferFormat.HttpDataPull ||
+            Is.empty(this._transferRetrievalStorage)) {
+            return false;
+        }
+        let datasetIdleMs;
+        if (datasetIdleCache.has(transfer.datasetId)) {
+            datasetIdleMs = datasetIdleCache.get(transfer.datasetId);
+        }
+        else {
+            const dataset = await this._dataspaceAppDatasetStorage.get(transfer.datasetId);
+            datasetIdleMs = dataset?.transferIdleTimeoutMs;
+            datasetIdleCache.set(transfer.datasetId, datasetIdleMs);
+        }
+        const idleMs = datasetIdleMs ?? this._providerTransferIdleTimeoutMs;
+        if (!Is.integer(idleMs) || idleMs === 0) {
+            return false;
+        }
+        const retrieval = await this._transferRetrievalStorage.get(transfer.consumerPid);
+        const lastRetrieved = retrieval?.dateLastRetrieved;
+        const lastActivity = Math.max(new Date(transfer.dateModified).getTime(), Is.stringValue(lastRetrieved) ? new Date(lastRetrieved).getTime() : 0);
+        if (now - lastActivity <= idleMs) {
+            return false;
+        }
+        const consumerPid = transfer.consumerPid;
+        await this._loggingComponent?.log({
+            level: "warn",
+            source: DataspaceControlPlaneService.CLASS_NAME,
+            ts: Date.now(),
+            message: "providerTransferIdleTimedOut",
+            data: {
+                consumerPid,
+                providerPid: transfer.providerPid,
+                idleTimeoutMs: idleMs
+            }
+        });
+        for (const [key, cb] of this._transferCallbacks.entries()) {
+            const onTimeout = cb.onTimeout?.bind(cb);
+            const onFailed = cb.onFailed?.bind(cb);
+            try {
+                if (Is.function(onTimeout)) {
+                    await onTimeout(consumerPid);
+                }
+                else if (Is.function(onFailed)) {
+                    await onFailed(consumerPid, TransferTerminationCode.IdleTimeout);
+                }
+            }
+            catch (error) {
+                await this._loggingComponent?.log({
+                    level: "error",
+                    source: DataspaceControlPlaneService.CLASS_NAME,
+                    ts: Date.now(),
+                    message: "transferCallbackError",
+                    data: {
+                        key,
+                        consumerPid,
+                        method: Is.function(onTimeout) ? "onTimeout" : "onFailed",
+                        error
+                    }
+                });
+            }
+        }
+        return this.terminateProviderPolicyTransfer(transfer, TransferTerminationCode.IdleTimeout);
+    }
+    /**
+     * Terminate a Provider-side transfer for a lifecycle policy: a self-issued provider token and
+     * the owning organization context (the per-tenant execute establishes only the tenant), then
+     * the standard terminateTransfer path. Failures are logged, never thrown.
+     * @param transfer The transfer process entity.
+     * @param code The termination code, also forwarded as the reason.
+     * @returns True when the transfer was terminated.
+     * @internal
+     */
+    async terminateProviderPolicyTransfer(transfer, code) {
+        const consumerPid = transfer.consumerPid;
+        if (!Is.stringValue(transfer.providerIdentity)) {
+            return false;
+        }
+        try {
+            const selfToken = await this._trustComponent.generate(transfer.providerIdentity, this._overrideTrustGeneratorType, {
+                subject: {
+                    consumerPid,
+                    providerPid: transfer.providerPid,
+                    agreementId: transfer.agreementId
+                }
+            });
+            const terminationMessage = {
+                "@context": [DataspaceProtocolContexts.Context],
+                "@type": DataspaceProtocolTransferProcessTypes.TransferTerminationMessage,
+                consumerPid,
+                providerPid: transfer.providerPid,
+                code,
+                reason: [code]
+            };
+            const contextIds = await ContextIdStore.getContextIds();
+            const result = await ContextIdStore.run({ ...contextIds, [ContextIdKeys.Organization]: transfer.organizationIdentity }, async () => this.terminateTransfer(terminationMessage, selfToken));
+            if (getJsonLdType(result) === DataspaceProtocolTransferProcessTypes.TransferError) {
+                await this._loggingComponent?.log({
+                    level: "error",
+                    source: DataspaceControlPlaneService.CLASS_NAME,
+                    ts: Date.now(),
+                    message: "providerTransferTerminateFailed",
+                    data: { consumerPid, code: result.code }
+                });
+                return false;
+            }
+            return true;
+        }
+        catch (error) {
+            await this._loggingComponent?.log({
+                level: "error",
+                source: DataspaceControlPlaneService.CLASS_NAME,
+                ts: Date.now(),
+                message: "providerTransferTerminateFailed",
+                error: BaseError.fromError(error),
+                data: { consumerPid }
+            });
+            return false;
+        }
+    }
     /**
      * Perform the provider-side start of a just-requested transfer (build dataAddress, transition to
      * STARTED, deliver to the consumer). Only invoked when the request opted into auto-start, so there is no