@dbos-inc/dbos-sdk 4.20.5-preview → 4.20.8-preview

package/dist/src/system_database.js

@@ -385,11 +385,15 @@ class SystemDatabase {
     notificationsMap = new NotificationMap();
     workflowEventsMap = new NotificationMap();
     streamsMap = new NotificationMap();
-    cancelWakeupMap = new NotificationMap();
     customPool = false;
+    /**
+     * Caps how many DB-backed polling reads (from wait operations) may run
+     * concurrently against the pool, so a polling storm cannot check out every
+     * client and starve control-plane operations. See {@link #pollWithLimiter}.
+     */
+    pollLimiter;
     runningWorkflowMap = new Map(); // Map from workflowID to workflow promise, queue name and partition key
-    workflowCancellationMap = new Map(); // Map from workflowID to its cancellation status.
-    constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true) {
+    constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency) {
         this.systemDatabaseUrl = systemDatabaseUrl;
         this.logger = logger;
         this.serializer = serializer;
@@ -408,6 +412,11 @@ class SystemDatabase {
             };
             this.pool = new pg_1.Pool(systemPoolConfig);
         }
+        // Default the polling limit to half the pool (minimum 1), reserving the rest
+        // of the pool for control-plane operations.
+        const effectivePoolSize = this.pool.options.max ?? sysDbPoolSize;
+        const pollingLimit = pollingConcurrency ?? Math.max(1, Math.floor(effectivePoolSize / 2));
+        this.pollLimiter = new utils_1.Semaphore(pollingLimit);
         this.pool.on('error', (err) => {
             this.logger.warn(`Unexpected error in pool: ${err}`);
         });
@@ -718,23 +727,11 @@ class SystemDatabase {
            completed_at = (EXTRACT(EPOCH FROM now()) * 1000)::bigint
        WHERE workflow_uuid = ANY($2)
          AND status NOT IN ($3, $4)`, [workflow_1.StatusString.CANCELLED, workflowIDs, workflow_1.StatusString.SUCCESS, workflow_1.StatusString.ERROR]);
-        for (const workflowID of workflowIDs) {
-            this.#setWFCancelMap(workflowID);
-        }
     }
     async checkIfCanceled(workflowID) {
-        const client = await this.pool.connect();
-        try {
-            await this.#checkIfCanceled(client, workflowID);
-        }
-        finally {
-            client.release();
-        }
+        await this.#checkIfCanceled(this.pool, workflowID);
     }
     async resumeWorkflows(workflowIDs, queueName) {
-        for (const workflowID of workflowIDs) {
-            this.#clearWFCancelMap(workflowID);
-        }
         await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
        SET status = $1, queue_name = $2, recovery_attempts = 0,
            workflow_deadline_epoch_ms = NULL, deduplication_id = NULL,
@@ -789,7 +786,6 @@ class SystemDatabase {
         await this.pool.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE workflow_uuid = ANY($1)`, [allIds]);
         for (const wfid of allIds) {
             this.runningWorkflowMap.delete(wfid);
-            this.workflowCancellationMap.delete(wfid);
         }
     }
     async forkWorkflow(workflowID, startStep, options = {}) {
@@ -1152,7 +1148,6 @@ class SystemDatabase {
             .finally(() => {
             // Remove itself from pending workflow map.
             this.runningWorkflowMap.delete(workflowID);
-            this.workflowCancellationMap.delete(workflowID);
         });
         this.runningWorkflowMap.set(workflowID, {
             promise: awaitWorkflowPromise,
@@ -1185,151 +1180,110 @@ class SystemDatabase {
             //throw new Error('Message notification map is not empty - shutdown is not clean.');
         }
     }
+    /**
+     * Run a DB-backed polling read under the polling concurrency limiter, so that
+     * high-fan-out wait loops cannot check out every pool client and starve
+     * control-plane operations. Only polling reads should go through here;
+     * control-plane work hits the pool directly and bypasses the limiter.
+     */
+    #pollWithLimiter(query) {
+        return this.pollLimiter.runExclusive(query);
+    }
+    /**
+     * Cancellation check for use inside polling wait loops: the status read runs
+     * under the polling limiter so it counts against the same concurrency budget
+     * as the rest of the loop's reads.
+     */
+    async #checkIfCanceledLimited(workflowID) {
+        await this.#pollWithLimiter(() => this.#checkIfCanceled(this.pool, workflowID));
+    }
     async awaitWorkflowResult(workflowID, timeoutSeconds, callerID, timerFuncID, pollingIntervalMs) {
         const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
         let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
         const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
+        // Record the durable timeout deadline once before polling. #durableSleep persists it on the
+        // first call and reads back the same value on recovery, so it never changes across iterations.
+        if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
+            finishTime = await this.#durableSleep(callerID, timerFuncID, timeoutms);
+        }
         while (true) {
-            let resolveNotification;
-            const statusPromise = new Promise((resolve) => {
-                resolveNotification = resolve;
-            });
-            const irh = this.cancelWakeupMap.registerCallback(workflowID, (_res) => {
-                resolveNotification();
-            });
-            const crh = callerID
-                ? this.cancelWakeupMap.registerCallback(callerID, (_res) => {
-                    resolveNotification();
-                })
-                : undefined;
+            if (callerID)
+                await this.#checkIfCanceledLimited(callerID);
             try {
-                if (callerID)
-                    await this.checkIfCanceled(callerID);
-                try {
-                    const { rows } = await this.pool.query(`SELECT status, output, error, serialization FROM "${this.schemaName}".workflow_status 
-             WHERE workflow_uuid=$1`, [workflowID]);
-                    if (rows.length > 0) {
-                        const status = rows[0].status;
-                        if (status === workflow_1.StatusString.SUCCESS) {
-                            return { output: rows[0].output, serialization: rows[0].serialization };
-                        }
-                        else if (status === workflow_1.StatusString.ERROR) {
-                            return { error: rows[0].error, serialization: rows[0].serialization };
-                        }
-                        else if (status === workflow_1.StatusString.CANCELLED) {
-                            return { cancelled: true };
-                        }
-                        else if (status === workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED) {
-                            return { maxRecoveryAttemptsExceeded: true };
-                        }
-                        else {
-                            // Status is not actionable
-                        }
+                const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status, output, error, serialization FROM "${this.schemaName}".workflow_status
+             WHERE workflow_uuid=$1`, [workflowID]));
+                if (rows.length > 0) {
+                    const status = rows[0].status;
+                    if (status === workflow_1.StatusString.SUCCESS) {
+                        return { output: rows[0].output, serialization: rows[0].serialization };
+                    }
+                    else if (status === workflow_1.StatusString.ERROR) {
+                        return { error: rows[0].error, serialization: rows[0].serialization };
+                    }
+                    else if (status === workflow_1.StatusString.CANCELLED) {
+                        return { cancelled: true };
+                    }
+                    else if (status === workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED) {
+                        return { maxRecoveryAttemptsExceeded: true };
+                    }
+                    else {
+                        // Status is not actionable
                     }
-                }
-                catch (e) {
-                    const err = e;
-                    this.logger.error(`Exception from system database: ${err}`, err);
-                    throw err;
-                }
-                const ct = Date.now();
-                if (finishTime && ct > finishTime)
-                    return undefined; // Time's up
-                let timeoutPromise = Promise.resolve();
-                let timeoutCancel = () => { };
-                if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
-                    const { promise, cancel, endTime } = await this.#durableSleep(callerID, timerFuncID, timeoutms, pollIntervalMs);
-                    finishTime = endTime;
-                    timeoutPromise = promise;
-                    timeoutCancel = cancel;
-                }
-                else {
-                    let poll = finishTime ? finishTime - ct : pollIntervalMs;
-                    poll = Math.min(pollIntervalMs, poll);
-                    const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
-                    timeoutPromise = promise;
-                    timeoutCancel = cancel;
-                }
-                try {
-                    await Promise.race([statusPromise, timeoutPromise]);
-                }
-                finally {
-                    timeoutCancel();
                 }
             }
-            finally {
-                this.cancelWakeupMap.deregisterCallback(irh);
-                if (crh)
-                    this.cancelWakeupMap.deregisterCallback(crh);
+            catch (e) {
+                const err = e;
+                this.logger.error(`Exception from system database: ${err}`, err);
+                throw err;
             }
+            const ct = Date.now();
+            if (finishTime && ct > finishTime)
+                return undefined; // Time's up
+            let poll = finishTime ? finishTime - Date.now() : pollIntervalMs;
+            poll = Math.min(pollIntervalMs, poll);
+            await (0, utils_1.sleepms)(poll);
         }
     }
     async awaitFirstWorkflowId(workflowIds, callerID, pollingIntervalMs) {
         const placeholders = workflowIds.map((_, i) => `$${i + 1}`).join(', ');
         const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
         while (true) {
-            let resolveNotification;
-            const wakeupPromise = new Promise((resolve) => {
-                resolveNotification = resolve;
-            });
-            // Register cancel callbacks for all target workflows and the caller.
-            const cbHandles = workflowIds.map((wfid) => this.cancelWakeupMap.registerCallback(wfid, () => resolveNotification()));
-            const callerCbHandle = callerID
-                ? this.cancelWakeupMap.registerCallback(callerID, () => resolveNotification())
-                : undefined;
-            try {
-                if (callerID)
-                    await this.checkIfCanceled(callerID);
-                const { rows } = await this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
+            if (callerID)
+                await this.#checkIfCanceledLimited(callerID);
+            const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
            WHERE workflow_uuid IN (${placeholders})
              AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
-           LIMIT 1`, workflowIds);
-                if (rows.length > 0) {
-                    return rows[0].workflow_uuid;
-                }
-                const { promise: sleepPromise, cancel: sleepCancel } = (0, utils_1.cancellableSleep)(pollIntervalMs);
-                try {
-                    await Promise.race([wakeupPromise, sleepPromise]);
-                }
-                finally {
-                    sleepCancel();
-                }
+           LIMIT 1`, workflowIds));
+            if (rows.length > 0) {
+                return rows[0].workflow_uuid;
             }
-            finally {
-                for (const h of cbHandles) {
-                    this.cancelWakeupMap.deregisterCallback(h);
-                }
-                if (callerCbHandle)
-                    this.cancelWakeupMap.deregisterCallback(callerCbHandle);
+            await (0, utils_1.sleepms)(pollIntervalMs);
+        }
+    }
+    async awaitWorkflowIds(workflowIds, callerID, pollingIntervalMs) {
+        const remainingWorkflowIds = new Set(workflowIds);
+        const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
+        while (remainingWorkflowIds.size > 0) {
+            const currentWorkflowIds = [...remainingWorkflowIds];
+            if (callerID)
+                await this.#checkIfCanceledLimited(callerID);
+            const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
+           WHERE workflow_uuid = ANY($1::text[])
+             AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, [currentWorkflowIds]));
+            for (const row of rows) {
+                remainingWorkflowIds.delete(row.workflow_uuid);
+            }
+            if (remainingWorkflowIds.size === 0) {
+                return;
             }
+            await (0, utils_1.sleepms)(pollIntervalMs);
         }
     }
     // ==================== Sleep ====================
     async durableSleepms(workflowID, functionID, durationMS) {
-        let cancelled = false;
-        let resolveNotification;
-        const cancelPromise = new Promise((resolve) => {
-            resolveNotification = () => {
-                cancelled = true;
-                resolve();
-            };
-        });
-        const cbr = this.cancelWakeupMap.registerCallback(workflowID, resolveNotification);
-        try {
-            const { cancel: cancelInitial, endTime } = await this.#durableSleep(workflowID, functionID, durationMS);
-            cancelInitial();
-            while (!cancelled && Date.now() < endTime) {
-                const { promise, cancel } = (0, utils_1.cancellableSleep)(Math.min(endTime - Date.now(), utils_1.sleepConfig.maxTimeoutMS));
-                try {
-                    await Promise.race([cancelPromise, promise]);
-                }
-                finally {
-                    cancel();
-                }
-            }
-        }
-        finally {
-            this.cancelWakeupMap.deregisterCallback(cbr);
+        const endTime = await this.#durableSleep(workflowID, functionID, durationMS);
+        while (Date.now() < endTime) {
+            await (0, utils_1.sleepms)(Math.min(endTime - Date.now(), utils_1.sleepConfig.maxTimeoutMS));
         }
         await this.checkIfCanceled(workflowID);
     }
@@ -1394,6 +1348,11 @@ class SystemDatabase {
         const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
         let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
         const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
+        // Record the durable timeout deadline once before polling. #durableSleep persists it on the
+        // first call and reads back the same value on recovery, so it never changes across iterations.
+        if (timeoutms) {
+            finishTime = await this.#durableSleep(workflowID, timeoutFunctionID, timeoutms);
+        }
         while (true) {
             // register the key with the global notifications listener.
             let resolveNotification;
@@ -1402,43 +1361,27 @@ class SystemDatabase {
             });
             const payload = `${workflowID}::${topic}`;
             const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
-            const crh = this.cancelWakeupMap.registerCallback(workflowID, (_res) => {
-                resolveNotification();
-            });
             try {
-                await this.checkIfCanceled(workflowID);
+                await this.#checkIfCanceledLimited(workflowID);
                 // Check if the key is already in the DB, then wait for the notification if it isn't.
-                const initRecvRows = (await this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE destination_uuid=$1 AND topic=$2 AND consumed = false;`, [workflowID, topic])).rows;
+                const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE destination_uuid=$1 AND topic=$2 AND consumed = false;`, [workflowID, topic]))).rows;
                 if (initRecvRows.length !== 0)
                     break;
                 const ct = Date.now();
                 if (finishTime && ct > finishTime)
                     break; // Time's up
-                let timeoutPromise = Promise.resolve();
-                let timeoutCancel = () => { };
-                if (timeoutms) {
-                    const { promise, cancel, endTime } = await this.#durableSleep(workflowID, timeoutFunctionID, timeoutms, pollIntervalMs);
-                    timeoutPromise = promise;
-                    timeoutCancel = cancel;
-                    finishTime = endTime;
-                }
-                else {
-                    let poll = finishTime ? finishTime - ct : pollIntervalMs;
-                    poll = Math.min(pollIntervalMs, poll);
-                    const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
-                    timeoutPromise = promise;
-                    timeoutCancel = cancel;
-                }
+                let poll = finishTime ? finishTime - Date.now() : pollIntervalMs;
+                poll = Math.min(pollIntervalMs, poll);
+                const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
                 try {
-                    await Promise.race([messagePromise, timeoutPromise]);
+                    await Promise.race([messagePromise, promise]);
                 }
                 finally {
-                    timeoutCancel();
+                    cancel();
                 }
             }
             finally {
                 this.notificationsMap.deregisterCallback(cbr);
-                this.cancelWakeupMap.deregisterCallback(crh);
             }
         }
         await this.checkIfCanceled(workflowID);
@@ -1531,6 +1474,12 @@ class SystemDatabase {
         const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
         let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
         const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalEventMs;
+        // If we have a callerWorkflow, we want a durable sleep, otherwise, not. Record the durable
+        // timeout deadline once before polling. #durableSleep persists it on the first call and reads
+        // back the same value on recovery, so it never changes across iterations.
+        if (callerWorkflow && timeoutms) {
+            finishTime = await this.#durableSleep(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms);
+        }
         // Register the key with the global notifications listener first... we do not want to look in the DB first
         //  or that would cause a timing hole.
         while (true) {
@@ -1539,18 +1488,13 @@ class SystemDatabase {
                 resolveNotification = resolve;
             });
             const cbr = this.workflowEventsMap.registerCallback(payloadKey, resolveNotification);
-            const crh = callerWorkflow?.workflowID
-                ? this.cancelWakeupMap.registerCallback(callerWorkflow.workflowID, (_res) => {
-                    resolveNotification();
-                })
-                : undefined;
             try {
                 if (callerWorkflow?.workflowID)
-                    await this.checkIfCanceled(callerWorkflow?.workflowID);
+                    await this.#checkIfCanceledLimited(callerWorkflow?.workflowID);
                 // Check if the key is already in the DB, then wait for the notification if it isn't.
-                const initRecvRows = (await this.pool.query(`SELECT key, value, serialization
+                const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT key, value, serialization
              FROM "${this.schemaName}".workflow_events
-             WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key])).rows;
+             WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key]))).rows;
                 if (initRecvRows.length > 0) {
                     value = initRecvRows[0].value;
                     valueSer = initRecvRows[0].serialization;
@@ -1559,33 +1503,18 @@ class SystemDatabase {
                 const ct = Date.now();
                 if (finishTime && ct > finishTime)
                     break; // Time's up
-                // If we have a callerWorkflow, we want a durable sleep, otherwise, not
-                let timeoutPromise = Promise.resolve();
-                let timeoutCancel = () => { };
-                if (callerWorkflow && timeoutms) {
-                    const { promise, cancel, endTime } = await this.#durableSleep(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms, pollIntervalMs);
-                    timeoutPromise = promise;
-                    timeoutCancel = cancel;
-                    finishTime = endTime;
-                }
-                else {
-                    let poll = finishTime ? finishTime - ct : pollIntervalMs;
-                    poll = Math.min(pollIntervalMs, poll);
-                    const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
-                    timeoutPromise = promise;
-                    timeoutCancel = cancel;
-                }
+                let poll = finishTime ? finishTime - Date.now() : pollIntervalMs;
+                poll = Math.min(pollIntervalMs, poll);
+                const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
                 try {
-                    await Promise.race([valuePromise, timeoutPromise]);
+                    await Promise.race([valuePromise, promise]);
                 }
                 finally {
-                    timeoutCancel();
+                    cancel();
                 }
             }
             finally {
                 this.workflowEventsMap.deregisterCallback(cbr);
-                if (crh)
-                    this.cancelWakeupMap.deregisterCallback(crh);
             }
         }
         // Record the output if it is inside a workflow.
@@ -2861,31 +2790,18 @@ class SystemDatabase {
         });
         return output;
     }
-    #setWFCancelMap(workflowID) {
-        if (this.runningWorkflowMap.has(workflowID)) {
-            this.workflowCancellationMap.set(workflowID, true);
-        }
-        this.cancelWakeupMap.callCallbacks(workflowID);
-    }
-    #clearWFCancelMap(workflowID) {
-        if (this.workflowCancellationMap.has(workflowID)) {
-            this.workflowCancellationMap.delete(workflowID);
-        }
-    }
     async #checkIfCanceled(client, workflowID) {
-        if (this.workflowCancellationMap.get(workflowID) === true) {
-            throw new error_1.DBOSWorkflowCancelledError(workflowID);
-        }
         const statusValue = await this.getWorkflowStatusValue(client, workflowID);
         if (statusValue === workflow_1.StatusString.CANCELLED) {
             throw new error_1.DBOSWorkflowCancelledError(workflowID);
         }
     }
-    async #durableSleep(workflowID, functionID, durationMS, maxSleepPerIteration) {
-        if (maxSleepPerIteration === undefined)
-            maxSleepPerIteration = durationMS;
-        const curTime = Date.now();
-        let endTimeMs = curTime + durationMS;
+    // Durably records (or, on recovery, reads back) the wakeup deadline for a sleep or
+    // timeout so it survives recovery. Returns the absolute end time in epoch ms; the
+    // caller is responsible for actually waiting until then. Throws if the workflow has
+    // been cancelled.
+    async #durableSleep(workflowID, functionID, durationMS) {
+        const endTimeMs = Date.now() + durationMS;
         const client = await this.pool.connect();
         try {
             const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
@@ -2893,18 +2809,13 @@ class SystemDatabase {
                 if (res.functionName !== exports.DBOS_FUNCNAME_SLEEP) {
                     throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, res.functionName);
                 }
-                endTimeMs = JSON.parse(res.output);
-            }
-            else {
-                await this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, Date.now(), Date.now(), {
-                    output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
-                    serialization: serialization_1.DBOSPortableJSON.name(),
-                });
+                return JSON.parse(res.output);
             }
-            return {
-                ...(0, utils_1.cancellableSleep)(Math.max(Math.min(maxSleepPerIteration, endTimeMs - curTime), 0)),
-                endTime: endTimeMs,
-            };
+            await this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, Date.now(), Date.now(), {
+                output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
+                serialization: serialization_1.DBOSPortableJSON.name(),
+            });
+            return endTimeMs;
         }
         finally {
             client.release();
@@ -3051,6 +2962,12 @@ __decorate([
     __metadata("design:paramtypes", [Array, String, Number]),
     __metadata("design:returntype", Promise)
 ], SystemDatabase.prototype, "awaitFirstWorkflowId", null);
+__decorate([
+    dbRetry(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array, String, Number]),
+    __metadata("design:returntype", Promise)
+], SystemDatabase.prototype, "awaitWorkflowIds", null);
 __decorate([
     dbRetry(),
     __metadata("design:type", Function),