@proompteng/temporal-bun-sdk 0.9.0 → 0.10.0

package/dist/src/worker/runtime.js

@@ -5,6 +5,7 @@ import { createGrpcTransport } from '@connectrpc/connect-node';
 import { Cause, Duration, Effect, Exit, Fiber, Schedule } from 'effect';
 import { makeActivityLifecycle, } from '../activities/lifecycle';
 import { buildTransportOptions, normalizeTemporalAddress } from '../client';
+import { withTemporalRetry } from '../client/retries';
 import { durationFromMillis, durationToMillis } from '../common/duration';
 import { buildCodecsFromConfig, createDefaultDataConverter, decodePayloadsToValues, encodeValuesToPayloads, } from '../common/payloads/converter';
 import { encodeErrorToFailure, encodeFailurePayloads, failureToError } from '../common/payloads/failure';
@@ -12,7 +13,7 @@ import { sleep } from '../common/sleep';
 import { loadTemporalConfig } from '../config';
 import { makeDefaultWorkerInterceptors, runWorkerInterceptors, } from '../interceptors/worker';
 import { createObservabilityServices } from '../observability';
-import { CommandSchema, RecordMarkerCommandAttributesSchema, } from '../proto/temporal/api/command/v1/message_pb';
+import { CancelWorkflowExecutionCommandAttributesSchema, CommandSchema, RecordMarkerCommandAttributesSchema, } from '../proto/temporal/api/command/v1/message_pb';
 import { PayloadsSchema, WorkflowExecutionSchema, } from '../proto/temporal/api/common/v1/message_pb';
 import { WorkerDeploymentOptionsSchema, } from '../proto/temporal/api/deployment/v1/message_pb';
 import { CommandType } from '../proto/temporal/api/enums/v1/command_type_pb';
@@ -206,7 +207,6 @@ export class WorkerRuntime {
         const versioningBehavior = workerVersioningMode === WorkerVersioningMode.VERSIONED
             ? (options.deployment?.versioningBehavior ?? VersioningBehavior.PINNED)
             : null;
-        // TODO(TBS-NDG-002): enforce strict versioning policy.
         if (workflowGuards === 'strict') {
             if (workerVersioningMode !== WorkerVersioningMode.VERSIONED) {
                 throw new Error('workflowGuards=strict requires worker versioning to be enabled (deployment.versioningMode=VERSIONED)');
@@ -679,10 +679,10 @@ export class WorkerRuntime {
                 timeoutMs: POLL_TIMEOUT_MS,
                 signal,
             });
-            this.#observeHistogram(this.#metrics.workflowPollLatency, Date.now() - start);
             if (!response.taskToken || response.taskToken.length === 0) {
                 return;
             }
+            this.#observeHistogram(this.#metrics.workflowPollLatency, Date.now() - start);
             await this.#enqueueWorkflowTask(response);
         }).pipe(Effect.catchAll((error) => this.#isRpcAbortError(error) ? Effect.void : this.#handleWorkflowPollerError(queueName, error)));
         return Effect.forever(pollOnce);
@@ -713,10 +713,10 @@ export class WorkerRuntime {
                 timeoutMs: POLL_TIMEOUT_MS,
                 signal,
             });
-            this.#observeHistogram(this.#metrics.activityPollLatency, Date.now() - start);
             if (!response.taskToken || response.taskToken.length === 0) {
                 return;
             }
+            this.#observeHistogram(this.#metrics.activityPollLatency, Date.now() - start);
             await this.#enqueueActivityTask(response);
         }).pipe(Effect.catchAll((error) => (this.#isRpcAbortError(error) ? Effect.void : this.#handleActivityPollerError(error))));
         return Effect.forever(pollOnce);
@@ -732,6 +732,23 @@ export class WorkerRuntime {
     #isRpcAbortError(error) {
         return isAbortError(error) || (error instanceof ConnectError && error.code === Code.Canceled);
     }
+    async #runWorkerResponseRpc(operation) {
+        const exit = await Effect.runPromiseExit(withTemporalRetry(Effect.tryPromise({
+            try: operation,
+            catch: (error) => error,
+        }), this.#config.rpcRetryPolicy));
+        if (Exit.isSuccess(exit)) {
+            return exit.value;
+        }
+        throw Cause.squash(exit.cause);
+    }
+    async #runActivityHeartbeat(effect) {
+        const exit = await Effect.runPromiseExit(effect);
+        if (Exit.isSuccess(exit)) {
+            return;
+        }
+        throw Cause.squash(exit.cause);
+    }
     #buildNormalTaskQueue(name) {
         return create(TaskQueueSchema, {
             name,
@@ -895,6 +912,14 @@ export class WorkerRuntime {
             historyEventCount: response.history?.events?.length ?? 0,
         });
         const stickyKey = this.#buildStickyKey(execution.workflowId, execution.runId);
+        if (!isLegacyQueryTask && hasOpenWorkflowCancellationRequest(historyEvents)) {
+            await this.#respondWorkflowCancellation(response, execution, baseLogFields);
+            if (stickyKey) {
+                await this.#removeStickyEntry(stickyKey);
+                await this.#removeStickyEntriesForWorkflow(stickyKey.workflowId);
+            }
+            return;
+        }
         const stickyEntry = stickyKey ? await this.#getStickyEntry(stickyKey) : undefined;
         const decodedArgs = await this.#decodeWorkflowArgs(historyEvents);
         let args = decodedArgs.args;
@@ -1233,7 +1258,7 @@ export class WorkerRuntime {
                     ...(updateProtocolMessages.length > 0 ? { messages: updateProtocolMessages } : {}),
                 });
                 try {
-                    await this.#workflowService.respondWorkflowTaskCompleted(completion, { timeoutMs: RESPOND_TIMEOUT_MS });
+                    await this.#runWorkerResponseRpc(() => this.#workflowService.respondWorkflowTaskCompleted(completion, { timeoutMs: RESPOND_TIMEOUT_MS }));
                     workflowTaskCommitted = true;
                 }
                 catch (rpcError) {
@@ -1336,6 +1361,34 @@ export class WorkerRuntime {
             await this.#failWorkflowTask(response, execution, error);
         }
     }
+    async #respondWorkflowCancellation(response, execution, logFields) {
+        const command = create(CommandSchema, {
+            commandType: CommandType.CANCEL_WORKFLOW_EXECUTION,
+            attributes: {
+                case: 'cancelWorkflowExecutionCommandAttributes',
+                value: create(CancelWorkflowExecutionCommandAttributesSchema, {}),
+            },
+        });
+        const completion = create(RespondWorkflowTaskCompletedRequestSchema, {
+            taskToken: response.taskToken,
+            commands: [command],
+            identity: this.#identity,
+            namespace: this.#namespace,
+            deploymentOptions: this.#rpcDeploymentOptions,
+            ...(this.#versioningBehavior !== null ? { versioningBehavior: this.#versioningBehavior } : {}),
+        });
+        try {
+            await this.#runWorkerResponseRpc(() => this.#workflowService.respondWorkflowTaskCompleted(completion, { timeoutMs: RESPOND_TIMEOUT_MS }));
+            this.#log('info', 'workflow cancellation request acknowledged', logFields);
+        }
+        catch (error) {
+            if (this.#isTaskNotFoundError(error)) {
+                this.#logWorkflowTaskNotFound('respondWorkflowCancellation', execution);
+                return;
+            }
+            throw error;
+        }
+    }
     #buildStickyKey(workflowId, runId) {
         if (!workflowId || !runId) {
             return null;
@@ -1873,7 +1926,7 @@ export class WorkerRuntime {
             failure: queryResult.failure,
             cause: WorkflowTaskFailedCause.UNSPECIFIED,
         });
-        await this.#workflowService.respondQueryTaskCompleted(request, { timeoutMs: RESPOND_TIMEOUT_MS });
+        await this.#runWorkerResponseRpc(() => this.#workflowService.respondQueryTaskCompleted(request, { timeoutMs: RESPOND_TIMEOUT_MS }));
     }
     async #respondLegacyQueryFailure(response, cause) {
         const failure = await encodeErrorToFailure(this.#dataConverter, cause);
@@ -1887,7 +1940,7 @@ export class WorkerRuntime {
             cause: WorkflowTaskFailedCause.UNSPECIFIED,
         });
         try {
-            await this.#workflowService.respondQueryTaskCompleted(request, { timeoutMs: RESPOND_TIMEOUT_MS });
+            await this.#runWorkerResponseRpc(() => this.#workflowService.respondQueryTaskCompleted(request, { timeoutMs: RESPOND_TIMEOUT_MS }));
         }
         catch (rpcError) {
             this.#log('error', 'respondQueryTaskCompleted failed for legacy query', {
@@ -2091,7 +2144,27 @@ export class WorkerRuntime {
         return stableStringify(signature);
     }
     #isTaskNotFoundError(error) {
-        return error instanceof ConnectError && error.code === Code.NotFound;
+        const underlying = this.#unwrapWorkerRpcError(error);
+        return underlying instanceof ConnectError && underlying.code === Code.NotFound;
+    }
+    #unwrapWorkerRpcError(error) {
+        if (error instanceof ConnectError) {
+            return error;
+        }
+        if (!error || typeof error !== 'object') {
+            return error;
+        }
+        const candidate = error;
+        if (candidate._tag === 'UnknownException') {
+            return this.#unwrapWorkerRpcError(candidate.cause ?? candidate.error ?? error);
+        }
+        if (candidate.cause) {
+            return this.#unwrapWorkerRpcError(candidate.cause);
+        }
+        if (candidate.error) {
+            return this.#unwrapWorkerRpcError(candidate.error);
+        }
+        return error;
     }
     #buildDeterminismMarkerCommand(details) {
         return create(CommandSchema, {
@@ -2339,7 +2412,7 @@ export class WorkerRuntime {
             deploymentOptions: this.#rpcDeploymentOptions,
         });
         try {
-            await this.#workflowService.respondWorkflowTaskFailed(failed, { timeoutMs: RESPOND_TIMEOUT_MS });
+            await this.#runWorkerResponseRpc(() => this.#workflowService.respondWorkflowTaskFailed(failed, { timeoutMs: RESPOND_TIMEOUT_MS }));
             this.#incrementCounter(this.#metrics.workflowFailures);
         }
         catch (rpcError) {
@@ -2405,7 +2478,7 @@ export class WorkerRuntime {
                 if (!registration) {
                     return;
                 }
-                await Effect.runPromise(registration.heartbeat(details));
+                await this.#runActivityHeartbeat(registration.heartbeat(details));
             };
         }
         catch (registrationError) {
@@ -2428,18 +2501,9 @@ export class WorkerRuntime {
         };
         try {
             while (true) {
+                let result;
                 try {
-                    const result = await runWithActivityContext(context, async () => await handler(...args));
-                    const payloads = await encodeValuesToPayloads(this.#dataConverter, result === undefined ? [] : [result]);
-                    const completion = create(RespondActivityTaskCompletedRequestSchema, {
-                        taskToken: response.taskToken,
-                        identity: this.#identity,
-                        namespace: this.#namespace,
-                        result: payloads && payloads.length > 0 ? create(PayloadsSchema, { payloads }) : undefined,
-                        deploymentOptions: this.#rpcDeploymentOptions,
-                    });
-                    await this.#workflowService.respondActivityTaskCompleted(completion, { timeoutMs: RESPOND_TIMEOUT_MS });
-                    break;
+                    result = await runWithActivityContext(context, async () => await handler(...args));
                 }
                 catch (error) {
                     if (isAbortError(error)) {
@@ -2465,6 +2529,19 @@ export class WorkerRuntime {
                     context.throwIfCancelled();
                     context.info.attempt = nextRetry.attempt;
                     retryState = nextRetry;
+                    continue;
+                }
+                try {
+                    const payloads = await encodeValuesToPayloads(this.#dataConverter, result === undefined ? [] : [result]);
+                    const completed = await this.#completeActivityTask(response, payloads);
+                    if (!completed) {
+                        return;
+                    }
+                    break;
+                }
+                catch (error) {
+                    await this.#failActivityTask(response, error, context);
+                    return;
                 }
             }
         }
@@ -2494,7 +2571,16 @@ export class WorkerRuntime {
             lastHeartbeatDetails,
             deploymentOptions: this.#rpcDeploymentOptions,
         });
-        await this.#workflowService.respondActivityTaskFailed(request, { timeoutMs: RESPOND_TIMEOUT_MS });
+        try {
+            await this.#runWorkerResponseRpc(() => this.#workflowService.respondActivityTaskFailed(request, { timeoutMs: RESPOND_TIMEOUT_MS }));
+        }
+        catch (rpcError) {
+            if (this.#isTaskNotFoundError(rpcError)) {
+                this.#logActivityTaskNotFound('respondActivityTaskFailed', response);
+                return;
+            }
+            throw rpcError;
+        }
         this.#incrementCounter(this.#metrics.activityFailures);
     }
     async #cancelActivityTask(response, context) {
@@ -2506,7 +2592,36 @@ export class WorkerRuntime {
             details,
             deploymentOptions: this.#rpcDeploymentOptions,
         });
-        await this.#workflowService.respondActivityTaskCanceled(request, { timeoutMs: RESPOND_TIMEOUT_MS });
+        try {
+            await this.#runWorkerResponseRpc(() => this.#workflowService.respondActivityTaskCanceled(request, { timeoutMs: RESPOND_TIMEOUT_MS }));
+        }
+        catch (rpcError) {
+            if (this.#isTaskNotFoundError(rpcError)) {
+                this.#logActivityTaskNotFound('respondActivityTaskCanceled', response);
+                return;
+            }
+            throw rpcError;
+        }
+    }
+    async #completeActivityTask(response, payloads) {
+        const completion = create(RespondActivityTaskCompletedRequestSchema, {
+            taskToken: response.taskToken,
+            identity: this.#identity,
+            namespace: this.#namespace,
+            result: payloads && payloads.length > 0 ? create(PayloadsSchema, { payloads }) : undefined,
+            deploymentOptions: this.#rpcDeploymentOptions,
+        });
+        try {
+            await this.#runWorkerResponseRpc(() => this.#workflowService.respondActivityTaskCompleted(completion, { timeoutMs: RESPOND_TIMEOUT_MS }));
+            return true;
+        }
+        catch (rpcError) {
+            if (this.#isTaskNotFoundError(rpcError)) {
+                this.#logActivityTaskNotFound('respondActivityTaskCompleted', response);
+                return false;
+            }
+            throw rpcError;
+        }
     }
     async #encodeHeartbeatPayloads(details, reason) {
         const finalDetails = details && details.length > 0
@@ -2521,7 +2636,7 @@ export class WorkerRuntime {
         if (!finalDetails) {
             return undefined;
         }
-        const payloads = await encodeValuesToPayloads(this.#dataConverter, finalDetails);
+        const payloads = (await encodeValuesToPayloads(this.#dataConverter, finalDetails)) ?? [];
         return payloads.length > 0 ? create(PayloadsSchema, { payloads }) : undefined;
     }
     #createActivityContext(response, cancelRequested, lastHeartbeatDetails) {
@@ -2693,6 +2808,15 @@ export class WorkerRuntime {
             runId: execution.runId,
         });
     }
+    #logActivityTaskNotFound(context, response) {
+        this.#log('warn', 'activity task already resolved', {
+            context,
+            workflowId: response.workflowExecution?.workflowId,
+            runId: response.workflowExecution?.runId,
+            activityId: response.activityId,
+            activityType: response.activityType?.name,
+        });
+    }
 }
 const mergeSchedulerHooks = (first, second) => {
     if (!first && !second) {
@@ -2731,6 +2855,27 @@ const markErrorNonRetryable = (error) => {
     }
 };
 const isActivityCancelRequested = (response) => Boolean(response.cancelRequested);
+const hasOpenWorkflowCancellationRequest = (events) => {
+    let cancelRequested = false;
+    for (const event of events) {
+        switch (event.eventType) {
+            case EventType.WORKFLOW_EXECUTION_CANCEL_REQUESTED:
+                cancelRequested = true;
+                break;
+            case EventType.WORKFLOW_EXECUTION_CANCELED:
+            case EventType.WORKFLOW_EXECUTION_COMPLETED:
+            case EventType.WORKFLOW_EXECUTION_CONTINUED_AS_NEW:
+            case EventType.WORKFLOW_EXECUTION_FAILED:
+            case EventType.WORKFLOW_EXECUTION_TERMINATED:
+            case EventType.WORKFLOW_EXECUTION_TIMED_OUT:
+                cancelRequested = false;
+                break;
+            default:
+                break;
+        }
+    }
+    return cancelRequested;
+};
 const isAbortError = (error) => error instanceof Error && error.name === 'AbortError';
 async function loadWorkflows(workflowsPath, overrides) {
     if (overrides && overrides.length > 0) {