@hotmeshio/hotmesh 0.12.1 → 0.14.0

package/build/services/durable/worker.js

@@ -3,6 +3,7 @@ var _a;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkerService = void 0;
 const enums_1 = require("../../modules/enums");
+const cancellationScope_1 = require("./workflow/cancellationScope");
 const errors_1 = require("../../modules/errors");
 const storage_1 = require("../../modules/storage");
 const utils_1 = require("../../modules/utils");
@@ -10,30 +11,95 @@ const hotmesh_1 = require("../hotmesh");
 const stream_1 = require("../../types/stream");
 const search_1 = require("./search");
 const factory_1 = require("./schemas/factory");
+const telemetry_1 = require("./telemetry");
+const telemetry_2 = require("../telemetry");
+const telemetry_3 = require("../../types/telemetry");
 const index_1 = require("./index");
 /**
- * The *Worker* service Registers worker functions and connects them to the mesh,
- * using the target backend provider/s (Postgres, NATS, etc).
+ * Hosts workflow and activity functions, connecting them to Postgres
+ * for durable execution, replay, and automatic retry.
  *
- * @example
+ * ## Connection Modes
+ *
+ * ### Standard (legacy) — full admin access
+ *
+ * The worker connects with the same Postgres credentials as the engine.
+ * Simple to set up; all workers share the same connection pool.
+ *
+ * ```typescript
+ * const worker = await Durable.Worker.create({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { connectionString: 'postgres://user:pass@host:5432/hotmesh' },
+ *   },
+ *   taskQueue: 'orders',
+ *   workflow: orderWorkflow,
+ * });
+ * ```
+ *
+ * ### Secured — scoped Postgres role (recommended for production)
+ *
+ * The worker connects as a restricted Postgres role that can only
+ * dequeue/ack/respond on its assigned stream names. All data access
+ * goes through SECURITY DEFINER stored procedures that validate the
+ * role's `app.allowed_streams` session variable before executing.
+ *
+ * **Step 1**: Provision a scoped credential (run once, from the engine/admin):
+ * ```typescript
+ * const cred = await Durable.provisionWorkerRole({
+ *   connection: { class: Postgres, options: adminPgOptions },
+ *   namespace: 'durable',
+ *   streamNames: ['orders-activity'],
+ * });
+ * // cred = { roleName: 'hmsh_wrk_durable_orders_activity', password: '...' }
+ * ```
+ *
+ * **Step 2**: Pass the credential when creating the worker:
  * ```typescript
- * import { Durable } from '@hotmeshio/hotmesh';
- * import { Client as Postgres } from 'pg';
- * import * as workflows from './workflows';
+ * const worker = await Durable.Worker.create({
+ *   connection: {
+ *     class: Postgres,
+ *     options: { host: 'pg.prod', port: 5432, database: 'hotmesh' },
+ *   },
+ *   taskQueue: 'orders',
+ *   workflow: orderWorkflow,
+ *   workerCredentials: { user: cred.roleName, password: cred.password },
+ * });
+ * ```
+ *
+ * The worker role **cannot**:
+ * - SELECT/INSERT/UPDATE/DELETE any table directly
+ * - Access `jobs`, `jobs_attributes`, or any engine tables
+ * - Dequeue messages from other workers' streams
+ * - LISTEN on other workers' notification channels
+ *
+ * See {@link Durable.provisionWorkerRole} for credential lifecycle management.
  *
- * async function run() {
- *   const worker = await Durable.Worker.create({
- *     connection: {
- *       class: Postgres,
- *       options: { connectionString: 'postgres://user:password@localhost:5432/db' }
- *     },
- *     taskQueue: 'default',
- *     workflow: workflows.example,
- *   });
+ * ## Telemetry
+ *
+ * Workers automatically emit OpenTelemetry spans when an OTel SDK is
+ * registered. Initialize the SDK **before** calling `create()`:
+ *
+ * ```typescript
+ * import { NodeSDK } from '@opentelemetry/sdk-node';
+ * import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
+ * import { resourceFromAttributes } from '@opentelemetry/resources';
+ * import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
  *
- *   await worker.run();
- * }
+ * const sdk = new NodeSDK({
+ *   resource: resourceFromAttributes({ [ATTR_SERVICE_NAME]: 'my-service' }),
+ *   traceExporter: new OTLPTraceExporter({
+ *     url: 'https://api.honeycomb.io/v1/traces',
+ *     headers: { 'x-honeycomb-team': process.env.HONEYCOMB_API_KEY },
+ *   }),
+ * });
+ * sdk.start();
  * ```
+ *
+ * | `HMSH_TELEMETRY` | Spans emitted |
+ * |-------|---------------|
+ * | `'info'` (default) | `WORKFLOW/START`, `WORKFLOW/COMPLETE`, `WORKFLOW/ERROR`, `ACTIVITY/{name}` |
+ * | `'debug'` | All `info` spans + `DISPATCH/RETURN` per operation + engine internals |
  */
 class WorkerService {
     static hashOptions(connection) {
@@ -151,7 +217,7 @@ class WorkerService {
      *     sendEmail: (to: string, subject: string) => Promise<void>;
      *   }>({
      *     taskQueue: 'payment',
-     *     retryPolicy: { maximumAttempts: 3 }
+     *     retry: { maximumAttempts: 3 }
      *   });
      *
      *   const result = await processPayment(amount);
@@ -180,28 +246,47 @@ class WorkerService {
      *   taskQueue: 'shared'
      * }, { auditLog, collectMetrics }, 'shared');
      *
-     * const interceptor: WorkflowInterceptor = {
+     * const interceptor: WorkflowInboundCallsInterceptor = {
      *   async execute(ctx, next) {
      *     const { auditLog } = Durable.workflow.proxyActivities<{
      *       auditLog: (id: string, action: string) => Promise<void>;
      *     }>({
      *       taskQueue: 'shared',
-     *       retryPolicy: { maximumAttempts: 3 }
+     *       retry: { maximumAttempts: 3 }
      *     });
      *     await auditLog(ctx.get('workflowId'), 'started');
      *     return next();
      *   }
      * };
      * ```
+     *
+     * @example
+     * ```typescript
+     * // Secured worker with scoped Postgres credentials (VNF-style isolation)
+     * // Step 1: Admin provisions a credential (one-time)
+     * const cred = await Durable.provisionWorkerRole({
+     *   connection: { class: Postgres, options: adminOptions },
+     *   streamNames: ['payment-activity'],
+     * });
+     *
+     * // Step 2: Worker connects with scoped role — can only access payment-activity
+     * await Durable.registerActivityWorker({
+     *   connection: { class: Postgres, options: { host: 'pg.prod', database: 'hotmesh' } },
+     *   taskQueue: 'payment',
+     *   workerCredentials: { user: cred.roleName, password: cred.password },
+     * }, { processPayment, refundPayment });
+     * ```
      */
     static async registerActivityWorker(config, activities, activityTaskQueue) {
+        // Register as durable namespace so engine-layer spans are suppressed in 'info' mode
+        const targetNamespace = config?.namespace ?? factory_1.APP_ID;
+        telemetry_2.TelemetryService.durableNamespaces.add(targetNamespace);
         // Register activities globally in the registry
         WorkerService.registerActivities(activities);
         // Use provided activityTaskQueue or fall back to config.taskQueue
         const taskQueue = activityTaskQueue || config.taskQueue || 'durable-activities';
         // Append '-activity' suffix for the worker topic
         const activityTopic = `${taskQueue}-activity`;
-        const targetNamespace = config?.namespace ?? factory_1.APP_ID;
         const optionsHash = WorkerService.hashOptions(config?.connection);
         const targetTopic = `${optionsHash}.${targetNamespace}.${activityTopic}`;
         // Return existing worker if already initialized (idempotent)
@@ -209,19 +294,21 @@ class WorkerService {
             return await WorkerService.instances.get(targetTopic);
         }
         // Create activity worker that listens on '{taskQueue}-activity' topic
+        const workerEntry = {
+            topic: activityTopic,
+            connection: config.connection,
+            callback: WorkerService.createActivityCallback(),
+        };
+        if (config.workerCredentials) {
+            workerEntry.workerCredentials = config.workerCredentials;
+        }
         const hotMeshWorker = await hotmesh_1.HotMesh.init({
             guid: config.guid ? `${config.guid}XA` : undefined,
             taskQueue,
             logLevel: config.options?.logLevel ?? enums_1.HMSH_LOGLEVEL,
             appId: targetNamespace,
             engine: { connection: config.connection },
-            workers: [
-                {
-                    topic: activityTopic,
-                    connection: config.connection,
-                    callback: WorkerService.createActivityCallback(),
-                },
-            ],
+            workers: [workerEntry],
         });
         WorkerService.instances.set(targetTopic, hotMeshWorker);
         return hotMeshWorker;
@@ -232,6 +319,7 @@ class WorkerService {
      */
     static createActivityCallback() {
         return async (data) => {
+            let activitySpan = null;
             try {
                 //always run the activity function when instructed; return the response
                 const activityInput = data.data;
@@ -240,7 +328,31 @@ class WorkerService {
                 if (!activityFunction) {
                     throw new Error(`Activity '${activityName}' not found in registry`);
                 }
-                const pojoResponse = await activityFunction.apply(null, activityInput.arguments);
+                const activityContext = new Map();
+                activityContext.set('activityName', activityName);
+                activityContext.set('arguments', activityInput.arguments);
+                activityContext.set('headers', activityInput.headers ?? {});
+                activityContext.set('workflowId', activityInput.workflowId);
+                activityContext.set('workflowTopic', activityInput.workflowTopic);
+                // Start an ACTIVITY span if telemetry is enabled
+                const trc = data.metadata.trc;
+                const spn = data.metadata.spn;
+                activitySpan =
+                    telemetry_1.DurableTelemetryService.isEnabled() && trc && spn
+                        ? telemetry_1.DurableTelemetryService.startSpan(trc, spn, `ACTIVITY/${activityName}`, {
+                            'durable.activity.name': activityName,
+                            'durable.workflow.id': activityInput.workflowId,
+                        })
+                        : null;
+                const interceptorService = index_1.Durable.getInterceptorService();
+                const pojoResponse = await storage_1.activityAsyncLocalStorage.run(activityContext, () => {
+                    const executeFn = () => activityFunction.apply(null, activityInput.arguments);
+                    if (interceptorService?.activityInbound?.length > 0) {
+                        return interceptorService.executeActivityInboundChain(activityName, activityInput.arguments, executeFn);
+                    }
+                    return executeFn();
+                });
+                activitySpan?.end();
                 return {
                     status: stream_1.StreamStatus.SUCCESS,
                     metadata: { ...data.metadata },
@@ -248,6 +360,11 @@ class WorkerService {
                 };
             }
             catch (err) {
+                activitySpan?.setStatus({
+                    code: telemetry_3.SpanStatusCode.ERROR,
+                    message: err.message,
+                });
+                activitySpan?.end();
                 // Log error (note: we don't have access to this.activityRunner here)
                 console.error('durable-worker-activity-err', {
                     name: err.name,
@@ -318,7 +435,7 @@ class WorkerService {
         };
     }
     /**
-     * Connects a worker to the mesh.
+     * Creates and starts a workflow worker.
      *
      * @example
      * ```typescript
@@ -343,6 +460,9 @@ class WorkerService {
      * ```
      */
     static async create(config) {
+        // Register as durable namespace so engine-layer spans are suppressed in 'info' mode
+        const targetNamespace = config?.namespace ?? factory_1.APP_ID;
+        telemetry_2.TelemetryService.durableNamespaces.add(targetNamespace);
         const workflow = config.workflow;
         const [workflowFunctionName, workflowFunction] = WorkerService.resolveWorkflowTarget(workflow);
         // Separate taskQueue from workflowName - no concatenation for stream_name
@@ -350,6 +470,10 @@ class WorkerService {
         const activityTopic = `${taskQueue}-activity`;
         // workflowTopic remains concatenated for engine-internal routing (graph.subscribes)
         const workflowTopic = `${taskQueue}-${workflowFunctionName}`;
+        // Register activities passed via config
+        if (config.activities) {
+            WorkerService.registerActivities(config.activities);
+        }
         //initialize supporting workflows
         const worker = new WorkerService();
         worker.activityRunner = await worker.initActivityWorker(config, activityTopic);
@@ -392,19 +516,21 @@ class WorkerService {
         if (WorkerService.instances.has(targetTopic)) {
             return await WorkerService.instances.get(targetTopic);
         }
+        const workerEntry = {
+            topic: activityTopic,
+            connection: providerConfig,
+            callback: this.wrapActivityFunctions().bind(this),
+        };
+        if (config.workerCredentials) {
+            workerEntry.workerCredentials = config.workerCredentials;
+        }
         const hotMeshWorker = await hotmesh_1.HotMesh.init({
             guid: config.guid ? `${config.guid}XA` : undefined,
             taskQueue: config.taskQueue,
             logLevel: config.options?.logLevel ?? enums_1.HMSH_LOGLEVEL,
             appId: targetNamespace,
             engine: { connection: providerConfig },
-            workers: [
-                {
-                    topic: activityTopic,
-                    connection: providerConfig,
-                    callback: this.wrapActivityFunctions().bind(this),
-                },
-            ],
+            workers: [workerEntry],
         });
         WorkerService.instances.set(targetTopic, hotMeshWorker);
         return hotMeshWorker;
@@ -414,12 +540,48 @@ class WorkerService {
      */
     wrapActivityFunctions() {
         return async (data) => {
+            let activitySpan = null;
             try {
                 //always run the activity function when instructed; return the response
                 const activityInput = data.data;
                 const activityName = activityInput.activityName;
                 const activityFunction = WorkerService.activityRegistry[activityName];
-                const pojoResponse = await activityFunction.apply(this, activityInput.arguments);
+                const activityContext = new Map();
+                activityContext.set('activityName', activityName);
+                activityContext.set('arguments', activityInput.arguments);
+                activityContext.set('headers', activityInput.headers ?? {});
+                activityContext.set('workflowId', activityInput.workflowId);
+                activityContext.set('workflowTopic', activityInput.workflowTopic);
+                const interceptorService = index_1.Durable.getInterceptorService();
+                // Start an ACTIVITY span if telemetry is enabled
+                const trc = data.metadata.trc;
+                const spn = data.metadata.spn;
+                activitySpan =
+                    telemetry_1.DurableTelemetryService.isEnabled() && trc && spn
+                        ? telemetry_1.DurableTelemetryService.startSpan(trc, spn, `ACTIVITY/${activityName}`, {
+                            'durable.activity.name': activityName,
+                            'durable.workflow.id': activityInput.workflowId,
+                        })
+                        : null;
+                const activityPromise = storage_1.activityAsyncLocalStorage.run(activityContext, () => {
+                    const executeFn = () => activityFunction.apply(this, activityInput.arguments);
+                    if (interceptorService?.activityInbound?.length > 0) {
+                        return interceptorService.executeActivityInboundChain(activityName, activityInput.arguments, executeFn);
+                    }
+                    return executeFn();
+                });
+                let pojoResponse;
+                if (activityInput.startToCloseTimeout && activityInput.startToCloseTimeout > 0) {
+                    const timeoutMs = activityInput.startToCloseTimeout * 1000;
+                    pojoResponse = await Promise.race([
+                        activityPromise,
+                        new Promise((_, reject) => setTimeout(() => reject(new errors_1.DurableTimeoutError(`Activity '${activityName}' exceeded startToCloseTimeout of ${activityInput.startToCloseTimeout}s`)), timeoutMs)),
+                    ]);
+                }
+                else {
+                    pojoResponse = await activityPromise;
+                }
+                activitySpan?.end();
                 return {
                     status: stream_1.StreamStatus.SUCCESS,
                     metadata: { ...data.metadata },
@@ -427,6 +589,11 @@ class WorkerService {
                 };
             }
             catch (err) {
+                activitySpan?.setStatus({
+                    code: telemetry_3.SpanStatusCode.ERROR,
+                    message: err.message,
+                });
+                activitySpan?.end();
                 this.activityRunner.engine.logger.error('durable-worker-activity-err', {
                     name: err.name,
                     message: err.message,
@@ -477,20 +644,22 @@ class WorkerService {
         const targetNamespace = config?.namespace ?? factory_1.APP_ID;
         const optionsHash = WorkerService.hashOptions(config?.connection);
         const targetTopic = `${optionsHash}.${targetNamespace}.${workflowTopic}`;
+        const workerEntry = {
+            topic: taskQueue,
+            workflowName: workflowFunctionName,
+            connection: providerConfig,
+            callback: this.wrapWorkflowFunction(workflowFunction, workflowTopic, workflowFunctionName, config).bind(this),
+        };
+        if (config.workerCredentials) {
+            workerEntry.workerCredentials = config.workerCredentials;
+        }
         const hotMeshWorker = await hotmesh_1.HotMesh.init({
             guid: config.guid,
             taskQueue: config.taskQueue,
             logLevel: config.options?.logLevel ?? enums_1.HMSH_LOGLEVEL,
             appId: config.namespace ?? factory_1.APP_ID,
             engine: { connection: providerConfig },
-            workers: [
-                {
-                    topic: taskQueue,
-                    workflowName: workflowFunctionName,
-                    connection: providerConfig,
-                    callback: this.wrapWorkflowFunction(workflowFunction, workflowTopic, workflowFunctionName, config).bind(this),
-                },
-            ],
+            workers: [workerEntry],
         });
         WorkerService.instances.set(targetTopic, hotMeshWorker);
         return hotMeshWorker;
@@ -502,7 +671,19 @@ class WorkerService {
         return async (data) => {
             const counter = { counter: 0 };
             const interruptionRegistry = [];
+            const patchMarkers = {};
             let isProcessing = false;
+            // Injects accumulated patch markers into any worker response so the
+            // engine can persist them via the YAML schema's job.maps mechanism.
+            // Include patchMarkers only when non-empty. The schema's
+            // job.maps patch-marker[-] is a no-op when the reference is absent.
+            const withPatchMarkers = (response) => {
+                if (Object.keys(patchMarkers).length > 0) {
+                    response.data = response.data || {};
+                    response.data.patchMarkers = patchMarkers;
+                }
+                return response;
+            };
             try {
                 //incoming data payload has arguments and workflowId
                 const workflowInput = data.data;
@@ -511,6 +692,7 @@ class WorkerService {
                 context.set('expire', workflowInput.expire);
                 context.set('counter', counter);
                 context.set('interruptionRegistry', interruptionRegistry);
+                context.set('patchMarkers', patchMarkers);
                 context.set('connection', config.connection);
                 context.set('namespace', config.namespace ?? factory_1.APP_ID);
                 context.set('raw', data);
@@ -530,6 +712,13 @@ class WorkerService {
                     context.set('workflowDimension', workflowInput.workflowDimension);
                     replayQuery = `-*${workflowInput.workflowDimension}-*`;
                 }
+                else if (workflowInput.continueGeneration) {
+                    //continueAsNew: use generation prefix for replay isolation;
+                    //old generation keys remain but are invisible to the new execution
+                    const genPrefix = `$${workflowInput.continueGeneration}`;
+                    context.set('workflowDimension', genPrefix);
+                    replayQuery = `-*${genPrefix}-*`;
+                }
                 else {
                     //last letter of words like 'hook', 'sleep', 'wait', 'signal', 'search', 'start', 'proxy', 'child', 'collator', 'trace', 'enrich', 'publish'
                     replayQuery = '-*[ehklptydr]-*';
@@ -546,6 +735,15 @@ class WorkerService {
                 context.set('activityInterceptorService', index_1.Durable.getInterceptorService());
                 // Execute workflow with interceptors
                 const workflowResponse = await storage_1.asyncLocalStorage.run(context, async () => {
+                    // Emit WORKFLOW/START on first execution (not replay)
+                    if (telemetry_1.DurableTelemetryService.isEnabled() &&
+                        Object.keys(replay).length === 0) {
+                        telemetry_1.DurableTelemetryService.emitPointSpan(data.metadata.trc, data.metadata.spn, `WORKFLOW/START/${workflowFunctionName}`, {
+                            'durable.workflow.id': workflowInput.workflowId,
+                            'durable.workflow.name': workflowFunctionName,
+                            'durable.workflow.topic': workflowTopic,
+                        });
+                    }
                     // Get the interceptor service
                     const interceptorService = index_1.Durable.getInterceptorService();
                     // Create the workflow execution function
@@ -553,8 +751,16 @@ class WorkerService {
                         return await workflowFunction.apply(this, workflowInput.arguments);
                     };
                     // Execute the workflow through the interceptor chain
-                    return await interceptorService.executeChain(context, execWorkflow);
+                    return await interceptorService.executeInboundChain(context, execWorkflow);
                 });
+                // Emit WORKFLOW/COMPLETE when the workflow finishes successfully
+                if (telemetry_1.DurableTelemetryService.isEnabled()) {
+                    telemetry_1.DurableTelemetryService.emitPointSpan(data.metadata.trc, data.metadata.spn, `WORKFLOW/COMPLETE/${workflowFunctionName}`, {
+                        'durable.workflow.id': workflowInput.workflowId,
+                        'durable.workflow.name': workflowFunctionName,
+                        'durable.workflow.topic': workflowTopic,
+                    });
+                }
                 //if the embedded function has a try/catch, it can interrup the throw
                 // throw here to interrupt the workflow if the embedded function caught and suppressed
                 if (interruptionRegistry.length > 0) {
@@ -568,6 +774,8 @@ class WorkerService {
                             throw new errors_1.DurableChildError(payload);
                         case 'DurableSleepError':
                             throw new errors_1.DurableSleepError(payload);
+                        case 'DurableContinueAsNewError':
+                            throw new errors_1.DurableContinueAsNewError(payload);
                         case 'DurableTimeoutError':
                             throw new errors_1.DurableTimeoutError(payload.message, payload.stack);
                         case 'DurableMaxedError':
@@ -580,12 +788,12 @@ class WorkerService {
                             throw new errors_1.DurableRetryError(`Unknown interruption type: ${payload.type}`);
                     }
                 }
-                return {
+                return withPatchMarkers({
                     code: 200,
                     status: stream_1.StreamStatus.SUCCESS,
                     metadata: { ...data.metadata },
                     data: { response: workflowResponse, done: true },
-                };
+                });
             }
             catch (err) {
                 if (isProcessing) {
@@ -594,12 +802,12 @@ class WorkerService {
                 if (err instanceof errors_1.DurableWaitForError ||
                     interruptionRegistry.length > 1) {
                     isProcessing = true;
-                    //NOTE: this type is spawned when `Promise.all` is used OR if the interruption is a `waitFor`
+                    //NOTE: this type is spawned when `Promise.all` is used OR if the interruption is a `condition`
                     const workflowInput = data.data;
                     const execIndex = counter.counter - interruptionRegistry.length + 1;
                     const { workflowId, workflowTopic, workflowDimension, originJobId, expire, } = workflowInput;
                     const collatorFlowId = `${(0, utils_1.guid)()}$C`;
-                    return {
+                    return withPatchMarkers({
                         status: stream_1.StreamStatus.SUCCESS,
                         code: enums_1.HMSH_CODE_DURABLE_ALL,
                         metadata: { ...data.metadata },
@@ -615,12 +823,12 @@ class WorkerService {
                             workflowTopic: workflowTopic,
                             expire,
                         },
-                    };
+                    });
                 }
                 else if (err instanceof errors_1.DurableSleepError) {
                     //return the sleep interruption
                     isProcessing = true;
-                    return {
+                    return withPatchMarkers({
                         status: stream_1.StreamStatus.SUCCESS,
                         code: err.code,
                         metadata: { ...data.metadata },
@@ -635,12 +843,12 @@ class WorkerService {
                             index: err.index,
                             workflowDimension: err.workflowDimension,
                         },
-                    };
+                    });
                 }
                 else if (err instanceof errors_1.DurableProxyError) {
                     //return the proxyActivity interruption
                     isProcessing = true;
-                    return {
+                    return withPatchMarkers({
                         status: stream_1.StreamStatus.SUCCESS,
                         code: err.code,
                         metadata: { ...data.metadata },
@@ -653,6 +861,7 @@ class WorkerService {
                                 dimension: err.workflowDimension,
                             }),
                             arguments: err.arguments,
+                            headers: err.headers,
                             workflowDimension: err.workflowDimension,
                             index: err.index,
                             originJobId: err.originJobId,
@@ -662,10 +871,12 @@ class WorkerService {
                             workflowTopic: err.workflowTopic,
                             activityName: err.activityName,
                             backoffCoefficient: err.backoffCoefficient,
+                            initialInterval: err.initialInterval,
                             maximumAttempts: err.maximumAttempts,
                             maximumInterval: err.maximumInterval,
+                            startToCloseTimeout: err.startToCloseTimeout,
                         },
-                    };
+                    });
                 }
                 else if (err instanceof errors_1.DurableChildError) {
                     //return the child interruption
@@ -675,7 +886,7 @@ class WorkerService {
                         workflowId: err.workflowId,
                         dimension: err.workflowDimension,
                     };
-                    return {
+                    return withPatchMarkers({
                         status: stream_1.StreamStatus.SUCCESS,
                         code: err.code,
                         metadata: { ...data.metadata },
@@ -685,6 +896,7 @@ class WorkerService {
                             backoffCoefficient: err.backoffCoefficient || enums_1.HMSH_DURABLE_EXP_BACKOFF,
                             code: err.code,
                             index: err.index,
+                            initialInterval: err.initialInterval,
                             message: JSON.stringify(msg),
                             maximumAttempts: err.maximumAttempts || enums_1.HMSH_DURABLE_MAX_ATTEMPTS,
                             maximumInterval: err.maximumInterval || (0, utils_1.s)(enums_1.HMSH_DURABLE_MAX_INTERVAL),
@@ -700,12 +912,54 @@ class WorkerService {
                             taskQueue: err.taskQueue,
                             workflowName: err.workflowName,
                         },
-                    };
+                    });
+                }
+                else if (err instanceof errors_1.DurableContinueAsNewError) {
+                    //return the continueAsNew interruption
+                    isProcessing = true;
+                    return withPatchMarkers({
+                        status: stream_1.StreamStatus.SUCCESS,
+                        code: err.code,
+                        metadata: { ...data.metadata },
+                        data: {
+                            code: err.code,
+                            arguments: err.arguments,
+                            index: err.index,
+                            workflowDimension: err.workflowDimension,
+                        },
+                    });
+                }
+                if (err instanceof cancellationScope_1.CancelledFailure) {
+                    // CancelledFailure that wasn't caught by the workflow:
+                    // treat as fatal (no retry) so the workflow terminates.
+                    isProcessing = true;
+                    return withPatchMarkers({
+                        status: stream_1.StreamStatus.SUCCESS,
+                        code: enums_1.HMSH_CODE_DURABLE_FATAL,
+                        metadata: { ...data.metadata },
+                        data: {
+                            $error: {
+                                message: err.message,
+                                type: 'CancelledFailure',
+                                name: 'CancelledFailure',
+                                stack: err.stack,
+                                code: enums_1.HMSH_CODE_DURABLE_FATAL,
+                            },
+                        },
+                    });
                 }
                 // ALL other errors are actual fatal errors (598, 597, 596)
                 //  OR will be retried (599)
+                if (telemetry_1.DurableTelemetryService.isEnabled()) {
+                    telemetry_1.DurableTelemetryService.emitPointSpan(data.metadata.trc, data.metadata.spn, `WORKFLOW/ERROR/${workflowFunctionName}`, {
+                        'durable.workflow.id': data.data.workflowId,
+                        'durable.workflow.name': workflowFunctionName,
+                        'error.message': err.message,
+                        'error.type': err.name || 'Error',
+                    }, telemetry_3.SpanStatusCode.ERROR, err.message);
+                }
                 isProcessing = true;
-                return {
+                return withPatchMarkers({
                     status: stream_1.StreamStatus.SUCCESS,
                     code: err.code || new errors_1.DurableRetryError(err.message).code,
                     metadata: { ...data.metadata },
@@ -718,7 +972,7 @@ class WorkerService {
                             code: err.code || new errors_1.DurableRetryError(err.message).code,
                         },
                     },
-                };
+                });
             }
         };
     }

package/build/services/durable/workflow/all.d.ts

@@ -5,7 +5,7 @@
  * multiple durable operations concurrently within a workflow function.
  *
  * In most cases, standard `Promise.all` works correctly for Durable
- * operations (e.g., parallel `waitFor` calls). Use `Durable.workflow.all`
+ * operations (e.g., parallel `condition` calls). Use `Durable.workflow.all`
  * when you observe counter-sequencing issues with complex parallel
  * patterns.
  *

package/build/services/durable/workflow/all.js

@@ -8,7 +8,7 @@ exports.all = void 0;
  * multiple durable operations concurrently within a workflow function.
  *
  * In most cases, standard `Promise.all` works correctly for Durable
- * operations (e.g., parallel `waitFor` calls). Use `Durable.workflow.all`
+ * operations (e.g., parallel `condition` calls). Use `Durable.workflow.all`
  * when you observe counter-sequencing issues with complex parallel
  * patterns.
  *