@proompteng/temporal-bun-sdk 0.4.0 → 0.5.0

package/dist/src/worker/runtime.js

@@ -2,7 +2,7 @@ import { pathToFileURL } from 'node:url';
 import { create } from '@bufbuild/protobuf';
 import { Code, ConnectError, createClient } from '@connectrpc/connect';
 import { createGrpcTransport } from '@connectrpc/connect-node';
-import { Cause, Effect, Exit, Fiber } from 'effect';
+import { Cause, Duration, Effect, Exit, Fiber, Schedule } from 'effect';
 import { makeActivityLifecycle, } from '../activities/lifecycle';
 import { buildTransportOptions, normalizeTemporalAddress } from '../client';
 import { durationFromMillis, durationToMillis } from '../common/duration';
@@ -24,10 +24,12 @@ import { HistoryEventFilterType, TimeoutType, VersioningBehavior } from '../prot
 import { StickyExecutionAttributesSchema, TaskQueueSchema, } from '../proto/temporal/api/taskqueue/v1/message_pb';
 import { GetWorkflowExecutionHistoryRequestSchema, PollActivityTaskQueueRequestSchema, PollWorkflowTaskQueueRequestSchema, RespondActivityTaskCanceledRequestSchema, RespondActivityTaskCompletedRequestSchema, RespondActivityTaskFailedRequestSchema, RespondQueryTaskCompletedRequestSchema, RespondWorkflowTaskCompletedRequestSchema, RespondWorkflowTaskFailedRequestSchema, } from '../proto/temporal/api/workflowservice/v1/request_response_pb';
 import { WorkflowService } from '../proto/temporal/api/workflowservice/v1/service_pb';
+import { intentsEqual, stableStringify } from '../workflow/determinism';
 import { WorkflowNondeterminismError } from '../workflow/errors';
 import { WorkflowExecutor } from '../workflow/executor';
+import { CHILD_WORKFLOW_COMPLETED_SIGNAL, } from '../workflow/inbound';
 import { WorkflowRegistry } from '../workflow/registry';
-import { DETERMINISM_MARKER_NAME, diffDeterminismState, encodeDeterminismMarkerDetails, ingestWorkflowHistory, resolveHistoryLastEventId, } from '../workflow/replay';
+import { DETERMINISM_MARKER_NAME, diffDeterminismState, encodeDeterminismMarkerDetailsWithSize, ingestWorkflowHistory, resolveHistoryLastEventId, } from '../workflow/replay';
 import { runWithActivityContext } from './activity-context';
 import { checkWorkerVersioningCapability, registerWorkerBuildIdCompatibility } from './build-id';
 import { makeWorkerScheduler, } from './concurrency';
@@ -53,11 +55,19 @@ const mergeUpdateInvocations = (historyInvocations, messageInvocations) => {
 const POLL_TIMEOUT_MS = 60_000;
 const RESPOND_TIMEOUT_MS = 15_000;
 const HISTORY_FETCH_TIMEOUT_MS = 60_000;
+const DEFAULT_METRICS_FLUSH_INTERVAL_MS = 10_000;
 const HEARTBEAT_RETRY_INITIAL_DELAY_MS = 250;
 const HEARTBEAT_RETRY_MAX_DELAY_MS = 5_000;
 const HEARTBEAT_RETRY_MAX_ATTEMPTS = 5;
 const HEARTBEAT_RETRY_BACKOFF = 2;
 const HEARTBEAT_RETRY_JITTER = 0.2;
+const parseMetricsFlushInterval = (value) => {
+    if (!value) {
+        return undefined;
+    }
+    const parsed = Number.parseInt(value, 10);
+    return Number.isFinite(parsed) ? parsed : undefined;
+};
 const STICKY_QUEUE_PREFIX = 'sticky';
 const COMPLETION_COMMAND_TYPES = new Set([
     CommandType.COMPLETE_WORKFLOW_EXECUTION,
@@ -90,7 +100,10 @@ export class WorkerRuntime {
             metricsRegistry: options.metrics,
             metricsExporter: options.metricsExporter,
         }));
-        const { logger, metricsRegistry, metricsExporter } = observability;
+        const { logger, metricsRegistry, metricsExporter, openTelemetry } = observability;
+        const metricsFlushIntervalMs = options.metricsFlushIntervalMs ??
+            parseMetricsFlushInterval(process.env.TEMPORAL_METRICS_FLUSH_INTERVAL_MS) ??
+            DEFAULT_METRICS_FLUSH_INTERVAL_MS;
         const dataConverter = options.dataConverter ??
             createDefaultDataConverter({
                 payloadCodecs: buildCodecsFromConfig(config.payloadCodecs),
@@ -102,6 +115,7 @@ export class WorkerRuntime {
         const executor = new WorkflowExecutor({
             registry,
             dataConverter,
+            logger,
         });
         const runtimeMetrics = await WorkerRuntime.#initMetrics(metricsRegistry);
         let workflowService;
@@ -162,6 +176,11 @@ export class WorkerRuntime {
         const stickySchedulingEnabled = options.stickyScheduling ??
             (config.stickySchedulingEnabled &&
                 (hasStickyCacheInstance ? config.workerStickyCacheSize > 0 : stickyCacheSize > 0));
+        const determinismMarkerMode = config.determinismMarkerMode;
+        const determinismMarkerIntervalTasks = config.determinismMarkerIntervalTasks;
+        const determinismMarkerFullSnapshotIntervalTasks = config.determinismMarkerFullSnapshotIntervalTasks;
+        const determinismMarkerSkipUnchanged = config.determinismMarkerSkipUnchanged;
+        const determinismMarkerMaxDetailBytes = config.determinismMarkerMaxDetailBytes;
         const deploymentName = options.deployment?.name ?? config.workerDeploymentName ?? WorkerRuntime.#defaultDeploymentName(taskQueue);
         const buildId = options.deployment?.buildId ?? config.workerBuildId ?? identity;
         const workerVersioningMode = options.deployment?.versioningMode ?? WorkerVersioningMode.UNVERSIONED;
@@ -173,6 +192,7 @@ export class WorkerRuntime {
             buildId,
             workerVersioningMode,
         });
+        const rpcDeploymentOptions = workerVersioningMode === WorkerVersioningMode.VERSIONED ? deploymentOptions : undefined;
         if (workerVersioningMode === WorkerVersioningMode.VERSIONED) {
             const capability = await checkWorkerVersioningCapability(workflowService, namespace, taskQueue);
             if (capability.supported) {
@@ -226,11 +246,17 @@ export class WorkerRuntime {
             workflowConcurrency,
             activityConcurrency,
             stickySchedulingEnabled,
+            determinismMarkerMode,
+            determinismMarkerIntervalTasks,
+            determinismMarkerFullSnapshotIntervalTasks,
+            determinismMarkerSkipUnchanged,
+            determinismMarkerMaxDetailBytes,
             deploymentName,
             buildId,
             logLevel: config.logLevel,
             logFormat: config.logFormat,
             metricsExporter: config.metricsExporter.type,
+            metricsFlushIntervalMs,
         }));
         return new WorkerRuntime({
             config,
@@ -242,7 +268,9 @@ export class WorkerRuntime {
             logger,
             metricsRegistry,
             metricsExporter,
+            metricsFlushIntervalMs,
             metrics: runtimeMetrics,
+            openTelemetry,
             namespace,
             taskQueue,
             identity,
@@ -252,8 +280,14 @@ export class WorkerRuntime {
             stickyQueue,
             stickyScheduleToStartTimeoutMs,
             deploymentOptions,
+            rpcDeploymentOptions,
             versioningBehavior,
             stickySchedulingEnabled,
+            determinismMarkerMode,
+            determinismMarkerIntervalTasks,
+            determinismMarkerFullSnapshotIntervalTasks,
+            determinismMarkerSkipUnchanged,
+            determinismMarkerMaxDetailBytes,
             workflowPollerCount,
             interceptors: workerInterceptors,
         });
@@ -268,6 +302,9 @@ export class WorkerRuntime {
     #metricsRegistry;
     #metrics;
     #metricsExporter;
+    #metricsFlushIntervalMs;
+    #metricsFlushInFlight = false;
+    #openTelemetry;
     #namespace;
     #taskQueue;
     #identity;
@@ -277,8 +314,14 @@ export class WorkerRuntime {
     #stickyCache;
     #stickyQueue;
     #stickySchedulingEnabled;
+    #determinismMarkerMode;
+    #determinismMarkerIntervalTasks;
+    #determinismMarkerFullSnapshotIntervalTasks;
+    #determinismMarkerSkipUnchanged;
+    #determinismMarkerMaxDetailBytes;
     #stickyAttributes;
     #deploymentOptions;
+    #rpcDeploymentOptions;
     #versioningBehavior;
     #workflowPollerCount;
     #running = false;
@@ -296,6 +339,8 @@ export class WorkerRuntime {
         this.#metricsRegistry = params.metricsRegistry;
         this.#metrics = params.metrics;
         this.#metricsExporter = params.metricsExporter;
+        this.#metricsFlushIntervalMs = Math.max(0, params.metricsFlushIntervalMs);
+        this.#openTelemetry = params.openTelemetry;
         this.#namespace = params.namespace;
         this.#taskQueue = params.taskQueue;
         this.#identity = params.identity;
@@ -305,7 +350,13 @@ export class WorkerRuntime {
         this.#stickyCache = params.stickyCache;
         this.#stickyQueue = params.stickyQueue;
         this.#stickySchedulingEnabled = params.stickySchedulingEnabled;
+        this.#determinismMarkerMode = params.determinismMarkerMode;
+        this.#determinismMarkerIntervalTasks = Math.max(1, params.determinismMarkerIntervalTasks);
+        this.#determinismMarkerFullSnapshotIntervalTasks = Math.max(1, params.determinismMarkerFullSnapshotIntervalTasks);
+        this.#determinismMarkerSkipUnchanged = params.determinismMarkerSkipUnchanged;
+        this.#determinismMarkerMaxDetailBytes = Math.max(0, params.determinismMarkerMaxDetailBytes);
         this.#deploymentOptions = params.deploymentOptions;
+        this.#rpcDeploymentOptions = params.rpcDeploymentOptions;
         this.#versioningBehavior = params.versioningBehavior;
         this.#workflowPollerCount = params.workflowPollerCount;
         this.#stickyAttributes = create(StickyExecutionAttributesSchema, {
@@ -406,6 +457,7 @@ export class WorkerRuntime {
         if (!runtimeFiber) {
             await this.#stopScheduler();
             await this.#flushMetrics();
+            await this.#shutdownOpenTelemetry();
             this.#running = false;
             this.#log('info', 'temporal worker shutdown complete', this.#runtimeLogFields({ drained: false }));
             return;
@@ -421,6 +473,7 @@ export class WorkerRuntime {
             this.#runFiber = null;
             this.#running = false;
         }
+        await this.#shutdownOpenTelemetry();
         this.#log('info', 'temporal worker shutdown complete', this.#runtimeLogFields({ drained: true }));
     }
     #buildRuntimeEffect() {
@@ -437,6 +490,10 @@ export class WorkerRuntime {
                 workflowPollers: runtime.#workflowPollerCount,
                 stickySchedulingEnabled: runtime.#stickySchedulingEnabled,
             }));
+            if (runtime.#metricsFlushIntervalMs > 0) {
+                const flushLoop = Effect.repeat(Effect.promise(() => runtime.#flushMetrics()), Schedule.spaced(Duration.millis(runtime.#metricsFlushIntervalMs)));
+                yield* Effect.forkScoped(flushLoop);
+            }
             if (pollerLoops.length > 0) {
                 yield* Effect.forEach(pollerLoops, (loop) => Effect.forkScoped(loop), { concurrency: 'unbounded' });
             }
@@ -444,9 +501,16 @@ export class WorkerRuntime {
         })).pipe(Effect.ensuring(Effect.promise(async () => {
             await runtime.#stopScheduler();
             await runtime.#flushMetrics();
+            await runtime.#shutdownOpenTelemetry();
             runtime.#log('info', 'temporal worker runtime stopped', runtime.#runtimeLogFields());
         })));
     }
+    async #shutdownOpenTelemetry() {
+        if (!this.#openTelemetry) {
+            return;
+        }
+        await this.#openTelemetry.shutdown();
+    }
     async #awaitRuntimeFiber(fiber) {
         const exit = await Effect.runPromiseExit(Fiber.join(fiber));
         if (Exit.isFailure(exit)) {
@@ -482,7 +546,7 @@ export class WorkerRuntime {
             namespace: this.#namespace,
             taskQueue: create(TaskQueueSchema, { name: queueName }),
             identity: this.#identity,
-            deploymentOptions: this.#deploymentOptions,
+            deploymentOptions: this.#rpcDeploymentOptions,
         });
         const pollOnce = this.#withRpcAbort(async (signal) => {
             const start = Date.now();
@@ -516,7 +580,7 @@ export class WorkerRuntime {
             namespace: this.#namespace,
             taskQueue: create(TaskQueueSchema, { name: this.#taskQueue }),
             identity: this.#identity,
-            deploymentOptions: this.#deploymentOptions,
+            deploymentOptions: this.#rpcDeploymentOptions,
         });
         const pollOnce = this.#withRpcAbort(async (signal) => {
             const start = Date.now();
@@ -638,9 +702,20 @@ export class WorkerRuntime {
         if (isLegacyQueryTask) {
             this.#incrementCounter(this.#metrics.queryTaskStarted);
         }
-        const historyEvents = await this.#collectWorkflowHistory(execution, response);
+        const queryCount = response.queries && typeof response.queries === 'object' && !Array.isArray(response.queries)
+            ? Object.keys(response.queries).length
+            : Array.isArray(response.queries)
+                ? response.queries.length
+                : 0;
+        const hasHistoryEvents = (response.history?.events?.length ?? 0) > 0;
+        const hasMoreHistory = (response.nextPageToken?.length ?? 0) > 0;
+        const isLegacyQueryOnly = isLegacyQueryTask && !hasHistoryEvents && !hasMoreHistory;
+        const hasQueryPayloads = Boolean(response.query) || queryCount > 0;
+        const historyEvents = await this.#collectWorkflowHistory(execution, response, {
+            forceFullHistory: (hasQueryPayloads && !isLegacyQueryOnly) || nondeterminismRetry > 0,
+            skipFetchOnMissingStart: isLegacyQueryOnly,
+        });
         const workflowType = this.#resolveWorkflowType(response, historyEvents);
-        const args = await this.#decodeWorkflowArgs(historyEvents);
         const workflowInfo = this.#buildWorkflowInfo(workflowType, execution);
         const collectedUpdates = await collectWorkflowUpdates({
             messages: response.messages ?? [],
@@ -677,6 +752,26 @@ export class WorkerRuntime {
         });
         const stickyKey = this.#buildStickyKey(execution.workflowId, execution.runId);
         const stickyEntry = stickyKey ? await this.#getStickyEntry(stickyKey) : undefined;
+        const decodedArgs = await this.#decodeWorkflowArgs(historyEvents);
+        let args = decodedArgs.args;
+        let argsSource = decodedArgs.hasStartEvent ? 'history' : 'unknown';
+        if (argsSource === 'unknown' && stickyEntry?.workflowArguments !== undefined) {
+            args = stickyEntry.workflowArguments;
+            argsSource = 'sticky';
+        }
+        if (argsSource === 'unknown' && !isLegacyQueryOnly) {
+            const fetchedArgs = await this.#fetchWorkflowStartArguments(execution);
+            if (fetchedArgs !== undefined) {
+                args = fetchedArgs;
+                argsSource = 'fetch';
+            }
+        }
+        if (argsSource !== 'history') {
+            this.#log('debug', 'workflow arguments resolved from fallback source', {
+                ...baseLogFields,
+                argsSource,
+            });
+        }
         const historyReplay = await this.#ingestDeterminismState(workflowInfo, historyEvents, {
             queryRequests,
         });
@@ -729,6 +824,19 @@ export class WorkerRuntime {
         try {
             const { results: activityResults, scheduledEventIds: activityScheduleEventIds } = await this.#extractActivityResolutions(historyEvents);
             const timerResults = await this.#extractTimerResolutions(historyEvents);
+            const mergedActivityResults = new Map(stickyEntry?.activityResults ?? []);
+            for (const [activityId, resolution] of activityResults.entries()) {
+                mergedActivityResults.set(activityId, resolution);
+            }
+            const mergedScheduleEventIds = new Map(stickyEntry?.activityScheduleEventIds ?? []);
+            for (const [activityId, scheduleId] of activityScheduleEventIds.entries()) {
+                mergedScheduleEventIds.set(activityId, scheduleId);
+            }
+            const mergedTimerResults = new Set(stickyEntry?.timerResults ?? []);
+            for (const timerId of timerResults) {
+                mergedTimerResults.add(timerId);
+            }
+            const pendingChildWorkflows = await this.#extractPendingChildWorkflows(historyEvents);
             const replayUpdates = historyReplay?.updates ?? [];
             const mergedUpdates = mergeUpdateInvocations(replayUpdates, collectedUpdates.invocations);
             const output = await this.#executor.execute({
@@ -739,10 +847,11 @@ export class WorkerRuntime {
                 taskQueue: this.#taskQueue,
                 arguments: args,
                 determinismState: previousState,
-                activityResults,
-                activityScheduleEventIds,
+                activityResults: mergedActivityResults,
+                activityScheduleEventIds: mergedScheduleEventIds,
+                pendingChildWorkflows,
                 signalDeliveries,
-                timerResults,
+                timerResults: mergedTimerResults,
                 queryRequests,
                 updates: mergedUpdates,
                 mode: isLegacyQueryTask ? 'query' : 'workflow',
@@ -788,8 +897,18 @@ export class WorkerRuntime {
                 return;
             }
             const cacheBaselineEventId = this.#resolveCurrentStartedEventId(response) ?? historyReplay?.lastEventId ?? null;
-            const shouldRecordMarker = output.completion === 'pending';
             let commandsForResponse = output.commands;
+            const workflowTaskCount = output.completion === 'pending'
+                ? (stickyEntry?.workflowTaskCount ?? 0) + 1
+                : (stickyEntry?.workflowTaskCount ?? 0);
+            let markerHash = stickyEntry?.lastDeterminismMarkerHash;
+            let lastMarkerTask = stickyEntry?.lastDeterminismMarkerTask;
+            let lastFullSnapshotTask = stickyEntry?.lastDeterminismFullSnapshotTask;
+            let lastMarkerState = stickyEntry?.lastDeterminismMarkerState;
+            let markerType = output.completion === 'pending' ? this.#resolveDeterminismMarkerType(workflowTaskCount, stickyEntry) : null;
+            let determinismDelta;
+            let markerLastEventId = null;
+            const markerBaseState = historyReplay?.markerState ?? lastMarkerState;
             const dispatchesForNewMessages = (output.updateDispatches ?? []).filter((dispatch) => {
                 if (dispatch.type === 'acceptance' || dispatch.type === 'rejection') {
                     return collectedUpdates.requestsByUpdateId.has(dispatch.updateId);
@@ -804,41 +923,153 @@ export class WorkerRuntime {
                 defaultIdentity: this.#identity,
                 log: (level, message, fields) => this.#log(level, message, fields),
             });
-            if (stickyKey) {
-                if (output.completion === 'pending') {
-                    await this.#upsertStickyEntry(stickyKey, output.determinismState, cacheBaselineEventId, workflowType);
-                    this.#log('debug', 'sticky cache snapshot persisted', {
+            if (markerType === 'delta') {
+                if (!markerBaseState) {
+                    this.#log('warn', 'determinism delta base unavailable; falling back to full snapshot', {
                         ...baseLogFields,
-                        cacheBaselineEventId,
                     });
+                    markerType = 'full';
                 }
                 else {
-                    await this.#removeStickyEntry(stickyKey);
-                    await this.#removeStickyEntriesForWorkflow(stickyKey.workflowId);
-                    this.#log('debug', 'sticky cache entry cleared (workflow completed)', baseLogFields);
+                    determinismDelta = this.#buildDeterminismDelta(markerBaseState, output.determinismState);
+                }
+                if (!determinismDelta) {
+                    markerType = 'full';
+                }
+                else if (this.#determinismMarkerSkipUnchanged && this.#isDeterminismDeltaEmpty(determinismDelta)) {
+                    markerType = null;
                 }
             }
-            if (shouldRecordMarker) {
-                const lastEventId = historyReplay?.lastEventId ??
-                    this.#resolveWorkflowHistoryLastEventId(response) ??
-                    stickyEntry?.lastEventId ??
-                    null;
-                const markerDetails = await Effect.runPromise(encodeDeterminismMarkerDetails(this.#dataConverter, {
-                    info: workflowInfo,
-                    determinismState: output.determinismState,
-                    lastEventId,
-                }));
-                const markerCommand = this.#buildDeterminismMarkerCommand(markerDetails);
-                commandsForResponse = this.#injectDeterminismMarker(commandsForResponse, markerCommand);
+            if (markerType) {
+                markerLastEventId =
+                    historyReplay?.lastEventId ??
+                        this.#resolveWorkflowHistoryLastEventId(response) ??
+                        stickyEntry?.lastEventId ??
+                        null;
+                let resolvedMarkerType = markerType;
+                let markerDetails = null;
+                let markerSizeBytes = 0;
+                const limitBytes = this.#determinismMarkerMaxDetailBytes;
+                const markerHashCandidate = this.#hashDeterminismMarker(this.#buildDeterminismMarkerSignature(output.determinismState));
+                if (this.#determinismMarkerSkipUnchanged && markerHashCandidate === stickyEntry?.lastDeterminismMarkerHash) {
+                    resolvedMarkerType = null;
+                }
+                if (resolvedMarkerType) {
+                    const buildMarkerDetails = async (targetType) => Effect.runPromise(encodeDeterminismMarkerDetailsWithSize(this.#dataConverter, {
+                        info: workflowInfo,
+                        determinismState: output.determinismState,
+                        determinismDelta: targetType === 'delta' ? determinismDelta : undefined,
+                        markerType: targetType,
+                        lastEventId: markerLastEventId,
+                    }));
+                    if (resolvedMarkerType === 'full') {
+                        const fullResult = await buildMarkerDetails('full');
+                        if (fullResult.sizeBytes > limitBytes && determinismDelta) {
+                            this.#log('debug', 'determinism full marker payload exceeds size limit; falling back to delta', {
+                                ...baseLogFields,
+                                sizeBytes: fullResult.sizeBytes,
+                                limitBytes,
+                            });
+                            const deltaResult = await buildMarkerDetails('delta');
+                            if (deltaResult.sizeBytes <= limitBytes) {
+                                resolvedMarkerType = 'delta';
+                                markerDetails = deltaResult.details;
+                                markerSizeBytes = deltaResult.sizeBytes;
+                            }
+                            else {
+                                this.#log('warn', 'determinism marker payload exceeds size limit', {
+                                    ...baseLogFields,
+                                    markerType: 'delta',
+                                    sizeBytes: deltaResult.sizeBytes,
+                                    limitBytes,
+                                });
+                                resolvedMarkerType = null;
+                            }
+                        }
+                        else if (fullResult.sizeBytes <= limitBytes) {
+                            markerDetails = fullResult.details;
+                            markerSizeBytes = fullResult.sizeBytes;
+                        }
+                        else {
+                            this.#log('warn', 'determinism marker payload exceeds size limit', {
+                                ...baseLogFields,
+                                markerType: 'full',
+                                sizeBytes: fullResult.sizeBytes,
+                                limitBytes,
+                            });
+                            resolvedMarkerType = null;
+                        }
+                    }
+                    else if (resolvedMarkerType === 'delta') {
+                        const deltaResult = await buildMarkerDetails('delta');
+                        if (deltaResult.sizeBytes <= limitBytes) {
+                            markerDetails = deltaResult.details;
+                            markerSizeBytes = deltaResult.sizeBytes;
+                        }
+                        else {
+                            this.#log('warn', 'determinism marker payload exceeds size limit', {
+                                ...baseLogFields,
+                                markerType: 'delta',
+                                sizeBytes: deltaResult.sizeBytes,
+                                limitBytes,
+                            });
+                            resolvedMarkerType = null;
+                        }
+                    }
+                }
+                if (resolvedMarkerType) {
+                    markerHash = markerHashCandidate;
+                    lastMarkerTask = workflowTaskCount;
+                    if (resolvedMarkerType === 'full') {
+                        lastFullSnapshotTask = workflowTaskCount;
+                    }
+                    lastMarkerState = output.determinismState;
+                    if (markerDetails) {
+                        this.#log('debug', 'determinism marker recorded', {
+                            ...baseLogFields,
+                            markerType: resolvedMarkerType,
+                            sizeBytes: markerSizeBytes,
+                        });
+                        const markerCommand = this.#buildDeterminismMarkerCommand(markerDetails);
+                        commandsForResponse = this.#injectDeterminismMarker(commandsForResponse, markerCommand);
+                    }
+                }
+            }
+            let pendingStickyUpdate = null;
+            let shouldClearSticky = false;
+            if (stickyKey) {
+                if (output.completion === 'pending') {
+                    const resolvedWorkflowArgs = argsSource === 'unknown' ? stickyEntry?.workflowArguments : args;
+                    pendingStickyUpdate = {
+                        determinismState: output.determinismState,
+                        cacheBaselineEventId,
+                        workflowType,
+                        metadata: {
+                            workflowTaskCount,
+                            lastDeterminismMarkerHash: markerHash,
+                            lastDeterminismMarkerTask: lastMarkerTask,
+                            lastDeterminismFullSnapshotTask: lastFullSnapshotTask,
+                            ...(lastMarkerState ? { lastDeterminismMarkerState: lastMarkerState } : {}),
+                            ...(resolvedWorkflowArgs !== undefined ? { workflowArguments: resolvedWorkflowArgs } : {}),
+                            activityResults: mergedActivityResults,
+                            activityScheduleEventIds: mergedScheduleEventIds,
+                            timerResults: mergedTimerResults,
+                        },
+                    };
+                }
+                else {
+                    shouldClearSticky = true;
+                }
             }
             const shouldRespondWorkflowTask = hasMultiQueries || !hasLegacyQueries;
+            let workflowTaskCommitted = false;
             if (shouldRespondWorkflowTask) {
                 const completion = create(RespondWorkflowTaskCompletedRequestSchema, {
                     taskToken: response.taskToken,
                     commands: commandsForResponse,
                     identity: this.#identity,
                     namespace: this.#namespace,
-                    deploymentOptions: this.#deploymentOptions,
+                    deploymentOptions: this.#rpcDeploymentOptions,
                     queryResults: multiQueryResults,
                     ...(this.#stickySchedulingEnabled && !hasLegacyQueries ? { stickyAttributes: this.#stickyAttributes } : {}),
                     ...(this.#versioningBehavior !== null ? { versioningBehavior: this.#versioningBehavior } : {}),
@@ -846,6 +1077,7 @@ export class WorkerRuntime {
                 });
                 try {
                     await this.#workflowService.respondWorkflowTaskCompleted(completion, { timeoutMs: RESPOND_TIMEOUT_MS });
+                    workflowTaskCommitted = true;
                 }
                 catch (rpcError) {
                     this.#log('error', 'debug: respondWorkflowTaskCompleted failed', {
@@ -854,6 +1086,10 @@ export class WorkerRuntime {
                     });
                     if (this.#isTaskNotFoundError(rpcError)) {
                         this.#logWorkflowTaskNotFound('respondWorkflowTaskCompleted', execution);
+                        if (stickyKey) {
+                            await this.#removeStickyEntry(stickyKey);
+                            await this.#removeStickyEntriesForWorkflow(stickyKey.workflowId);
+                        }
                         return;
                     }
                     throw rpcError;
@@ -862,6 +1098,20 @@ export class WorkerRuntime {
             if (legacyQueryResult) {
                 await this.#respondLegacyQueryTask(response, legacyQueryResult);
             }
+            if (stickyKey && workflowTaskCommitted) {
+                if (pendingStickyUpdate) {
+                    await this.#upsertStickyEntry(stickyKey, pendingStickyUpdate.determinismState, pendingStickyUpdate.cacheBaselineEventId, pendingStickyUpdate.workflowType, pendingStickyUpdate.metadata);
+                    this.#log('debug', 'sticky cache snapshot persisted', {
+                        ...baseLogFields,
+                        cacheBaselineEventId: pendingStickyUpdate.cacheBaselineEventId,
+                    });
+                }
+                else if (shouldClearSticky) {
+                    await this.#removeStickyEntry(stickyKey);
+                    await this.#removeStickyEntriesForWorkflow(stickyKey.workflowId);
+                    this.#log('debug', 'sticky cache entry cleared (workflow completed)', baseLogFields);
+                }
+            }
         }
         catch (error) {
             let stickyEntryCleared = false;
@@ -950,20 +1200,93 @@ export class WorkerRuntime {
             queries: options?.queryRequests,
         }));
     }
-    async #collectWorkflowHistory(execution, _response) {
+    async #collectWorkflowHistory(execution, response, options) {
         const events = [];
-        let token;
-        while (true) {
+        const initialEvents = response.history?.events ?? [];
+        if (initialEvents.length > 0) {
+            events.push(...initialEvents);
+        }
+        let token = response.nextPageToken && response.nextPageToken.length > 0 ? response.nextPageToken : undefined;
+        while (token && token.length > 0) {
             const page = await this.#fetchWorkflowHistoryPage(execution, token);
             if (page.events.length > 0) {
                 events.push(...page.events);
             }
-            if (!page.nextPageToken || page.nextPageToken.length === 0) {
+            token = page.nextPageToken && page.nextPageToken.length > 0 ? page.nextPageToken : undefined;
+        }
+        let sorted = this.#sortHistoryEvents(events);
+        const shouldFetchFullHistory = options?.forceFullHistory || (!options?.skipFetchOnMissingStart && !this.#findWorkflowStartedEvent(sorted));
+        if (shouldFetchFullHistory) {
+            const fullHistory = await this.#fetchWorkflowHistoryFromStart(execution);
+            if (fullHistory.length > 0) {
+                sorted = this.#sortHistoryEvents([...fullHistory, ...sorted]);
+            }
+        }
+        const bounded = this.#limitHistoryToStartedEvent(sorted, response.startedEventId);
+        return this.#dedupeHistoryEvents(bounded);
+    }
+    #sortHistoryEvents(events) {
+        return events.slice().sort((left, right) => {
+            const leftId = this.#resolveEventIdForSort(left.eventId);
+            const rightId = this.#resolveEventIdForSort(right.eventId);
+            if (leftId === rightId) {
+                return 0;
+            }
+            return leftId < rightId ? -1 : 1;
+        });
+    }
+    #dedupeHistoryEvents(events) {
+        if (events.length <= 1) {
+            return events;
+        }
+        const deduped = [];
+        let lastId = null;
+        for (const event of events) {
+            const currentId = this.#resolveEventIdForSort(event.eventId);
+            if (currentId !== 0n) {
+                if (lastId !== null && currentId === lastId) {
+                    continue;
+                }
+                lastId = currentId;
+            }
+            deduped.push(event);
+        }
+        return deduped;
+    }
+    #limitHistoryToStartedEvent(events, startedEventId) {
+        const limit = this.#resolveEventIdForSort(startedEventId);
+        if (limit <= 0n) {
+            return events;
+        }
+        const bounded = [];
+        for (const event of events) {
+            const currentId = this.#resolveEventIdForSort(event.eventId);
+            if (currentId !== 0n && currentId > limit) {
                 break;
             }
-            token = page.nextPageToken;
+            bounded.push(event);
         }
-        return events;
+        return bounded;
+    }
+    #resolveEventIdForSort(value) {
+        if (value === undefined || value === null) {
+            return 0n;
+        }
+        if (typeof value === 'bigint') {
+            return value;
+        }
+        if (typeof value === 'number' && Number.isFinite(value)) {
+            return BigInt(value);
+        }
+        if (typeof value === 'string' && value.length > 0) {
+            try {
+                return BigInt(value);
+            }
+            catch {
+                return 0n;
+            }
+        }
+        return 0n;
     }
     async #extractActivityResolutions(events) {
         const resolutions = new Map();
@@ -1077,28 +1400,86 @@ export class WorkerRuntime {
             }
             return value.toString();
         };
-        for (const event of events) {
-            if (event.eventType !== EventType.WORKFLOW_EXECUTION_SIGNALED) {
-                continue;
-            }
-            if (event.attributes?.case !== 'workflowExecutionSignaledEventAttributes') {
-                continue;
+        const recordChildCompletion = (event, status, attrs) => {
+            const workflowId = attrs.workflowExecution?.workflowId;
+            if (!workflowId) {
+                return;
             }
-            const attrs = event.attributes.value;
-            const args = await decodePayloadsToValues(this.#dataConverter, attrs.input?.payloads ?? []);
-            const workflowTaskCompletedEventId = 'workflowTaskCompletedEventId' in attrs
-                ? normalizeEventId(attrs
-                    .workflowTaskCompletedEventId)
-                : null;
             deliveries.push({
-                name: attrs.signalName ?? 'unknown',
-                args,
+                name: CHILD_WORKFLOW_COMPLETED_SIGNAL,
+                args: [
+                    {
+                        workflowId,
+                        ...(attrs.workflowExecution?.runId ? { runId: attrs.workflowExecution.runId } : {}),
+                        status,
+                    },
+                ],
                 metadata: {
                     eventId: normalizeEventId(event.eventId),
-                    workflowTaskCompletedEventId,
-                    identity: attrs.identity ?? null,
                 },
             });
+        };
+        for (const event of events) {
+            switch (event.eventType) {
+                case EventType.WORKFLOW_EXECUTION_SIGNALED: {
+                    if (event.attributes?.case !== 'workflowExecutionSignaledEventAttributes') {
+                        break;
+                    }
+                    const attrs = event.attributes.value;
+                    const args = await decodePayloadsToValues(this.#dataConverter, attrs.input?.payloads ?? []);
+                    const workflowTaskCompletedEventId = 'workflowTaskCompletedEventId' in attrs
+                        ? normalizeEventId(attrs
+                            .workflowTaskCompletedEventId)
+                        : null;
+                    deliveries.push({
+                        name: attrs.signalName ?? 'unknown',
+                        args,
+                        metadata: {
+                            eventId: normalizeEventId(event.eventId),
+                            workflowTaskCompletedEventId,
+                            identity: attrs.identity ?? null,
+                        },
+                    });
+                    break;
+                }
+                case EventType.CHILD_WORKFLOW_EXECUTION_COMPLETED: {
+                    if (event.attributes?.case !== 'childWorkflowExecutionCompletedEventAttributes') {
+                        break;
+                    }
+                    recordChildCompletion(event, 'completed', event.attributes.value);
+                    break;
+                }
+                case EventType.CHILD_WORKFLOW_EXECUTION_FAILED: {
+                    if (event.attributes?.case !== 'childWorkflowExecutionFailedEventAttributes') {
+                        break;
+                    }
+                    recordChildCompletion(event, 'failed', event.attributes.value);
+                    break;
+                }
+                case EventType.CHILD_WORKFLOW_EXECUTION_CANCELED: {
+                    if (event.attributes?.case !== 'childWorkflowExecutionCanceledEventAttributes') {
+                        break;
+                    }
+                    recordChildCompletion(event, 'canceled', event.attributes.value);
+                    break;
+                }
+                case EventType.CHILD_WORKFLOW_EXECUTION_TERMINATED: {
+                    if (event.attributes?.case !== 'childWorkflowExecutionTerminatedEventAttributes') {
+                        break;
+                    }
+                    recordChildCompletion(event, 'terminated', event.attributes.value);
+                    break;
+                }
+                case EventType.CHILD_WORKFLOW_EXECUTION_TIMED_OUT: {
+                    if (event.attributes?.case !== 'childWorkflowExecutionTimedOutEventAttributes') {
+                        break;
+                    }
+                    recordChildCompletion(event, 'timed_out', event.attributes.value);
+                    break;
+                }
+                default:
+                    break;
+            }
         }
         return deliveries;
     }
@@ -1118,6 +1499,56 @@ export class WorkerRuntime {
         }
         return fired;
     }
+    async #extractPendingChildWorkflows(events) {
+        const pending = new Map();
+        const normalizeEventId = (value) => {
+            if (value === undefined || value === null) {
+                return null;
+            }
+            if (typeof value === 'string') {
+                return value;
+            }
+            return value.toString();
+        };
+        for (const event of events) {
+            switch (event.eventType) {
+                case EventType.START_CHILD_WORKFLOW_EXECUTION_INITIATED: {
+                    if (event.attributes?.case !== 'startChildWorkflowExecutionInitiatedEventAttributes') {
+                        break;
+                    }
+                    const attrs = event.attributes.value;
+                    const initiatedId = normalizeEventId(event.eventId);
+                    if (initiatedId && attrs.workflowId) {
+                        pending.set(initiatedId, attrs.workflowId);
+                    }
+                    break;
+                }
+                case EventType.CHILD_WORKFLOW_EXECUTION_STARTED: {
+                    if (event.attributes?.case !== 'childWorkflowExecutionStartedEventAttributes') {
+                        break;
+                    }
+                    const initiatedId = normalizeEventId(event.attributes.value.initiatedEventId);
+                    if (initiatedId) {
+                        pending.delete(initiatedId);
+                    }
+                    break;
+                }
+                case EventType.START_CHILD_WORKFLOW_EXECUTION_FAILED: {
+                    if (event.attributes?.case !== 'startChildWorkflowExecutionFailedEventAttributes') {
+                        break;
+                    }
+                    const initiatedId = normalizeEventId(event.attributes.value.initiatedEventId);
+                    if (initiatedId) {
+                        pending.delete(initiatedId);
+                    }
+                    break;
+                }
+                default:
+                    break;
+            }
+        }
+        return new Set(pending.values());
+    }
     async #extractWorkflowQueryRequests(response) {
         const requests = [];
         const map = response.queries ?? {};
@@ -1235,6 +1666,18 @@ export class WorkerRuntime {
             nextPageToken: historyResponse.nextPageToken ?? new Uint8Array(),
         };
     }
+    async #fetchWorkflowHistoryFromStart(execution) {
+        const events = [];
+        let token;
+        do {
+            const page = await this.#fetchWorkflowHistoryPage(execution, token);
+            if (page.events.length > 0) {
+                events.push(...page.events);
+            }
+            token = page.nextPageToken && page.nextPageToken.length > 0 ? page.nextPageToken : undefined;
+        } while (token && token.length > 0);
+        return events;
+    }
     #buildWorkflowInfo(workflowType, execution) {
         return {
             namespace: this.#namespace,
@@ -1247,19 +1690,145 @@ export class WorkerRuntime {
     async #getStickyEntry(key) {
         return await Effect.runPromise(this.#stickyCache.get(key));
     }
-    async #upsertStickyEntry(key, state, lastEventId, workflowType) {
+    async #upsertStickyEntry(key, state, lastEventId, workflowType, metadata) {
         const entry = {
             key,
             determinismState: state,
             lastEventId,
             lastAccessed: Date.now(),
             workflowType,
+            ...metadata,
         };
         await Effect.runPromise(this.#stickyCache.upsert(entry));
     }
     #resolveWorkflowHistoryLastEventId(response) {
         return resolveHistoryLastEventId(response.history?.events ?? []);
     }
+    #resolveDeterminismMarkerType(taskIndex, stickyEntry) {
+        if (this.#determinismMarkerMode === 'never') {
+            return null;
+        }
+        if (this.#determinismMarkerMode === 'always') {
+            return 'full';
+        }
+        const interval = Math.max(1, this.#determinismMarkerIntervalTasks);
+        const shouldRecord = taskIndex === 1 || interval === 1 || taskIndex % interval === 0;
+        if (!shouldRecord) {
+            return null;
+        }
+        if (this.#determinismMarkerMode === 'delta') {
+            const lastFullSnapshot = stickyEntry?.lastDeterminismFullSnapshotTask ?? 0;
+            const fullInterval = Math.max(1, this.#determinismMarkerFullSnapshotIntervalTasks);
+            const fullDue = taskIndex === 1 || fullInterval === 1 || taskIndex - lastFullSnapshot >= fullInterval;
+            return fullDue ? 'full' : 'delta';
+        }
+        return 'full';
+    }
+    #buildDeterminismDelta(previous, current) {
+        if (!previous) {
+            return undefined;
+        }
+        const normalizeOptional = (value) => (value === undefined ? null : value);
+        const optionalEquals = (left, right) => normalizeOptional(left) === normalizeOptional(right);
+        const signalsEqual = (left, right) => left.signalName === right.signalName &&
+            left.handlerName === right.handlerName &&
+            left.payloadHash === right.payloadHash &&
+            optionalEquals(left.eventId, right.eventId) &&
+            optionalEquals(left.workflowTaskCompletedEventId, right.workflowTaskCompletedEventId) &&
+            optionalEquals(left.identity, right.identity);
+        const queriesEqual = (left, right) => left.queryName === right.queryName &&
+            left.handlerName === right.handlerName &&
+            left.requestHash === right.requestHash &&
+            optionalEquals(left.identity, right.identity) &&
+            optionalEquals(left.queryId, right.queryId) &&
+            left.resultHash === right.resultHash &&
+            left.failureHash === right.failureHash;
+        const updatesEqual = (left, right) => left.updateId === right.updateId &&
+            left.stage === right.stage &&
+            left.handlerName === right.handlerName &&
+            left.identity === right.identity &&
+            left.sequencingEventId === right.sequencingEventId &&
+            left.messageId === right.messageId &&
+            left.acceptedEventId === right.acceptedEventId &&
+            left.outcome === right.outcome &&
+            left.failureMessage === right.failureMessage &&
+            left.historyEventId === right.historyEventId;
+        const sliceAppend = (currentList, previousList, equals = Object.is) => {
+            if (currentList.length < previousList.length) {
+                return undefined;
+            }
+            for (let index = 0; index < previousList.length; index += 1) {
+                if (!equals(previousList[index], currentList[index])) {
+                    return undefined;
+                }
+            }
+            return currentList.slice(previousList.length);
+        };
+        const commandHistory = sliceAppend(current.commandHistory, previous.commandHistory, (left, right) => intentsEqual(left?.intent, right?.intent));
+        const randomValues = sliceAppend(current.randomValues, previous.randomValues);
+        const timeValues = sliceAppend(current.timeValues, previous.timeValues);
+        const signals = sliceAppend(current.signals, previous.signals, signalsEqual);
+        const queries = sliceAppend(current.queries, previous.queries, queriesEqual);
+        if (!commandHistory || !randomValues || !timeValues || !signals || !queries) {
+            return undefined;
+        }
+        const updatesPrev = previous.updates ?? [];
+        const updatesNext = current.updates ?? [];
+        if (updatesNext.length < updatesPrev.length) {
+            return undefined;
+        }
+        const updates = sliceAppend(updatesNext, updatesPrev, updatesEqual);
+        if (!updates) {
+            return undefined;
+        }
+        const logCount = current.logCount !== undefined && current.logCount !== previous.logCount ? current.logCount : undefined;
+        const failureMetadata = current.failureMetadata && previous.failureMetadata
+            ? current.failureMetadata.eventId !== previous.failureMetadata.eventId ||
+                current.failureMetadata.eventType !== previous.failureMetadata.eventType ||
+                current.failureMetadata.failureType !== previous.failureMetadata.failureType ||
+                current.failureMetadata.failureMessage !== previous.failureMetadata.failureMessage ||
+                current.failureMetadata.retryState !== previous.failureMetadata.retryState
+                ? current.failureMetadata
+                : undefined
+            : (current.failureMetadata ?? undefined);
+        return {
+            commandHistory,
+            randomValues,
+            timeValues,
+            signals,
+            queries,
+            ...(updates.length > 0 ? { updates } : {}),
+            ...(logCount !== undefined ? { logCount } : {}),
+            ...(failureMetadata ? { failureMetadata } : {}),
+        };
+    }
+    #isDeterminismDeltaEmpty(delta) {
+        const updatesCount = delta.updates ? delta.updates.length : 0;
+        return ((delta.commandHistory?.length ?? 0) === 0 &&
+            (delta.randomValues?.length ?? 0) === 0 &&
+            (delta.timeValues?.length ?? 0) === 0 &&
+            (delta.signals?.length ?? 0) === 0 &&
+            (delta.queries?.length ?? 0) === 0 &&
+            updatesCount === 0 &&
+            delta.logCount === undefined &&
+            delta.failureMetadata === undefined);
+    }
+    #buildDeterminismMarkerSignature(state) {
+        const failureMetadataHash = state.failureMetadata ? stableStringify(state.failureMetadata) : undefined;
+        return {
+            commandHistoryLength: state.commandHistory.length,
+            randomValuesLength: state.randomValues.length,
+            timeValuesLength: state.timeValues.length,
+            signalsLength: state.signals.length,
+            queriesLength: state.queries.length,
+            updatesLength: state.updates?.length ?? 0,
+            logCount: state.logCount ?? 0,
+            ...(failureMetadataHash ? { failureMetadataHash } : {}),
+        };
+    }
+    #hashDeterminismMarker(signature) {
+        return stableStringify(signature);
+    }
     #isTaskNotFoundError(error) {
         return error instanceof ConnectError && error.code === Code.NotFound;
     }
@@ -1506,7 +2075,7 @@ export class WorkerRuntime {
             failure: encoded,
             identity: this.#identity,
             namespace: this.#namespace,
-            deploymentOptions: this.#deploymentOptions,
+            deploymentOptions: this.#rpcDeploymentOptions,
         });
         try {
             await this.#workflowService.respondWorkflowTaskFailed(failed, { timeoutMs: RESPOND_TIMEOUT_MS });
@@ -1606,7 +2175,7 @@ export class WorkerRuntime {
                         identity: this.#identity,
                         namespace: this.#namespace,
                         result: payloads && payloads.length > 0 ? create(PayloadsSchema, { payloads }) : undefined,
-                        deploymentOptions: this.#deploymentOptions,
+                        deploymentOptions: this.#rpcDeploymentOptions,
                     });
                     await this.#workflowService.respondActivityTaskCompleted(completion, { timeoutMs: RESPOND_TIMEOUT_MS });
                     break;
@@ -1662,7 +2231,7 @@ export class WorkerRuntime {
             namespace: this.#namespace,
             failure: encoded,
             lastHeartbeatDetails,
-            deploymentOptions: this.#deploymentOptions,
+            deploymentOptions: this.#rpcDeploymentOptions,
         });
         await this.#workflowService.respondActivityTaskFailed(request, { timeoutMs: RESPOND_TIMEOUT_MS });
         this.#incrementCounter(this.#metrics.activityFailures);
@@ -1674,7 +2243,7 @@ export class WorkerRuntime {
             identity: this.#identity,
             namespace: this.#namespace,
             details,
-            deploymentOptions: this.#deploymentOptions,
+            deploymentOptions: this.#rpcDeploymentOptions,
         });
         await this.#workflowService.respondActivityTaskCanceled(request, { timeoutMs: RESPOND_TIMEOUT_MS });
     }
@@ -1789,6 +2358,10 @@ export class WorkerRuntime {
         return scheduleDeadline ?? startDeadline ?? undefined;
     }
     async #flushMetrics() {
+        if (this.#metricsFlushInFlight) {
+            return;
+        }
+        this.#metricsFlushInFlight = true;
         try {
             await Effect.runPromise(this.#metricsExporter.flush());
         }
@@ -1797,21 +2370,36 @@ export class WorkerRuntime {
                 error: error instanceof Error ? error.message : String(error),
             });
         }
+        finally {
+            this.#metricsFlushInFlight = false;
+        }
     }
     async #decodeWorkflowArgs(events) {
         const startEvent = this.#findWorkflowStartedEvent(events);
         if (!startEvent) {
-            return [];
+            return { args: [], hasStartEvent: false };
         }
+        const args = (await this.#decodeWorkflowArgsFromStartEvent(startEvent)) ?? [];
+        return { args, hasStartEvent: true };
+    }
+    async #decodeWorkflowArgsFromStartEvent(startEvent) {
         const attributes = startEvent.attributes?.case === 'workflowExecutionStartedEventAttributes'
             ? startEvent.attributes.value
             : undefined;
         if (!attributes) {
-            return [];
+            return undefined;
         }
         const inputPayloads = attributes.input?.payloads ?? [];
         return await decodePayloadsToValues(this.#dataConverter, inputPayloads);
     }
+    async #fetchWorkflowStartArguments(execution) {
+        const page = await this.#fetchWorkflowHistoryPage(execution);
+        const startEvent = this.#findWorkflowStartedEvent(page.events);
+        if (!startEvent) {
+            return undefined;
+        }
+        return await this.#decodeWorkflowArgsFromStartEvent(startEvent);
+    }
     #resolveWorkflowType(response, events) {
         if (response.workflowType?.name) {
             return response.workflowType.name;