@temporalio/core-bridge 1.16.0 → 1.16.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@temporalio/core-bridge",
-  "version": "1.16.0",
+  "version": "1.16.2",
   "description": "Temporal.io SDK Core<>Node bridge",
   "main": "index.js",
   "types": "lib/index.d.ts",
@@ -14,7 +14,7 @@
   "license": "MIT",
   "dependencies": {
     "@grpc/grpc-js": "^1.12.4",
-    "@temporalio/common": "1.16.0"
+    "@temporalio/common": "1.16.2"
   },
   "devDependencies": {
     "arg": "^5.0.2",

package/sdk-core/.cargo/config.toml

@@ -1,6 +1,6 @@
 [env]
 # This temporarily overrides the version of the CLI used for integration tests, locally and in CI
-#CLI_VERSION_OVERRIDE = "v1.4.1-cloud-v1-29-0-139-2.0"
+CLI_VERSION_OVERRIDE = "v1.6.3-serverless"
 
 [alias]
 # Not sure why --all-features doesn't work

package/sdk-core/crates/sdk-core/src/core_tests/workers.rs

@@ -13,7 +13,15 @@ use crate::{
     },
 };
 use futures_util::{stream, stream::StreamExt};
+use std::sync::{
+    Arc,
+    atomic::{AtomicBool, Ordering},
+};
 use std::{cell::RefCell, collections::HashMap, time::Duration};
+use temporalio_common::protos::temporal::api::{
+    namespace::v1::{NamespaceInfo, namespace_info::Capabilities},
+    workflowservice::v1::DescribeNamespaceResponse,
+};
 use temporalio_common::{
     protos::{
         canned_histories,
@@ -51,6 +59,7 @@ use temporalio_common::{
     },
     worker::WorkerTaskTypes,
 };
+use tokio::sync::Notify;
 use tokio::sync::{Barrier, watch};
 use uuid::Uuid;
 
@@ -1209,3 +1218,106 @@ async fn nexus_start_operation_failure_converts_to_legacy_for_old_server(
     worker.shutdown().await;
     worker.finalize_shutdown().await;
 }
+
+/// Verifies that `initiate_shutdown` sends the `ShutdownWorker` RPC so that the server can
+/// complete in-flight polls. Without this, graceful poll shutdown deadlocks: the SDK waits for
+/// polls to drain, but the server was never told to flush them.
+#[tokio::test]
+async fn graceful_shutdown_sends_shutdown_worker_rpc_during_initiate() {
+    let shutdown_rpc_called = Arc::new(AtomicBool::new(false));
+    let shutdown_rpc_called_clone = shutdown_rpc_called.clone();
+    // When the shutdown_worker RPC fires, it signals polls to complete (simulating server
+    // behavior where ShutdownWorker causes the server to return empty poll responses).
+    let poll_releaser = Arc::new(Notify::new());
+    let poll_releaser_for_rpc = poll_releaser.clone();
+
+    let mut mock_client = MockWorkerClient::new();
+    mock_client
+        .expect_capabilities()
+        .returning(|| Some(*DEFAULT_TEST_CAPABILITIES));
+    mock_client
+        .expect_workers()
+        .returning(|| DEFAULT_WORKERS_REGISTRY.clone());
+    mock_client.expect_is_mock().returning(|| true);
+    mock_client
+        .expect_sdk_name_and_version()
+        .returning(|| ("test-core".to_string(), "0.0.0".to_string()));
+    mock_client
+        .expect_identity()
+        .returning(|| "test-identity".to_string());
+    mock_client
+        .expect_worker_grouping_key()
+        .returning(Uuid::new_v4);
+    mock_client
+        .expect_worker_instance_key()
+        .returning(Uuid::new_v4);
+    mock_client
+        .expect_set_heartbeat_client_fields()
+        .returning(|hb| {
+            hb.sdk_name = "test-core".to_string();
+            hb.sdk_version = "0.0.0".to_string();
+            hb.worker_identity = "test-identity".to_string();
+            hb.heartbeat_time = Some(std::time::SystemTime::now().into());
+        });
+    // Return the worker_poll_complete_on_shutdown capability so validate() enables graceful mode
+    mock_client.expect_describe_namespace().returning(move || {
+        Ok(DescribeNamespaceResponse {
+            namespace_info: Some(NamespaceInfo {
+                capabilities: Some(Capabilities {
+                    worker_poll_complete_on_shutdown: true,
+                    ..Capabilities::default()
+                }),
+                ..NamespaceInfo::default()
+            }),
+            ..DescribeNamespaceResponse::default()
+        })
+    });
+    // When shutdown_worker RPC is called, mark it and release polls
+    mock_client
+        .expect_shutdown_worker()
+        .returning(move |_, _, _, _| {
+            shutdown_rpc_called_clone.store(true, Ordering::SeqCst);
+            poll_releaser_for_rpc.notify_waiters();
+            Ok(ShutdownWorkerResponse {})
+        });
+    mock_client
+        .expect_complete_workflow_task()
+        .returning(|_| Ok(RespondWorkflowTaskCompletedResponse::default()));
+
+    // Polls block until shutdown_worker RPC releases them (simulating server holding polls
+    // open until it receives the ShutdownWorker signal)
+    let poll_releaser_for_stream = poll_releaser.clone();
+    let stream = stream::unfold(poll_releaser_for_stream, |releaser| async move {
+        releaser.notified().await;
+        Some((
+            Ok(PollWorkflowTaskQueueResponse::default().try_into().unwrap()),
+            releaser,
+        ))
+    });
+
+    let mw = MockWorkerInputs::new(stream.boxed());
+    let worker = mock_worker(MocksHolder::from_mock_worker(mock_client, mw));
+
+    // validate() reads describe_namespace and sets capabilities.graceful_poll_shutdown = true
+    worker.validate().await.unwrap();
+
+    let poll_fut = worker.poll_workflow_activation();
+    let shutdown_fut = async {
+        // initiate_shutdown must send the ShutdownWorker RPC, which releases the polls
+        worker.initiate_shutdown();
+    };
+
+    let (poll_result, _) = tokio::time::timeout(Duration::from_secs(5), async {
+        tokio::join!(poll_fut, shutdown_fut)
+    })
+    .await
+    .expect("Shutdown should complete within 5s -- if it hangs, the ShutdownWorker RPC was not sent during initiate_shutdown");
+
+    assert_matches!(poll_result.unwrap_err(), PollError::ShutDown);
+    assert!(
+        shutdown_rpc_called.load(Ordering::SeqCst),
+        "ShutdownWorker RPC must be called during initiate_shutdown"
+    );
+
+    worker.finalize_shutdown().await;
+}

package/sdk-core/crates/sdk-core/src/lib.rs

@@ -42,13 +42,13 @@ pub use temporalio_common::protos::TaskToken;
 pub use url::Url;
 pub use worker::{
     ActivitySlotKind, CompleteActivityError, CompleteNexusError, CompleteWfError,
-    FixedSizeSlotSupplier, LocalActivitySlotKind, NexusSlotKind, PollError, PollerBehavior,
-    ResourceBasedSlotsOptions, ResourceBasedSlotsOptionsBuilder, ResourceBasedTuner,
-    ResourceSlotOptions, SlotInfo, SlotInfoTrait, SlotKind, SlotKindType, SlotMarkUsedContext,
-    SlotReleaseContext, SlotReservationContext, SlotSupplier, SlotSupplierOptions,
-    SlotSupplierPermit, TunerBuilder, TunerHolder, TunerHolderOptions, TunerHolderOptionsBuilder,
-    Worker, WorkerConfig, WorkerConfigBuilder, WorkerTuner, WorkerValidationError,
-    WorkerVersioningStrategy, WorkflowErrorType, WorkflowSlotKind,
+    FixedSizeSlotSupplier, LocalActivitySlotKind, NamespaceCapabilities, NexusSlotKind, PollError,
+    PollerBehavior, ResourceBasedSlotsOptions, ResourceBasedSlotsOptionsBuilder,
+    ResourceBasedTuner, ResourceSlotOptions, SlotInfo, SlotInfoTrait, SlotKind, SlotKindType,
+    SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext, SlotSupplier,
+    SlotSupplierOptions, SlotSupplierPermit, TunerBuilder, TunerHolder, TunerHolderOptions,
+    TunerHolderOptionsBuilder, Worker, WorkerConfig, WorkerConfigBuilder, WorkerTuner,
+    WorkerValidationError, WorkerVersioningStrategy, WorkflowErrorType, WorkflowSlotKind,
 };
 
 use crate::{

package/sdk-core/crates/sdk-core/src/pollers/mod.rs

@@ -157,6 +157,11 @@ where
                         return match state.poller.poll().await {
                             Some(Ok((task, permit))) => {
                                 if task == Default::default() {
+                                    if state.poller_was_shutdown {
+                                        // Server sent an empty response after we initiated
+                                        // shutdown — this is the graceful shutdown signal.
+                                        return None;
+                                    }
                                     // We get the default proto in the event that the long poll
                                     // times out.
                                     debug!("Poll {} task timeout", T::task_name());
@@ -276,3 +281,93 @@ pub(crate) fn new_nexus_task_poller(
     )
     .into_stream()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        abstractions::tests::fixed_size_permit_dealer, pollers::MockPermittedPollBuffer,
+        test_help::mock_poller, worker::ActivitySlotKind,
+    };
+    use futures_util::{StreamExt, pin_mut};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+
+    /// Verify that empty responses after shutdown are not treated as poll timeout and retried
+    /// indefinitely
+    #[tokio::test]
+    async fn empty_response_after_shutdown_terminates_stream() {
+        let poll_count = Arc::new(AtomicUsize::new(0));
+        let poll_count_clone = poll_count.clone();
+
+        let mut mock_poller = mock_poller();
+        mock_poller.expect_poll().returning(move || {
+            poll_count_clone.fetch_add(1, Ordering::SeqCst);
+            Some(Ok(PollActivityTaskQueueResponse::default()))
+        });
+
+        let sem = Arc::new(fixed_size_permit_dealer::<ActivitySlotKind>(10));
+        let shutdown_token = CancellationToken::new();
+
+        let stream = new_activity_task_poller(
+            Box::new(MockPermittedPollBuffer::new(sem, mock_poller)),
+            MetricsContext::no_op(),
+            shutdown_token.clone(),
+        );
+        pin_mut!(stream);
+
+        shutdown_token.cancel();
+
+        let result = tokio::time::timeout(std::time::Duration::from_secs(2), stream.next()).await;
+        assert!(
+            result.is_ok(),
+            "Stream should terminate promptly after shutdown, not hang"
+        );
+        assert!(
+            result.unwrap().is_none(),
+            "Stream should return None (terminated) on empty response after shutdown"
+        );
+
+        let total = poll_count.load(Ordering::SeqCst);
+        assert!(
+            total < 5,
+            "Expected stream to terminate quickly, but poller was called {total} times"
+        );
+    }
+
+    #[tokio::test]
+    async fn empty_response_before_shutdown_retries() {
+        let mut mock_poller = mock_poller();
+        let call_count = Arc::new(AtomicUsize::new(0));
+        let call_count_clone = call_count.clone();
+        mock_poller.expect_poll().returning(move || {
+            let n = call_count_clone.fetch_add(1, Ordering::SeqCst);
+            if n < 2 {
+                Some(Ok(PollActivityTaskQueueResponse::default()))
+            } else {
+                None
+            }
+        });
+
+        let sem = Arc::new(fixed_size_permit_dealer::<ActivitySlotKind>(10));
+        let shutdown_token = CancellationToken::new();
+
+        let stream = new_activity_task_poller(
+            Box::new(MockPermittedPollBuffer::new(sem, mock_poller)),
+            MetricsContext::no_op(),
+            shutdown_token,
+        );
+        pin_mut!(stream);
+
+        // Without shutdown, empty responses should be skipped and the stream terminates
+        // only when the poller returns None.
+        let result = stream.next().await;
+        assert!(
+            result.is_none(),
+            "Stream should end when poller returns None"
+        );
+        assert_eq!(call_count.load(Ordering::SeqCst), 3);
+    }
+}

package/sdk-core/crates/sdk-core/src/pollers/poll_buffer.rs

@@ -2,8 +2,8 @@ use crate::{
     abstractions::{ActiveCounter, MeteredPermitDealer, OwnedMeteredSemPermit, dbg_panic},
     pollers::{self, Poller},
     worker::{
-        ActivitySlotKind, NexusSlotKind, PollerBehavior, SlotKind, WFTPollerShared,
-        WorkflowSlotKind,
+        ActivitySlotKind, NamespaceCapabilities, NexusSlotKind, PollerBehavior, SlotKind,
+        WFTPollerShared, WorkflowSlotKind,
         client::{PollActivityOptions, PollOptions, PollWorkflowOptions, WorkerClient},
     },
 };
@@ -77,7 +77,7 @@ impl LongPollBuffer<PollWorkflowTaskQueueResponse, WorkflowSlotKind> {
         num_pollers_handler: Option<impl Fn(usize) + Send + Sync + 'static>,
         options: WorkflowTaskOptions,
         last_successful_poll_time: Arc<AtomicCell<Option<SystemTime>>>,
-        graceful_poll_shutdown: Arc<AtomicBool>,
+        capabilities: Arc<NamespaceCapabilities>,
     ) -> Self {
         let is_sticky = sticky_queue.is_some();
         let poll_scaler = PollScaler::new(
@@ -140,7 +140,7 @@ impl LongPollBuffer<PollWorkflowTaskQueueResponse, WorkflowSlotKind> {
             poll_scaler,
             pre_permit_delay,
             post_poll_fn,
-            graceful_poll_shutdown,
+            capabilities,
         )
     }
 }
@@ -156,7 +156,7 @@ impl LongPollBuffer<PollActivityTaskQueueResponse, ActivitySlotKind> {
         num_pollers_handler: Option<impl Fn(usize) + Send + Sync + 'static>,
         options: ActivityTaskOptions,
         last_successful_poll_time: Arc<AtomicCell<Option<SystemTime>>>,
-        graceful_poll_shutdown: Arc<AtomicBool>,
+        capabilities: Arc<NamespaceCapabilities>,
     ) -> Self {
         let pre_permit_delay = options
             .max_worker_acts_per_second
@@ -209,7 +209,7 @@ impl LongPollBuffer<PollActivityTaskQueueResponse, ActivitySlotKind> {
             poll_scaler,
             pre_permit_delay,
             None::<fn(&PollActivityTaskQueueResponse)>,
-            graceful_poll_shutdown,
+            capabilities,
         )
     }
 }
@@ -225,7 +225,7 @@ impl LongPollBuffer<PollNexusTaskQueueResponse, NexusSlotKind> {
         num_pollers_handler: Option<impl Fn(usize) + Send + Sync + 'static>,
         last_successful_poll_time: Arc<AtomicCell<Option<SystemTime>>>,
         send_heartbeat: bool,
-        graceful_poll_shutdown: Arc<AtomicBool>,
+        capabilities: Arc<NamespaceCapabilities>,
     ) -> Self {
         let no_retry = if matches!(poller_behavior, PollerBehavior::Autoscaling { .. }) {
             Some(NoRetryOnMatching {
@@ -262,7 +262,7 @@ impl LongPollBuffer<PollNexusTaskQueueResponse, NexusSlotKind> {
             ),
             None::<fn() -> BoxFuture<'static, ()>>,
             None::<fn(&PollNexusTaskQueueResponse)>,
-            graceful_poll_shutdown,
+            capabilities,
         )
     }
 }
@@ -288,7 +288,7 @@ where
         mut poll_scaler: PollScaler<F>,
         pre_permit_delay: Option<impl Fn() -> DelayFut + Send + Sync + 'static>,
         post_poll_fn: Option<impl Fn(&T) + Send + Sync + 'static>,
-        graceful_shutdown: Arc<AtomicBool>,
+        capabilities: Arc<NamespaceCapabilities>,
     ) -> Self
     where
         FT: Future<Output = pollers::Result<T>> + Send,
@@ -359,11 +359,9 @@ where
                         } else {
                             None
                         };
-                    let graceful_shutdown = graceful_shutdown.clone();
+                    let capabilities = capabilities.clone();
                     let poll_task = tokio::spawn(async move {
-                        let shutdown_clone = shutdown.clone();
-
-                        let r = if graceful_shutdown.load(Ordering::Relaxed) {
+                        let r = if capabilities.graceful_poll_shutdown() {
                             pf(timeout_override).await
                         } else {
                             let poll_interruptor = shutdown.cancelled().then(|_| async move {
@@ -383,10 +381,11 @@ where
                         }
                         let (should_forward, backoff_duration) = report_handle.poll_result(&r);
                         if let Some(duration) = backoff_duration {
-                            // Apply backoff BEFORE dropping active_guard to prevent next poll from starting
+                            // Apply backoff BEFORE dropping active_guard to prevent next poll from
+                            // starting
                             tokio::select! {
                                 _ = tokio::time::sleep(duration) => return,
-                                _ = shutdown_clone.cancelled() => (),
+                                _ = shutdown.cancelled() => (),
                             };
                         }
                         drop(active_guard);
@@ -853,7 +852,9 @@ mod tests {
                 wft_poller_shared: Some(Arc::new(WFTPollerShared::new(Some(10)))),
             },
             Arc::new(AtomicCell::new(None)),
-            Arc::new(AtomicBool::new(false)),
+            Arc::new(NamespaceCapabilities {
+                graceful_poll_shutdown: AtomicBool::new(false),
+            }),
         );
 
         // Poll a bunch of times, "interrupting" it each time, we should only actually have polled
@@ -910,7 +911,9 @@ mod tests {
                 wft_poller_shared: Some(Arc::new(WFTPollerShared::new(Some(1)))),
             },
             Arc::new(AtomicCell::new(None)),
-            Arc::new(AtomicBool::new(false)),
+            Arc::new(NamespaceCapabilities {
+                graceful_poll_shutdown: AtomicBool::new(false),
+            }),
         );
 
         // Should not see error, unwraps should get empty response
@@ -987,7 +990,9 @@ mod tests {
                 wft_poller_shared: Some(Arc::new(WFTPollerShared::new(Some(10)))),
             },
             Arc::new(AtomicCell::new(None)),
-            Arc::new(AtomicBool::new(false)),
+            Arc::new(NamespaceCapabilities {
+                graceful_poll_shutdown: AtomicBool::new(false),
+            }),
         );
 
         let first_task = pb.poll().await.expect("Should get first task");
@@ -1093,7 +1098,9 @@ mod tests {
                 wft_poller_shared: Some(Arc::new(WFTPollerShared::new(Some(10)))),
             },
             Arc::new(AtomicCell::new(None)),
-            Arc::new(AtomicBool::new(false)),
+            Arc::new(NamespaceCapabilities {
+                graceful_poll_shutdown: AtomicBool::new(false),
+            }),
         ));
 
         // Trigger the first poll to initialize and get the scaling decision
@@ -1174,7 +1181,9 @@ mod tests {
                 wft_poller_shared: None,
             },
             Arc::new(AtomicCell::new(None)),
-            Arc::new(AtomicBool::new(graceful)),
+            Arc::new(NamespaceCapabilities {
+                graceful_poll_shutdown: AtomicBool::new(graceful),
+            }),
         );
 
         let first = pb.poll().await.unwrap().unwrap();

package/sdk-core/crates/sdk-core/src/worker/activities.rs

@@ -733,7 +733,7 @@ mod tests {
         abstractions::tests::fixed_size_permit_dealer,
         pollers::{ActivityTaskOptions, LongPollBuffer},
         prost_dur,
-        worker::{PollerBehavior, client::mocks::mock_worker_client},
+        worker::{NamespaceCapabilities, PollerBehavior, client::mocks::mock_worker_client},
     };
     use crossbeam_utils::atomic::AtomicCell;
     use temporalio_common::protos::coresdk::activity_result::ActivityExecutionResult;
@@ -781,7 +781,9 @@ mod tests {
                 max_tps: None,
             },
             Arc::new(AtomicCell::new(None)),
-            Arc::new(AtomicBool::new(false)),
+            Arc::new(NamespaceCapabilities {
+                graceful_poll_shutdown: AtomicBool::new(false),
+            }),
         );
         let atm = WorkerActivityTasks::new(
             sem.clone(),
@@ -874,7 +876,9 @@ mod tests {
                 max_tps: None,
             },
             Arc::new(AtomicCell::new(None)),
-            Arc::new(AtomicBool::new(false)),
+            Arc::new(NamespaceCapabilities {
+                graceful_poll_shutdown: AtomicBool::new(false),
+            }),
         );
         let atm = WorkerActivityTasks::new(
             sem.clone(),
@@ -949,7 +953,9 @@ mod tests {
                 max_tps: None,
             },
             Arc::new(AtomicCell::new(None)),
-            Arc::new(AtomicBool::new(false)),
+            Arc::new(NamespaceCapabilities {
+                graceful_poll_shutdown: AtomicBool::new(false),
+            }),
         );
         let atm = WorkerActivityTasks::new(
             sem.clone(),

package/sdk-core/crates/sdk-core/src/worker/mod.rs

@@ -61,7 +61,7 @@ use anyhow::bail;
 use crossbeam_utils::atomic::AtomicCell;
 use futures_util::{StreamExt, stream};
 use gethostname::gethostname;
-use parking_lot::RwLock;
+use parking_lot::{Mutex, RwLock};
 use slot_provider::SlotProvider;
 use std::{
     any::Any,
@@ -415,9 +415,24 @@ pub struct Worker {
     client_worker_registrator: Arc<ClientWorkerRegistrator>,
     /// Status of the worker
     status: Arc<RwLock<WorkerStatus>>,
-    /// Set during validate() when server supports graceful poll cancellation on shutdown.
-    /// Shared with pollers so they can decide per-poll whether to hard-kill or wait.
-    graceful_poll_shutdown: Arc<AtomicBool>,
+    /// Capabilities as returned by a describe namespace rpc. Not set until after validate() is
+    /// called.
+    capabilities: Arc<NamespaceCapabilities>,
+    /// Handle for the spawned ShutdownWorker RPC task, awaited during shutdown.
+    shutdown_rpc_handle: Mutex<Option<tokio::task::JoinHandle<()>>>,
+}
+
+/// Namespace capabilities discovered via `describe_namespace` during worker validation.
+pub struct NamespaceCapabilities {
+    pub(crate) graceful_poll_shutdown: AtomicBool,
+}
+
+impl NamespaceCapabilities {
+    /// Returns true if the server supports graceful poll cancellation on shutdown, so pollers
+    /// can let in-flight polls complete rather than hard-killing them.
+    pub fn graceful_poll_shutdown(&self) -> bool {
+        self.graceful_poll_shutdown.load(Ordering::Relaxed)
+    }
 }
 
 struct AllPermitsTracker {
@@ -490,11 +505,12 @@ impl Worker {
                         memo_size_limit_error: api_limits.memo_size_limit_error,
                     })
                 });
-                if ns_info
-                    .and_then(|ns| ns.capabilities)
-                    .is_some_and(|caps| caps.worker_poll_complete_on_shutdown)
+                if let Some(caps) = ns_info.and_then(|ns| ns.capabilities)
+                    && caps.worker_poll_complete_on_shutdown
                 {
-                    self.graceful_poll_shutdown.store(true, Ordering::Relaxed);
+                    self.capabilities
+                        .graceful_poll_shutdown
+                        .store(true, Ordering::Relaxed);
                 }
                 Ok(NamespaceInfo { limits })
             }
@@ -616,7 +632,9 @@ impl Worker {
         let wf_sticky_last_suc_poll_time = Arc::new(AtomicCell::new(None));
         let act_last_suc_poll_time = Arc::new(AtomicCell::new(None));
         let nexus_last_suc_poll_time = Arc::new(AtomicCell::new(None));
-        let graceful_poll_shutdown = Arc::new(AtomicBool::new(false));
+        let capabilities = Arc::new(NamespaceCapabilities {
+            graceful_poll_shutdown: AtomicBool::new(false),
+        });
 
         let nexus_slots = MeteredPermitDealer::new(
             tuner.nexus_task_slot_supplier(),
@@ -637,7 +655,7 @@ impl Worker {
                         &wft_slots,
                         wf_last_suc_poll_time.clone(),
                         wf_sticky_last_suc_poll_time.clone(),
-                        graceful_poll_shutdown.clone(),
+                        capabilities.clone(),
                     )
                     .boxed();
                     let stream = if !client.is_mock() {
@@ -667,7 +685,7 @@ impl Worker {
                             max_tps: config.max_task_queue_activities_per_second,
                         },
                         act_last_suc_poll_time.clone(),
-                        graceful_poll_shutdown.clone(),
+                        capabilities.clone(),
                     );
                     Some(Box::from(ap) as BoxedActPoller)
                 } else {
@@ -685,7 +703,7 @@ impl Worker {
                         Some(move |np| np_metrics.record_num_pollers(np)),
                         nexus_last_suc_poll_time.clone(),
                         shared_namespace_worker,
-                        graceful_poll_shutdown.clone(),
+                        capabilities.clone(),
                     )) as BoxedNexusPoller)
                 } else {
                     None
@@ -905,7 +923,8 @@ impl Worker {
             nexus_mgr,
             client_worker_registrator,
             status: worker_status,
-            graceful_poll_shutdown,
+            capabilities,
+            shutdown_rpc_handle: Mutex::new(None),
         })
     }
 
@@ -923,43 +942,12 @@ impl Worker {
     /// [Worker::finalize_shutdown].
     pub async fn shutdown(&self) {
         self.initiate_shutdown();
-        {
-            *self.status.write() = WorkerStatus::ShuttingDown;
-        }
-        let heartbeat = self
-            .client_worker_registrator
-            .heartbeat_manager
-            .as_ref()
-            .map(|hm| hm.heartbeat_callback.clone()());
-        let sticky_name = self
-            .workflows
-            .as_ref()
-            .and_then(|wf| wf.get_sticky_queue_name())
-            .unwrap_or_default();
-        // This is a best effort call and we can still shutdown the worker if it fails
-        let task_queue_types = self.config.task_types.to_task_queue_types();
-        match self
-            .client
-            .shutdown_worker(
-                sticky_name,
-                self.config.task_queue.clone(),
-                task_queue_types,
-                heartbeat,
-            )
-            .await
-        {
-            Err(err)
-                if !matches!(
-                    err.code(),
-                    tonic::Code::Unimplemented | tonic::Code::Unavailable
-                ) =>
-            {
-                warn!(
-                    "shutdown_worker rpc errored during worker shutdown: {:?}",
-                    err
-                );
-            }
-            _ => {}
+
+        // Ensure the ShutdownWorker RPC completes before waiting for polls to drain,
+        // otherwise graceful poll shutdown deadlocks.
+        let handle = self.shutdown_rpc_handle.lock().take();
+        if let Some(handle) = handle {
+            let _ = handle.await;
         }
 
         // We need to wait for all local activities to finish so no more workflow task heartbeats
@@ -1354,8 +1342,15 @@ impl Worker {
         &self.config
     }
 
-    /// Initiate shutdown. See [Worker::shutdown], this is just a sync version that starts the
-    /// process. You can then wait on `shutdown` or [Worker::finalize_shutdown].
+    /// Returns the namespace capabilities discovered during [Worker::validate].
+    pub fn get_namespace_capabilities(&self) -> &NamespaceCapabilities {
+        &self.capabilities
+    }
+
+    /// Initiate shutdown, including spawning the `ShutdownWorker` RPC so the server can complete
+    /// in-flight polls. The RPC runs in a background task and is awaited in [Worker::shutdown].
+    ///
+    /// You can then wait on `shutdown` or [Worker::finalize_shutdown].
     pub fn initiate_shutdown(&self) {
         if !self.shutdown_token.is_cancelled() {
             info!(
@@ -1364,6 +1359,7 @@ impl Worker {
                 "Initiated shutdown",
             );
         }
+        let already_initiated_shutdown = self.shutdown_token.is_cancelled();
         self.shutdown_token.cancel();
         {
             *self.status.write() = WorkerStatus::ShuttingDown;
@@ -1398,6 +1394,47 @@ impl Worker {
                 la_mgr.workflows_have_shutdown();
             }
         }
+
+        // Spawn the ShutdownWorker RPC so the server can complete in-flight polls.
+        // The handle is stored and awaited in shutdown() to ensure completion.
+        let mut guard = self.shutdown_rpc_handle.lock();
+        if guard.is_some() || already_initiated_shutdown {
+            return;
+        }
+
+        let client = self.client.clone();
+        let sticky_name = self
+            .workflows
+            .as_ref()
+            .and_then(|wf| wf.get_sticky_queue_name())
+            .unwrap_or_default();
+        let task_queue = self.config.task_queue.clone();
+        let task_queue_types = self.config.task_types.to_task_queue_types();
+        let heartbeat = self
+            .client_worker_registrator
+            .heartbeat_manager
+            .as_ref()
+            .map(|hm| hm.heartbeat_callback.clone()());
+        let handle = tokio::spawn(async move {
+            match client
+                .shutdown_worker(sticky_name, task_queue, task_queue_types, heartbeat)
+                .await
+            {
+                Err(err)
+                    if !matches!(
+                        err.code(),
+                        tonic::Code::Unimplemented | tonic::Code::Unavailable
+                    ) =>
+                {
+                    warn!(
+                        "shutdown_worker rpc errored during worker shutdown: {:?}",
+                        err
+                    );
+                }
+                _ => {}
+            }
+        });
+        *guard = Some(handle);
     }
 
     /// Unique identifier for this worker instance.

package/sdk-core/crates/sdk-core/src/worker/workflow/wft_poller.rs

@@ -4,12 +4,12 @@ use crate::{
     pollers::{BoxedWFPoller, LongPollBuffer, Poller, WorkflowTaskOptions, WorkflowTaskPoller},
     protosext::ValidPollWFTQResponse,
     telemetry::metrics::{workflow_poller, workflow_sticky_poller},
-    worker::{WorkflowSlotKind, client::WorkerClient, wft_poller_behavior},
+    worker::{NamespaceCapabilities, WorkflowSlotKind, client::WorkerClient, wft_poller_behavior},
 };
 use crossbeam_utils::atomic::AtomicCell;
 use futures_util::{Stream, stream};
 use std::{
-    sync::{Arc, OnceLock, atomic::AtomicBool},
+    sync::{Arc, OnceLock},
     time::SystemTime,
 };
 use temporalio_common::protos::temporal::api::workflowservice::v1::PollWorkflowTaskQueueResponse;
@@ -26,7 +26,7 @@ pub(crate) fn make_wft_poller(
     wft_slots: &MeteredPermitDealer<WorkflowSlotKind>,
     last_successful_poll_time: Arc<AtomicCell<Option<SystemTime>>>,
     sticky_last_successful_poll_time: Arc<AtomicCell<Option<SystemTime>>>,
-    graceful_poll_shutdown: Arc<AtomicBool>,
+    capabilities: Arc<NamespaceCapabilities>,
 ) -> impl Stream<
     Item = Result<
         (
@@ -60,7 +60,7 @@ pub(crate) fn make_wft_poller(
             wft_poller_shared: wft_poller_shared.clone(),
         },
         last_successful_poll_time,
-        graceful_poll_shutdown.clone(),
+        capabilities.clone(),
     );
     let sticky_queue_poller = sticky_queue_name.as_ref().map(|sqn| {
         let sticky_metrics = metrics.with_new_attrs([workflow_sticky_poller()]);
@@ -76,7 +76,7 @@ pub(crate) fn make_wft_poller(
             }),
             WorkflowTaskOptions { wft_poller_shared },
             sticky_last_successful_poll_time,
-            graceful_poll_shutdown,
+            capabilities,
         )
     });
     let wf_task_poll_buffer = Box::new(WorkflowTaskPoller::new(
@@ -228,11 +228,11 @@ fn new_wft_poller(
                     warn!(error=?e, "Error while polling for workflow tasks");
                     Some((Err(e), (poller, metrics)))
                 }
-                // If poller returns None, it's dead, thus we also return None to terminate this
-                // stream.
+                // If poller returns None, it's dead, thus we also return None to terminate
+                // this stream.
                 None => {
-                    // Make sure we call the actual shutdown function here to propagate any panics
-                    // inside the polling tasks as errors.
+                    // Make sure we call the actual shutdown function here to propagate any
+                    // panics inside the polling tasks as errors.
                     poller.shutdown_box().await;
                     None
                 }
@@ -281,6 +281,25 @@ mod tests {
         assert_matches!(stream.next().await, None);
     }
 
+    /// When the underlying poller returns None (indicating shutdown), the wrapping WFT stream
+    /// should also return None to terminate.
+    #[tokio::test]
+    async fn poller_returning_none_terminates_wft_stream() {
+        let mut mock_poller = mock_poller();
+        mock_poller.expect_poll().times(1).returning(|| None);
+        mock_poller.expect_shutdown().times(1).returning(|| ());
+
+        let sem = Arc::new(fixed_size_permit_dealer::<WorkflowSlotKind>(10));
+
+        let stream = new_wft_poller(
+            Box::new(MockPermittedPollBuffer::new(sem, mock_poller)),
+            MetricsContext::no_op(),
+        );
+        pin_mut!(stream);
+
+        assert_matches!(stream.next().await, None);
+    }
+
     #[tokio::test]
     async fn poll_errors_do_produce_responses() {
         let mut mock_poller = mock_poller();

package/sdk-core/crates/sdk-core/tests/cloud_tests.rs

@@ -32,3 +32,8 @@ async fn grpc_message_too_large_test() {
 async fn priority_values_sent_to_server() {
     shared_tests::priority::priority_values_sent_to_server().await
 }
+
+#[tokio::test]
+async fn shutdown_during_active_timer_activity_workflows() {
+    shared_tests::shutdown_during_active_timer_activity_workflows().await
+}

package/sdk-core/crates/sdk-core/tests/common/mod.rs

@@ -1076,6 +1076,10 @@ pub(crate) fn integ_dev_server_config(
             "frontend.WorkerHeartbeatsEnabled=true".to_owned(),
             "--dynamic-config-value".to_owned(),
             "frontend.ListWorkersEnabled=true".to_owned(),
+            "--dynamic-config-value".to_owned(),
+            "frontend.enableCancelWorkerPollsOnShutdown=true".to_owned(),
+            "--dynamic-config-value".to_owned(),
+            "matching.rps=12000".to_owned(),
             "--search-attribute".to_string(),
             format!("{SEARCH_ATTR_TXT}=Text"),
             "--search-attribute".to_string(),

package/sdk-core/crates/sdk-core/tests/integ_tests/worker_heartbeat_tests.rs

@@ -247,43 +247,67 @@ async fn docker_worker_heartbeat_basic(#[values("otel", "prom", "no_metrics")] b
         );
         in_activity_checks(heartbeat, &start_time, &heartbeat_time);
         acts_done.notify_one();
+
+        // Poll until the heartbeat reflects shutdown with the second WFT processed.
+        // The worker stays alive (join! waits for both futures) so heartbeats keep firing.
+        eventually(
+            || {
+                let mut rc = raw_client.clone();
+                let ns = client.namespace().to_owned();
+                async move {
+                    let workers_list = WorkflowService::list_workers(
+                        &mut rc,
+                        ListWorkersRequest {
+                            namespace: ns,
+                            page_size: 100,
+                            next_page_token: Vec::new(),
+                            query: String::new(),
+                        }
+                        .into_request(),
+                    )
+                    .await
+                    .unwrap()
+                    .into_inner();
+                    #[allow(deprecated)]
+                    let hb = workers_list
+                        .workers_info
+                        .iter()
+                        .find_map(|wi| {
+                            wi.worker_heartbeat.as_ref().filter(|hb| {
+                                hb.worker_instance_key == worker_instance_key.to_string()
+                            })
+                        })
+                        .unwrap()
+                        .clone();
+                    let tasks_done = hb
+                        .workflow_task_slots_info
+                        .as_ref()
+                        .is_some_and(|s| s.total_processed_tasks >= 2);
+                    let is_shutting_down = hb.status == WorkerStatus::ShuttingDown as i32;
+                    if tasks_done && is_shutting_down {
+                        Ok(hb)
+                    } else {
+                        Err(anyhow::anyhow!(
+                            "Heartbeat not ready: tasks={}, shutting_down={}",
+                            hb.workflow_task_slots_info
+                                .as_ref()
+                                .map_or(0, |s| s.total_processed_tasks),
+                            is_shutting_down,
+                        ))
+                    }
+                }
+            },
+            Duration::from_secs(5),
+        )
+        .await
+        .map(|hb| after_shutdown_checks(&hb, &wf_name, &start_time, &heartbeat_time))
+        .unwrap();
     };
 
     let runner = async move {
         worker.run_until_done().await.unwrap();
     };
     tokio::join!(test_fut, runner);
-
-    let client = starter.get_client().await;
-    let mut raw_client = client.clone();
-    let workers_list = WorkflowService::list_workers(
-        &mut raw_client,
-        ListWorkersRequest {
-            namespace: client.namespace().to_owned(),
-            page_size: 100,
-            next_page_token: Vec::new(),
-            query: String::new(),
-        }
-        .into_request(),
-    )
-    .await
-    .unwrap()
-    .into_inner();
-    // Since list_workers finds all workers in the namespace, must find specific worker used in this
-    // test
-    let worker_info = workers_list
-        .workers_info
-        .iter()
-        .find(|worker_info| {
-            if let Some(hb) = worker_info.worker_heartbeat.as_ref() {
-                hb.worker_instance_key == worker_instance_key.to_string()
-            } else {
-                false
-            }
-        })
-        .unwrap();
-    let heartbeat = worker_info.worker_heartbeat.as_ref().unwrap();
-    after_shutdown_checks(heartbeat, &wf_name, &start_time, &heartbeat_time);
 }
 
 // Tests that rely on Prometheus running in a docker container need to start
@@ -521,23 +545,7 @@ fn after_shutdown_checks(
     ));
 
     let workflow_task_slots = heartbeat.workflow_task_slots_info.clone().unwrap();
-    assert_eq!(workflow_task_slots.current_available_slots, 5);
-    assert_eq!(workflow_task_slots.current_used_slots, 1);
     assert_eq!(workflow_task_slots.total_processed_tasks, 2);
-    assert_eq!(workflow_task_slots.slot_supplier_kind, "Fixed");
-    let activity_task_slots = heartbeat.activity_task_slots_info.clone().unwrap();
-    assert_eq!(activity_task_slots.current_available_slots, 5);
-    assert_eq!(workflow_task_slots.current_used_slots, 1);
-    assert_eq!(activity_task_slots.slot_supplier_kind, "Fixed");
-    assert_eq!(activity_task_slots.last_interval_processed_tasks, 1);
-    let nexus_task_slots = heartbeat.nexus_task_slots_info.clone().unwrap();
-    assert_eq!(nexus_task_slots.current_available_slots, 0);
-    assert_eq!(nexus_task_slots.current_used_slots, 0);
-    assert_eq!(nexus_task_slots.slot_supplier_kind, "Fixed");
-    let local_activity_task_slots = heartbeat.local_activity_slots_info.clone().unwrap();
-    assert_eq!(local_activity_task_slots.current_available_slots, 100);
-    assert_eq!(local_activity_task_slots.current_used_slots, 0);
-    assert_eq!(local_activity_task_slots.slot_supplier_kind, "Fixed");
 
     let workflow_poller_info = heartbeat.workflow_poller_info.unwrap();
     assert!(!workflow_poller_info.is_autoscaling);
@@ -559,7 +567,6 @@ fn after_shutdown_checks(
     ));
 
     assert_eq!(heartbeat.total_sticky_cache_hit, 1);
-    assert_eq!(heartbeat.current_sticky_cache_size, 0);
     assert_eq!(
         heartbeat.plugins,
         vec![

package/sdk-core/crates/sdk-core/tests/integ_tests/worker_tests.rs

@@ -983,3 +983,8 @@ fn test_default_build_id() {
     assert!(!o.deployment_options.version.build_id.is_empty());
     assert_ne!(o.deployment_options.version.build_id, "undetermined");
 }
+
+#[tokio::test]
+async fn shutdown_during_active_timer_activity_workflows() {
+    shared_tests::shutdown_during_active_timer_activity_workflows().await
+}

package/sdk-core/crates/sdk-core/tests/shared_tests/mod.rs

@@ -1,9 +1,15 @@
 //! Shared tests that are meant to be run against both local dev server and cloud
 
-use crate::common::CoreWfStarter;
-use std::sync::{
-    Arc,
-    atomic::{AtomicBool, Ordering::Relaxed},
+use crate::common::{CoreWfStarter, activity_functions::StdActivities};
+use std::{
+    sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering::Relaxed},
+    },
+    time::Duration,
+};
+use temporalio_client::{
+    UntypedWorkflow, WorkflowFetchHistoryOptions, WorkflowStartOptions, WorkflowTerminateOptions,
 };
 use temporalio_common::{
     protos::temporal::api::{
@@ -15,7 +21,7 @@ use temporalio_common::{
     worker::WorkerTaskTypes,
 };
 use temporalio_macros::{workflow, workflow_methods};
-use temporalio_sdk::{WorkflowContext, WorkflowResult};
+use temporalio_sdk::{ActivityOptions, WorkflowContext, WorkflowResult, WorkflowTermination};
 
 pub(crate) mod priority;
 
@@ -92,3 +98,115 @@ pub(crate) fn is_oversize_grpc_event(
             false
         }
 }
+
+#[workflow]
+#[derive(Default)]
+struct ShutdownTimerActivityLoopWf;
+
+#[workflow_methods]
+impl ShutdownTimerActivityLoopWf {
+    #[run]
+    async fn run(ctx: &mut WorkflowContext<Self>) -> WorkflowResult<()> {
+        loop {
+            ctx.timer(Duration::from_millis(10)).await;
+            ctx.start_activity(
+                StdActivities::no_op,
+                (),
+                ActivityOptions {
+                    start_to_close_timeout: Some(Duration::from_secs(10)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .map_err(|e| WorkflowTermination::from(anyhow::Error::from(e)))?;
+        }
+    }
+}
+
+/// Starts 10 workflows that each run a tight timer+activity loop, then shuts down the worker
+/// and verifies:
+///   1. Shutdown completes rapidly (< 5s)
+///   2. No workflow task failures or timeouts appear in any workflow's history
+pub(crate) async fn shutdown_during_active_timer_activity_workflows() {
+    let wf_name = "shutdown_during_active_timer_activity_workflows";
+    let num_workflows = 10;
+
+    let mut starter =
+        if let Some(wfs) = CoreWfStarter::new_cloud_or_local(wf_name, ">=1.6.3-serverless").await {
+            wfs
+        } else {
+            return;
+        };
+    starter.sdk_config.register_activities(StdActivities);
+    let mut worker = starter.worker().await;
+    worker.register_workflow::<ShutdownTimerActivityLoopWf>();
+
+    let core = worker.core_worker();
+    core.validate().await.unwrap();
+    assert!(
+        core.get_namespace_capabilities().graceful_poll_shutdown(),
+        "Server must support graceful poll shutdown for this test"
+    );
+
+    let task_queue = starter.get_task_queue().to_owned();
+    let mut wf_ids = Vec::with_capacity(num_workflows);
+    for i in 0..num_workflows {
+        let wf_id = format!("{task_queue}-{i}");
+        worker
+            .submit_workflow(
+                ShutdownTimerActivityLoopWf::run,
+                (),
+                WorkflowStartOptions::new(task_queue.clone(), wf_id.clone()).build(),
+            )
+            .await
+            .unwrap();
+        wf_ids.push(wf_id);
+    }
+    // Don't wait for workflow completion — these loop forever
+    worker.fetch_results = false;
+
+    let shutdown_handle = worker.inner_mut().shutdown_handle();
+    let run_fut = async { worker.run_until_done().await.unwrap() };
+
+    let shutdown_fut = async {
+        // Let workflows run a few iterations
+        tokio::time::sleep(Duration::from_secs(2)).await;
+        shutdown_handle();
+    };
+
+    let shutdown_start = std::time::Instant::now();
+    tokio::join!(run_fut, shutdown_fut);
+    let shutdown_elapsed = shutdown_start.elapsed();
+
+    assert!(
+        shutdown_elapsed < Duration::from_secs(5),
+        "Worker shutdown took {shutdown_elapsed:?}, expected < 5s"
+    );
+
+    let client = starter.get_client().await;
+    for wf_id in &wf_ids {
+        client
+            .get_workflow_handle::<UntypedWorkflow>(wf_id)
+            .terminate(WorkflowTerminateOptions::default())
+            .await
+            .unwrap();
+
+        let history = client
+            .get_workflow_handle::<UntypedWorkflow>(wf_id)
+            .fetch_history(WorkflowFetchHistoryOptions::default())
+            .await
+            .unwrap();
+        let bad_events: Vec<_> = history
+            .events()
+            .iter()
+            .filter(|e| {
+                e.event_type() == EventType::WorkflowTaskFailed
+                    || e.event_type() == EventType::WorkflowTaskTimedOut
+            })
+            .collect();
+        assert!(
+            bad_events.is_empty(),
+            "Workflow {wf_id} had unexpected WFT failures/timeouts: {bad_events:?}"
+        );
+    }
+}

package/sdk-core/crates/sdk-core-c-bridge/src/worker.rs

@@ -929,6 +929,7 @@ pub extern "C" fn temporal_core_worker_request_workflow_eviction(
 #[unsafe(no_mangle)]
 pub extern "C" fn temporal_core_worker_initiate_shutdown(worker: *mut Worker) {
     let worker = unsafe { &*worker };
+    enter_sync!(worker.runtime);
     worker.worker.as_ref().unwrap().initiate_shutdown();
 }
 

package/src/worker.rs

@@ -339,6 +339,12 @@ pub fn worker_complete_nexus_task(
 #[js_function]
 pub fn worker_initiate_shutdown(worker: OpaqueInboundHandle<Worker>) -> BridgeResult<()> {
     let worker_ref = worker.borrow()?;
+
+    // Core worker shutdown now spawns a Tokio task, so this sync Neon binding must
+    // enter Core's Tokio runtime before initiating shutdown.
+    let runtime = worker_ref.core_runtime.clone();
+    enter_sync!(runtime);
+
     worker_ref.core_worker.initiate_shutdown();
     Ok(())
 }
@@ -868,7 +874,7 @@ mod custom_slot_supplier {
                     }
                     Err(err) => {
                         warn!("Error reserving slot: {err:?}");
-                        tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
+                        tokio::time::sleep(std::time::Duration::from_secs(1)).await;
                     }
                 }
             }