@temporalio/core-bridge 1.13.2 → 1.14.0

package/sdk-core/{core → crates/sdk-core}/src/worker/workflow/mod.rs

@@ -23,7 +23,11 @@ use crate::{
     internal_flags::InternalFlags,
     pollers::TrackedPermittedTqResp,
     protosext::{ValidPollWFTQResponse, protocol_messages::IncomingProtocolMessage},
-    telemetry::{TelemetryInstance, VecDisplayer, set_trace_subscriber_for_current_thread},
+    telemetry::{
+        VecDisplayer,
+        metrics::{self, FailureReason},
+        set_trace_subscriber_for_current_thread,
+    },
     worker::{
         LocalActRequest, LocalActivityExecutionResult, LocalActivityResolution,
         PostActivateHookData,
@@ -31,6 +35,7 @@ use crate::{
         client::{LegacyQueryResult, WorkerClient, WorkflowTaskCompletion},
         workflow::{
             history_update::HistoryPaginator,
+            machines::MachineError,
             managed_run::RunUpdateAct,
             wft_extraction::{HistoryFetchReq, WFTExtractor, WFTStreamIn},
             wft_poller::validate_wft,
@@ -42,7 +47,6 @@ use anyhow::anyhow;
 use futures_util::{Stream, StreamExt, future::abortable, stream, stream::BoxStream};
 use itertools::Itertools;
 use prost_types::TimestampError;
-use rustfsm::MachineError;
 use std::{
     cell::RefCell,
     collections::VecDeque,
@@ -55,35 +59,36 @@ use std::{
     thread,
     time::{Duration, Instant},
 };
-use temporal_client::MESSAGE_TOO_LARGE_KEY;
-use temporal_sdk_core_api::{
+use temporalio_client::MESSAGE_TOO_LARGE_KEY;
+use temporalio_common::{
     errors::{CompleteWfError, PollError},
-    worker::{ActivitySlotKind, WorkerConfig, WorkflowSlotKind},
-};
-use temporal_sdk_core_protos::{
-    TaskToken,
-    coresdk::{
-        workflow_activation::{
-            QueryWorkflow, WorkflowActivation, WorkflowActivationJob,
-            remove_from_cache::EvictionReason, workflow_activation_job,
+    protos::{
+        TaskToken,
+        coresdk::{
+            workflow_activation::{
+                QueryWorkflow, WorkflowActivation, WorkflowActivationJob,
+                remove_from_cache::EvictionReason, workflow_activation_job,
+            },
+            workflow_commands::*,
+            workflow_completion::{
+                self, Failure, WorkflowActivationCompletion, workflow_activation_completion,
+            },
         },
-        workflow_commands::*,
-        workflow_completion,
-        workflow_completion::{
-            Failure, WorkflowActivationCompletion, workflow_activation_completion,
+        temporal::api::{
+            command::v1::{Command as ProtoCommand, Command, command::Attributes},
+            common::v1::{
+                Memo, MeteringMetadata, RetryPolicy, SearchAttributes, WorkflowExecution,
+            },
+            enums::v1::{VersioningBehavior, WorkflowTaskFailedCause},
+            failure::v1::{ApplicationFailureInfo, failure::FailureInfo},
+            protocol::v1::Message as ProtocolMessage,
+            query::v1::WorkflowQuery,
+            sdk::v1::{UserMetadata, WorkflowTaskCompletedMetadata},
+            taskqueue::v1::StickyExecutionAttributes,
+            workflowservice::v1::{PollActivityTaskQueueResponse, get_system_info_response},
         },
     },
-    temporal::api::{
-        command::v1::{Command as ProtoCommand, Command, command::Attributes},
-        common::v1::{Memo, MeteringMetadata, RetryPolicy, SearchAttributes, WorkflowExecution},
-        enums::v1::{VersioningBehavior, WorkflowTaskFailedCause},
-        failure::v1::{ApplicationFailureInfo, failure::FailureInfo},
-        protocol::v1::Message as ProtocolMessage,
-        query::v1::WorkflowQuery,
-        sdk::v1::{UserMetadata, WorkflowTaskCompletedMetadata},
-        taskqueue::v1::StickyExecutionAttributes,
-        workflowservice::v1::{PollActivityTaskQueueResponse, get_system_info_response},
-    },
+    worker::{ActivitySlotKind, WorkerConfig, WorkflowSlotKind},
 };
 use tokio::{
     sync::{
@@ -93,8 +98,9 @@ use tokio::{
     task::{LocalSet, spawn_blocking},
 };
 use tokio_stream::wrappers::UnboundedReceiverStream;
+use tokio_util::either::Either;
 use tokio_util::sync::CancellationToken;
-use tracing::Span;
+use tracing::{Span, Subscriber};
 
 /// Id used by server for "legacy" queries. IE: Queries that come in the `query` rather than
 /// `queries` field of a WFT, and are responded to on the separate `respond_query_task_completed`
@@ -128,9 +134,10 @@ pub(crate) struct Workflows {
     activity_tasks_handle: Option<ActivitiesFromWFTsHandle>,
     /// Ensures we stay at or below this worker's maximum concurrent workflow task limit
     wft_semaphore: MeteredPermitDealer<WorkflowSlotKind>,
-    local_act_mgr: Arc<LocalActivityManager>,
+    local_act_mgr: Option<Arc<LocalActivityManager>>,
     ever_polled: AtomicBool,
     default_versioning_behavior: Option<VersioningBehavior>,
+    metrics: MetricsContext,
 }
 
 pub(crate) struct WorkflowBasics {
@@ -163,16 +170,17 @@ impl Workflows {
         client: Arc<dyn WorkerClient>,
         wft_semaphore: MeteredPermitDealer<WorkflowSlotKind>,
         wft_stream: impl Stream<Item = WFTStreamIn> + Send + 'static,
-        local_activity_request_sink: impl LocalActivityRequestSink,
-        local_act_mgr: Arc<LocalActivityManager>,
-        heartbeat_timeout_rx: UnboundedReceiver<HeartbeatTimeoutMsg>,
+        local_activity_request_sink: Option<impl LocalActivityRequestSink>,
+        local_act_mgr: Option<Arc<LocalActivityManager>>,
+        heartbeat_timeout_rx: Option<UnboundedReceiver<HeartbeatTimeoutMsg>>,
         activity_tasks_handle: Option<ActivitiesFromWFTsHandle>,
-        telem_instance: Option<&TelemetryInstance>,
+        tracing_sub: Option<Arc<dyn Subscriber + Send + Sync>>,
     ) -> Self {
         let (local_tx, local_rx) = unbounded_channel();
         let (fetch_tx, fetch_rx) = unbounded_channel();
         let shutdown_tok = basics.shutdown_token.clone();
         let task_queue = basics.worker_config.task_queue.clone();
+        let metrics = basics.metrics.clone();
         let default_versioning_behavior = basics.default_versioning_behavior;
         let extracted_wft_stream = WFTExtractor::build(
             client.clone(),
@@ -180,15 +188,18 @@ impl Workflows {
             wft_stream,
             UnboundedReceiverStream::new(fetch_rx),
         );
-        let locals_stream = stream::select(
-            UnboundedReceiverStream::new(local_rx),
-            UnboundedReceiverStream::new(heartbeat_timeout_rx).map(Into::into),
-        );
+        let locals_stream = if let Some(hb_rx) = heartbeat_timeout_rx {
+            Either::Left(stream::select(
+                UnboundedReceiverStream::new(local_rx),
+                UnboundedReceiverStream::new(hb_rx).map(Into::into),
+            ))
+        } else {
+            Either::Right(UnboundedReceiverStream::new(local_rx))
+        };
         let (activation_tx, activation_rx) = unbounded_channel();
         let (start_polling_tx, start_polling_rx) = oneshot::channel();
         // We must spawn a task to constantly poll the activation stream, because otherwise
         // activation completions would not cause anything to happen until the next poll.
-        let tracing_sub = telem_instance.and_then(|ti| ti.trace_subscriber());
         let processing_task = thread::Builder::new()
             .name("workflow-processing".to_string())
             .spawn(move || {
@@ -201,13 +212,6 @@ impl Workflows {
                     .unwrap();
                 let local = LocalSet::new();
                 local.block_on(&rt, async move {
-                    let mut stream = WFStream::build(
-                        basics,
-                        extracted_wft_stream,
-                        locals_stream,
-                        local_activity_request_sink,
-                    );
-
                     // However, we want to avoid plowing ahead until we've been asked to poll at
                     // least once. This supports activity-only workers.
                     let do_poll = tokio::select! {
@@ -221,6 +225,14 @@ impl Workflows {
                     if !do_poll {
                         return;
                     }
+
+                    let mut stream = WFStream::build(
+                        basics,
+                        extracted_wft_stream,
+                        locals_stream,
+                        local_activity_request_sink,
+                    );
+
                     while let Some(output) = stream.next().await {
                         match output {
                             Ok(o) => {
@@ -260,6 +272,7 @@ impl Workflows {
             local_act_mgr,
             ever_polled: AtomicBool::new(false),
             default_versioning_behavior,
+            metrics,
         }
     }
 
@@ -340,18 +353,25 @@ impl Workflows {
                         force_new_wft,
                         sdk_metadata,
                         mut versioning_behavior,
+                        attempt,
                     },
             } => {
                 let reserved_act_permits =
                     self.reserve_activity_slots_for_outgoing_commands(commands.as_mut_slice());
                 debug!(commands=%commands.display(), query_responses=%query_responses.display(),
-                           messages=%messages.display(), force_new_wft,
-                           "Sending responses to server");
+                       messages=%messages.display(), force_new_wft,
+                       "Sending responses to server");
                 if let Some(default_vb) = self.default_versioning_behavior.as_ref()
                     && versioning_behavior == VersioningBehavior::Unspecified
                 {
                     versioning_behavior = *default_vb;
                 }
+                let nonfirst_local_activity_execution_attempts =
+                    if let Some(ref la_mgr) = self.local_act_mgr {
+                        la_mgr.get_nonfirst_attempt_count(run_id) as u32
+                    } else {
+                        0
+                    };
                 let mut completion = WorkflowTaskCompletion {
                     task_token: task_token.clone(),
                     commands,
@@ -362,10 +382,7 @@ impl Workflows {
                     force_create_new_workflow_task: force_new_wft,
                     sdk_metadata,
                     metering_metadata: MeteringMetadata {
-                        nonfirst_local_activity_execution_attempts: self
-                            .local_act_mgr
-                            .get_nonfirst_attempt_count(run_id)
-                            as u32,
+                        nonfirst_local_activity_execution_attempts,
                     },
                     versioning_behavior,
                 };
@@ -392,10 +409,14 @@ impl Workflows {
                                 response.activity_tasks,
                             );
                         }
-                        Err(e) if e.metadata().contains_key(MESSAGE_TOO_LARGE_KEY) => {
+                        // Reply with a task failure if we got grpc too large from server, but
+                        // not if this is a nonfirst attempt to avoid spamming.
+                        Err(e)
+                            if e.metadata().contains_key(MESSAGE_TOO_LARGE_KEY) && attempt < 2 =>
+                        {
                             let failure = Failure {
                                 failure: Some(
-                                    temporal_sdk_core_protos::temporal::api::failure::v1::Failure {
+                                    temporalio_common::protos::temporal::api::failure::v1::Failure {
                                         message: "GRPC Message too large".to_string(),
                                         failure_info: Some(FailureInfo::ApplicationFailureInfo(
                                             ApplicationFailureInfo {
@@ -416,6 +437,12 @@ impl Workflows {
                             );
                             self.handle_activation_failed(run_id, completion_time, new_outcome)
                                 .await;
+                            self.metrics
+                                .with_new_attrs([metrics::failure_reason(
+                                    FailureReason::GrpcMessageTooLarge,
+                                )])
+                                .wf_task_failed();
+                            return Err(e);
                         }
                         e => {
                             e?;
@@ -618,6 +645,12 @@ impl Workflows {
         rx
     }
 
+    /// Send a `BumpStream` message to the workflow stream. This ensures that any pending
+    /// workflow activation polls will resolve, like during shutdown, even if there are no other inputs.
+    pub(super) fn bump_stream(&self) {
+        self.send_local(LocalInputs::BumpStream);
+    }
+
     /// Query the state of workflow management. Can return `None` if workflow state is shut down.
     pub(super) fn get_state_info(&self) -> impl Future<Output = Option<WorkflowStateInfo>> {
         let rx = self.send_get_state_info_msg();
@@ -640,7 +673,7 @@ impl Workflows {
                 let mut interval = tokio::time::interval(Duration::from_millis(10));
                 loop {
                     interval.tick().await;
-                    let _ = self.get_state_info().await;
+                    self.bump_stream();
                 }
             });
             let (_, jh_res) = tokio::join!(
@@ -703,7 +736,7 @@ impl Workflows {
     fn send_local(&self, msg: impl Into<LocalInputs>) -> bool {
         let msg = msg.into();
         let print_err = match &msg {
-            LocalInputs::GetStateInfo(_) => false,
+            LocalInputs::GetStateInfo(_) | LocalInputs::BumpStream => false,
             LocalInputs::LocalResolution(lr) if lr.res.is_la_cancel_confirmation() => false,
             _ => true,
         };
@@ -991,6 +1024,7 @@ pub(crate) enum ActivationAction {
         force_new_wft: bool,
         sdk_metadata: WorkflowTaskCompletedMetadata,
         versioning_behavior: VersioningBehavior,
+        attempt: u32,
     },
     /// We should respond to a legacy query request
     RespondLegacyQuery { result: Box<QueryResult> },
@@ -1418,7 +1452,79 @@ pub(crate) enum WFMachinesError {
     Fatal(String),
 }
 
+/// Helper macro to create Nondeterminism errors with automatic assertion
+///
+/// Usage: `nondeterminism!("Activity id mismatch: {} vs {}", id1, id2)`
+macro_rules! nondeterminism {
+    ($($arg:tt)*) => {{
+        let msg = format!($($arg)*);
+        #[cfg(feature = "antithesis_assertions")]
+        $crate::antithesis::assert_unreachable!(
+            "Nondeterminism error detected",
+            ::serde_json::json!({
+                "error_message": &msg,
+                "error_type": "Nondeterminism",
+                "location": format!("{}:{}", file!(), line!())
+            })
+        );
+        WFMachinesError::Nondeterminism(msg)
+    }};
+}
+pub(crate) use nondeterminism;
+
+/// Helper macro to create Fatal errors with automatic assertion
+///
+/// Usage: `fatal!("Invalid state transition: {:?}", state)`
+macro_rules! fatal {
+    ($($arg:tt)*) => {{
+        let msg = format!($($arg)*);
+        #[cfg(feature = "antithesis_assertions")]
+        $crate::antithesis::assert_unreachable!(
+            "Fatal error detected",
+            ::serde_json::json!({
+                "error_message": &msg,
+                "error_type": "Fatal",
+                "location": format!("{}:{}", file!(), line!())
+            })
+        );
+        WFMachinesError::Fatal(msg)
+    }};
+}
+pub(crate) use fatal;
+
 impl WFMachinesError {
+    /// Create a new Nondeterminism error with assertion instrumentation
+    pub(crate) fn nondeterminism(message: impl Into<String>) -> Self {
+        let msg = message.into();
+
+        #[cfg(feature = "antithesis_assertions")]
+        crate::antithesis::assert_unreachable!(
+            "Nondeterminism error detected",
+            ::serde_json::json!({
+                "error_message": msg,
+                "error_type": "Nondeterminism"
+            })
+        );
+
+        WFMachinesError::Nondeterminism(msg)
+    }
+
+    /// Create a new Fatal error with assertion instrumentation
+    pub(crate) fn fatal(message: impl Into<String>) -> Self {
+        let msg = message.into();
+
+        #[cfg(feature = "antithesis_assertions")]
+        crate::antithesis::assert_unreachable!(
+            "Fatal error detected",
+            ::serde_json::json!({
+                "error_message": msg,
+                "error_type": "Fatal"
+            })
+        );
+
+        WFMachinesError::Fatal(msg)
+    }
+
     fn evict_reason(&self) -> EvictionReason {
         match self {
             WFMachinesError::Nondeterminism(_) => EvictionReason::Nondeterminism,
@@ -1429,7 +1535,7 @@ impl WFMachinesError {
     fn as_failure(&self) -> Failure {
         Failure {
             failure: Some(
-                temporal_sdk_core_protos::temporal::api::failure::v1::Failure::application_failure(
+                temporalio_common::protos::temporal::api::failure::v1::Failure::application_failure(
                     self.to_string(),
                     false,
                 ),
@@ -1444,7 +1550,7 @@ impl From<MachineError<WFMachinesError>> for WFMachinesError {
         match v {
             MachineError::InvalidTransition => {
                 // TODO: Get states back
-                WFMachinesError::Nondeterminism("Invalid transition in state machine".to_string())
+                WFMachinesError::nondeterminism("Invalid transition in state machine")
             }
             MachineError::Underlying(e) => e,
         }
@@ -1453,13 +1559,13 @@ impl From<MachineError<WFMachinesError>> for WFMachinesError {
 
 impl From<TimestampError> for WFMachinesError {
     fn from(_: TimestampError) -> Self {
-        Self::Fatal("Could not decode timestamp".to_string())
+        Self::fatal("Could not decode timestamp")
     }
 }
 
 impl From<anyhow::Error> for WFMachinesError {
     fn from(value: anyhow::Error) -> Self {
-        WFMachinesError::Fatal(value.to_string())
+        WFMachinesError::fatal(value.to_string())
     }
 }
 
@@ -1547,7 +1653,7 @@ fn prepare_to_ship_activation(wfa: &mut WorkflowActivation) {
 mod tests {
     use super::*;
     use itertools::Itertools;
-    use temporal_sdk_core_protos::coresdk::workflow_activation::SignalWorkflow;
+    use temporalio_common::protos::coresdk::workflow_activation::SignalWorkflow;
 
     #[test]
     fn jobs_sort() {

package/sdk-core/{core → crates/sdk-core}/src/worker/workflow/run_cache.rs

@@ -8,10 +8,12 @@ use crate::{
 };
 use lru::LruCache;
 use std::{num::NonZeroUsize, rc::Rc, sync::Arc};
-use temporal_sdk_core_api::worker::WorkerConfig;
-use temporal_sdk_core_protos::{
-    coresdk::workflow_activation::remove_from_cache::EvictionReason,
-    temporal::api::workflowservice::v1::get_system_info_response,
+use temporalio_common::{
+    protos::{
+        coresdk::workflow_activation::remove_from_cache::EvictionReason,
+        temporal::api::workflowservice::v1::get_system_info_response,
+    },
+    worker::WorkerConfig,
 };
 
 pub(super) struct RunCache {
@@ -20,7 +22,7 @@ pub(super) struct RunCache {
     server_capabilities: get_system_info_response::Capabilities,
     /// Run id -> Data
     runs: LruCache<String, ManagedRun>,
-    local_activity_request_sink: Rc<dyn LocalActivityRequestSink>,
+    local_activity_request_sink: Option<Rc<dyn LocalActivityRequestSink>>,
 
     metrics: MetricsContext,
 }
@@ -30,7 +32,7 @@ impl RunCache {
         worker_config: Arc<WorkerConfig>,
         sdk_name_and_version: (String, String),
         server_capabilities: get_system_info_response::Capabilities,
-        local_activity_request_sink: impl LocalActivityRequestSink,
+        local_activity_request_sink: Option<impl LocalActivityRequestSink>,
         metrics: MetricsContext,
     ) -> Self {
         // The cache needs room for at least one run, otherwise we couldn't do anything. In
@@ -47,7 +49,8 @@ impl RunCache {
             runs: LruCache::new(
                 NonZeroUsize::new(lru_size).expect("LRU size is guaranteed positive"),
             ),
-            local_activity_request_sink: Rc::new(local_activity_request_sink),
+            local_activity_request_sink: local_activity_request_sink
+                .map(|s| Rc::new(s) as Rc<dyn LocalActivityRequestSink>),
             metrics,
         }
     }

package/sdk-core/{core → crates/sdk-core}/src/worker/workflow/wft_extraction.rs

@@ -11,8 +11,10 @@ use crate::{
 };
 use futures_util::{FutureExt, Stream, StreamExt, stream, stream::PollNext};
 use std::{future, sync::Arc};
-use temporal_sdk_core_api::worker::WorkflowSlotKind;
-use temporal_sdk_core_protos::{TaskToken, coresdk::WorkflowSlotInfo};
+use temporalio_common::{
+    protos::{TaskToken, coresdk::WorkflowSlotInfo},
+    worker::WorkflowSlotKind,
+};
 use tracing::Span;
 
 /// Transforms incoming validated WFTs and history fetching requests into [PermittedWFT]s ready

package/sdk-core/{core → crates/sdk-core}/src/worker/workflow/wft_poller.rs

@@ -6,13 +6,20 @@ use crate::{
     telemetry::metrics::{workflow_poller, workflow_sticky_poller},
     worker::{client::WorkerClient, wft_poller_behavior},
 };
+use crossbeam_utils::atomic::AtomicCell;
 use futures_util::{Stream, stream};
-use std::sync::{Arc, OnceLock};
-use temporal_sdk_core_api::worker::{WorkerConfig, WorkflowSlotKind};
-use temporal_sdk_core_protos::temporal::api::workflowservice::v1::PollWorkflowTaskQueueResponse;
+use std::{
+    sync::{Arc, OnceLock},
+    time::SystemTime,
+};
+use temporalio_common::{
+    protos::temporal::api::workflowservice::v1::PollWorkflowTaskQueueResponse,
+    worker::{WorkerConfig, WorkflowSlotKind},
+};
 use tokio::sync::watch;
 use tokio_util::sync::CancellationToken;
 
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn make_wft_poller(
     config: &WorkerConfig,
     sticky_queue_name: &Option<String>,
@@ -20,6 +27,8 @@ pub(crate) fn make_wft_poller(
     metrics: &MetricsContext,
     shutdown_token: &CancellationToken,
     wft_slots: &MeteredPermitDealer<WorkflowSlotKind>,
+    last_successful_poll_time: Arc<AtomicCell<Option<SystemTime>>>,
+    sticky_last_successful_poll_time: Arc<AtomicCell<Option<SystemTime>>>,
 ) -> impl Stream<
     Item = Result<
         (
@@ -52,6 +61,7 @@ pub(crate) fn make_wft_poller(
         WorkflowTaskOptions {
             wft_poller_shared: wft_poller_shared.clone(),
         },
+        last_successful_poll_time,
     );
     let sticky_queue_poller = sticky_queue_name.as_ref().map(|sqn| {
         let sticky_metrics = metrics.with_new_attrs([workflow_sticky_poller()]);
@@ -66,6 +76,7 @@ pub(crate) fn make_wft_poller(
                 sticky_metrics.record_num_pollers(np);
             }),
             WorkflowTaskOptions { wft_poller_shared },
+            sticky_last_successful_poll_time,
         )
     });
     let wf_task_poll_buffer = Box::new(WorkflowTaskPoller::new(
@@ -251,7 +262,7 @@ mod tests {
     };
     use futures_util::{StreamExt, pin_mut};
     use std::sync::Arc;
-    use temporal_sdk_core_api::worker::WorkflowSlotKind;
+    use temporalio_common::worker::WorkflowSlotKind;
 
     #[tokio::test]
     async fn poll_timeouts_do_not_produce_responses() {

package/sdk-core/{core → crates/sdk-core}/src/worker/workflow/workflow_stream.rs

@@ -10,8 +10,9 @@ use crate::{
 };
 use futures_util::{Stream, StreamExt, stream, stream::PollNext};
 use std::{collections::VecDeque, fmt::Debug, future, sync::Arc};
-use temporal_sdk_core_api::errors::PollError;
-use temporal_sdk_core_protos::coresdk::workflow_activation::remove_from_cache::EvictionReason;
+use temporalio_common::{
+    errors::PollError, protos::coresdk::workflow_activation::remove_from_cache::EvictionReason,
+};
 use tokio_util::sync::CancellationToken;
 use tracing::{Level, Span};
 
@@ -63,7 +64,7 @@ impl WFStream {
         basics: WorkflowBasics,
         wft_stream: impl Stream<Item = Result<WFTExtractorOutput, tonic::Status>> + Send + 'static,
         local_rx: impl Stream<Item = LocalInput> + Send + 'static,
-        local_activity_request_sink: impl LocalActivityRequestSink,
+        local_activity_request_sink: Option<impl LocalActivityRequestSink>,
     ) -> impl Stream<Item = Result<WFStreamOutput, PollError>> {
         let all_inputs = stream::select_with_strategy(
             local_rx.map(Into::into),
@@ -81,7 +82,7 @@ impl WFStream {
     fn build_internal(
         all_inputs: impl Stream<Item = WFStreamInput>,
         basics: WorkflowBasics,
-        local_activity_request_sink: impl LocalActivityRequestSink,
+        local_activity_request_sink: Option<impl LocalActivityRequestSink>,
     ) -> impl Stream<Item = Result<WFStreamOutput, PollError>> {
         let mut state = WFStream {
             buffered_polls_need_cache_slot: Default::default(),
@@ -146,6 +147,11 @@ impl WFStream {
                                 });
                                 None
                             }
+                            LocalInputs::BumpStream => {
+                                // Ensures pending workflow activation polls resolve for
+                                // scenarios like during shutdown
+                                None
+                            }
                         }
                     }
                     WFStreamInput::FailedFetch {
@@ -287,17 +293,24 @@ impl WFStream {
                     failure,
                     is_autocomplete,
                     ..
-                } => rh.failed_completion(
-                    failure.force_cause(),
-                    if is_autocomplete {
-                        EvictionReason::Unspecified
-                    } else {
-                        EvictionReason::LangFail
-                    },
-                    failure,
-                    is_autocomplete,
-                    complete.response_tx,
-                ),
+                } => {
+                    let forced_cause = failure.force_cause();
+                    rh.failed_completion(
+                        if forced_cause == WorkflowTaskFailedCause::Unspecified {
+                            WorkflowTaskFailedCause::WorkflowWorkerUnhandledFailure
+                        } else {
+                            forced_cause
+                        },
+                        if is_autocomplete {
+                            EvictionReason::Unspecified
+                        } else {
+                            EvictionReason::LangFail
+                        },
+                        failure,
+                        is_autocomplete,
+                        complete.response_tx,
+                    )
+                }
             },
             NewOrFetchedComplete::Fetched(update, paginator) => {
                 rh.fetched_page_completion(update, *paginator)
@@ -625,6 +638,7 @@ pub(super) enum LocalInputs {
     RequestEviction(RequestEvictMsg),
     HeartbeatTimeout(String),
     GetStateInfo(GetStateInfoMsg),
+    BumpStream,
 }
 impl LocalInputs {
     fn run_id(&self) -> Option<&str> {
@@ -635,7 +649,7 @@ impl LocalInputs {
             LocalInputs::PostActivation(pa) => &pa.run_id,
             LocalInputs::RequestEviction(re) => &re.run_id,
             LocalInputs::HeartbeatTimeout(hb) => hb,
-            LocalInputs::GetStateInfo(_) => return None,
+            LocalInputs::GetStateInfo(_) | LocalInputs::BumpStream => return None,
         })
     }
 }

package/sdk-core/{tests → crates/sdk-core/tests}/c_bridge_smoke_test.c

@@ -7,4 +7,4 @@ int main(void) {
     temporal_core_cancellation_token_free(tok);
     printf("C bridge smoke test passed!\n");
     return 0;
-}
+}

package/sdk-core/{tests → crates/sdk-core/tests}/cloud_tests.rs

@@ -4,7 +4,7 @@ mod common;
 mod shared_tests;
 
 use common::get_cloud_client;
-use temporal_client::WorkflowClientTrait;
+use temporalio_client::WorkflowClientTrait;
 
 #[tokio::test]
 async fn tls_test() {