@temporalio/core-bridge 1.13.1 → 1.14.0

package/sdk-core/{core → crates/sdk-core}/src/worker/workflow/managed_run.rs

@@ -28,22 +28,25 @@ use std::{
     sync::{Arc, mpsc::Sender},
     time::{Duration, Instant},
 };
-use temporal_sdk_core_api::{errors::WorkflowErrorType, worker::WorkerConfig};
-use temporal_sdk_core_protos::{
-    TaskToken,
-    coresdk::{
-        workflow_activation::{
-            WorkflowActivation, create_evict_activation, query_to_job,
-            remove_from_cache::EvictionReason, workflow_activation_job,
+use temporalio_common::{
+    errors::WorkflowErrorType,
+    protos::{
+        TaskToken,
+        coresdk::{
+            workflow_activation::{
+                WorkflowActivation, create_evict_activation, query_to_job,
+                remove_from_cache::EvictionReason, workflow_activation_job,
+            },
+            workflow_commands::{FailWorkflowExecution, QueryResult},
+            workflow_completion,
+        },
+        temporal::api::{
+            command::v1::command::Attributes as CmdAttribs,
+            enums::v1::{VersioningBehavior, WorkflowTaskFailedCause},
+            failure::v1::Failure,
         },
-        workflow_commands::{FailWorkflowExecution, QueryResult},
-        workflow_completion,
-    },
-    temporal::api::{
-        command::v1::command::Attributes as CmdAttribs,
-        enums::v1::{VersioningBehavior, WorkflowTaskFailedCause},
-        failure::v1::Failure,
     },
+    worker::WorkerConfig,
 };
 use tokio::sync::oneshot;
 use tracing::Span;
@@ -70,7 +73,9 @@ pub(super) struct ManagedRun {
     /// pushing things out and then directly back in. The downside is this is the only "impure" part
     /// of the in/out nature of workflow state management. If there's ever a sensible way to lift it
     /// up, that'd be nice.
-    local_activity_request_sink: Rc<dyn LocalActivityRequestSink>,
+    ///
+    /// This field is `None` when `WorkerTaskTypes.enable_local_activities` is false.
+    local_activity_request_sink: Option<Rc<dyn LocalActivityRequestSink>>,
     /// Set if the run is currently waiting on the execution of some local activities.
     waiting_on_la: Option<WaitingOnLAs>,
     /// Is set to true if the machines encounter an error and the only subsequent thing we should
@@ -105,7 +110,7 @@ impl ManagedRun {
     pub(super) fn new(
         basics: RunBasics,
         wft: PermittedWFT,
-        local_activity_request_sink: Rc<dyn LocalActivityRequestSink>,
+        local_activity_request_sink: Option<Rc<dyn LocalActivityRequestSink>>,
     ) -> (Self, RunUpdateAct) {
         let metrics = basics.metrics.clone();
         let config = basics.worker_config.clone();
@@ -240,10 +245,10 @@ impl ManagedRun {
             let r = self.wfm.get_next_activation()?;
             if r.jobs.is_empty() {
                 return Err(RunUpdateErr {
-                    source: WFMachinesError::Fatal(format!(
+                    source: crate::worker::workflow::fatal!(
                         "Machines created for {} with no jobs",
                         self.wfm.machines.run_id
-                    )),
+                    ),
                     complete_resp_chan: None,
                 });
             }
@@ -261,7 +266,7 @@ impl ManagedRun {
                     lawait.hb_timeout_handle.abort();
                     lawait.hb_timeout_handle = sink_heartbeat_timeout_start(
                         self.wfm.machines.run_id.clone(),
-                        self.local_activity_request_sink.as_ref(),
+                        self.local_activity_request_sink.as_deref(),
                         start_time,
                         lawait.wft_timeout,
                     );
@@ -307,9 +312,13 @@ impl ManagedRun {
                 self.wfm.machines.reset_last_started_id(id);
             }
             // Tell the LA manager that we're done with the WFT
-            self.local_activity_request_sink.sink_reqs(vec![
-                LocalActRequest::IndicateWorkflowTaskCompleted(self.wfm.machines.run_id.clone()),
-            ]);
+            if let Some(ref local_act_request_sink) = self.local_activity_request_sink {
+                local_act_request_sink.sink_reqs(vec![
+                    LocalActRequest::IndicateWorkflowTaskCompleted(
+                        self.wfm.machines.run_id.clone(),
+                    ),
+                ]);
+            }
         }
 
         retme
@@ -402,7 +411,7 @@ impl ManagedRun {
             }) {
                 ActivationCompleteOutcome::ReportWFTFail(FailedActivationWFTReport::Report(
                     tt,
-                    WorkflowTaskFailedCause::Unspecified,
+                    WorkflowTaskFailedCause::WorkflowWorkerUnhandledFailure,
                     Failure::application_failure(reason, true).into(),
                 ))
             } else {
@@ -759,7 +768,7 @@ impl ManagedRun {
                     completion_dat: Some((data, completion.resp_chan)),
                     hb_timeout_handle: sink_heartbeat_timeout_start(
                         self.run_id().to_string(),
-                        self.local_activity_request_sink.as_ref(),
+                        self.local_activity_request_sink.as_deref(),
                         start_t,
                         wft_timeout,
                     ),
@@ -885,7 +894,7 @@ impl ManagedRun {
             // We just checked it is some, unwrap OK.
             let c = self.completion_waiting_on_page_fetch.take().unwrap();
             let run_upd = self.failed_completion(
-                WorkflowTaskFailedCause::Unspecified,
+                WorkflowTaskFailedCause::WorkflowWorkerUnhandledFailure,
                 info.reason,
                 Failure::application_failure(info.message, false).into(),
                 true,
@@ -1019,7 +1028,7 @@ impl ManagedRun {
                     let fail_cause = if matches!(&fail.source, WFMachinesError::Nondeterminism(_)) {
                         WorkflowTaskFailedCause::NonDeterministicError
                     } else {
-                        WorkflowTaskFailedCause::Unspecified
+                        WorkflowTaskFailedCause::WorkflowWorkerUnhandledFailure
                     };
                     self.failed_completion(
                         fail_cause,
@@ -1139,6 +1148,7 @@ impl ManagedRun {
                 }
             }
 
+            let attempt = self.wft.as_ref().map(|t| t.info.attempt).unwrap_or(1);
             ActivationCompleteOutcome::ReportWFTSuccess(ServerCommandsWithWorkflowInfo {
                 task_token: data.task_token,
                 action: ActivationAction::WftComplete {
@@ -1148,6 +1158,7 @@ impl ManagedRun {
                     query_responses,
                     sdk_metadata: machines_wft_response.metadata_for_complete(),
                     versioning_behavior: data.versioning_behavior,
+                    attempt,
                 },
             })
         } else {
@@ -1168,7 +1179,12 @@ impl ManagedRun {
         &mut self,
         new_local_acts: Vec<LocalActRequest>,
     ) -> Result<(), WFMachinesError> {
-        let immediate_resolutions = self.local_activity_request_sink.sink_reqs(new_local_acts);
+        let immediate_resolutions =
+            if let Some(ref local_act_request_sink) = self.local_activity_request_sink {
+                local_act_request_sink.sink_reqs(new_local_acts)
+            } else {
+                Vec::new()
+            };
         for resolution in immediate_resolutions {
             self.wfm
                 .notify_of_local_result(LocalResolution::LocalActivity(resolution))?;
@@ -1306,21 +1322,23 @@ fn put_queries_in_act(act: &mut WorkflowActivation, wft: &mut OutstandingTask) {
 }
 fn sink_heartbeat_timeout_start(
     run_id: String,
-    sink: &dyn LocalActivityRequestSink,
+    sink: Option<&dyn LocalActivityRequestSink>,
     wft_start_time: Instant,
     wft_timeout: Duration,
 ) -> AbortHandle {
     // The heartbeat deadline is 80% of the WFT timeout
     let deadline = wft_start_time.add(wft_timeout.mul_f32(WFT_HEARTBEAT_TIMEOUT_FRACTION));
     let (abort_handle, abort_reg) = AbortHandle::new_pair();
-    sink.sink_reqs(vec![LocalActRequest::StartHeartbeatTimeout {
-        send_on_elapse: HeartbeatTimeoutMsg {
-            run_id,
-            span: Span::current(),
-        },
-        deadline,
-        abort_reg,
-    }]);
+    if let Some(la_sink) = sink {
+        la_sink.sink_reqs(vec![LocalActRequest::StartHeartbeatTimeout {
+            send_on_elapse: HeartbeatTimeoutMsg {
+                run_id,
+                span: Span::current(),
+            },
+            deadline,
+            abort_reg,
+        }]);
+    }
     abort_handle
 }
 
@@ -1606,7 +1624,7 @@ mod tests {
     }
 
     mod command_utils {
-        use temporal_sdk_core_protos::coresdk::workflow_commands::{
+        use temporalio_common::protos::coresdk::workflow_commands::{
             CancelWorkflowExecution, CompleteWorkflowExecution, QueryResult, UpdateResponse,
         };
 

package/sdk-core/{core → crates/sdk-core}/src/worker/workflow/mod.rs

@@ -23,7 +23,11 @@ use crate::{
     internal_flags::InternalFlags,
     pollers::TrackedPermittedTqResp,
     protosext::{ValidPollWFTQResponse, protocol_messages::IncomingProtocolMessage},
-    telemetry::{TelemetryInstance, VecDisplayer, set_trace_subscriber_for_current_thread},
+    telemetry::{
+        VecDisplayer,
+        metrics::{self, FailureReason},
+        set_trace_subscriber_for_current_thread,
+    },
     worker::{
         LocalActRequest, LocalActivityExecutionResult, LocalActivityResolution,
         PostActivateHookData,
@@ -31,6 +35,7 @@ use crate::{
         client::{LegacyQueryResult, WorkerClient, WorkflowTaskCompletion},
         workflow::{
             history_update::HistoryPaginator,
+            machines::MachineError,
             managed_run::RunUpdateAct,
             wft_extraction::{HistoryFetchReq, WFTExtractor, WFTStreamIn},
             wft_poller::validate_wft,
@@ -42,7 +47,6 @@ use anyhow::anyhow;
 use futures_util::{Stream, StreamExt, future::abortable, stream, stream::BoxStream};
 use itertools::Itertools;
 use prost_types::TimestampError;
-use rustfsm::MachineError;
 use std::{
     cell::RefCell,
     collections::VecDeque,
@@ -55,35 +59,36 @@ use std::{
     thread,
     time::{Duration, Instant},
 };
-use temporal_client::MESSAGE_TOO_LARGE_KEY;
-use temporal_sdk_core_api::{
+use temporalio_client::MESSAGE_TOO_LARGE_KEY;
+use temporalio_common::{
     errors::{CompleteWfError, PollError},
-    worker::{ActivitySlotKind, WorkerConfig, WorkflowSlotKind},
-};
-use temporal_sdk_core_protos::{
-    TaskToken,
-    coresdk::{
-        workflow_activation::{
-            QueryWorkflow, WorkflowActivation, WorkflowActivationJob,
-            remove_from_cache::EvictionReason, workflow_activation_job,
+    protos::{
+        TaskToken,
+        coresdk::{
+            workflow_activation::{
+                QueryWorkflow, WorkflowActivation, WorkflowActivationJob,
+                remove_from_cache::EvictionReason, workflow_activation_job,
+            },
+            workflow_commands::*,
+            workflow_completion::{
+                self, Failure, WorkflowActivationCompletion, workflow_activation_completion,
+            },
         },
-        workflow_commands::*,
-        workflow_completion,
-        workflow_completion::{
-            Failure, WorkflowActivationCompletion, workflow_activation_completion,
+        temporal::api::{
+            command::v1::{Command as ProtoCommand, Command, command::Attributes},
+            common::v1::{
+                Memo, MeteringMetadata, RetryPolicy, SearchAttributes, WorkflowExecution,
+            },
+            enums::v1::{VersioningBehavior, WorkflowTaskFailedCause},
+            failure::v1::{ApplicationFailureInfo, failure::FailureInfo},
+            protocol::v1::Message as ProtocolMessage,
+            query::v1::WorkflowQuery,
+            sdk::v1::{UserMetadata, WorkflowTaskCompletedMetadata},
+            taskqueue::v1::StickyExecutionAttributes,
+            workflowservice::v1::{PollActivityTaskQueueResponse, get_system_info_response},
         },
     },
-    temporal::api::{
-        command::v1::{Command as ProtoCommand, Command, command::Attributes},
-        common::v1::{Memo, MeteringMetadata, RetryPolicy, SearchAttributes, WorkflowExecution},
-        enums::v1::{VersioningBehavior, WorkflowTaskFailedCause},
-        failure::v1::{ApplicationFailureInfo, failure::FailureInfo},
-        protocol::v1::Message as ProtocolMessage,
-        query::v1::WorkflowQuery,
-        sdk::v1::{UserMetadata, WorkflowTaskCompletedMetadata},
-        taskqueue::v1::StickyExecutionAttributes,
-        workflowservice::v1::{PollActivityTaskQueueResponse, get_system_info_response},
-    },
+    worker::{ActivitySlotKind, WorkerConfig, WorkflowSlotKind},
 };
 use tokio::{
     sync::{
@@ -93,10 +98,15 @@ use tokio::{
     task::{LocalSet, spawn_blocking},
 };
 use tokio_stream::wrappers::UnboundedReceiverStream;
+use tokio_util::either::Either;
 use tokio_util::sync::CancellationToken;
-use tracing::Span;
+use tracing::{Span, Subscriber};
 
-pub(crate) const LEGACY_QUERY_ID: &str = "legacy_query";
+/// Id used by server for "legacy" queries. IE: Queries that come in the `query` rather than
+/// `queries` field of a WFT, and are responded to on the separate `respond_query_task_completed`
+/// rpc.
+#[allow(unreachable_pub)] // re-exported in supermodule
+pub const LEGACY_QUERY_ID: &str = "legacy_query";
 /// What percentage of a WFT timeout we are willing to wait before sending a WFT heartbeat when
 /// necessary.
 const WFT_HEARTBEAT_TIMEOUT_FRACTION: f32 = 0.8;
@@ -124,9 +134,10 @@ pub(crate) struct Workflows {
     activity_tasks_handle: Option<ActivitiesFromWFTsHandle>,
     /// Ensures we stay at or below this worker's maximum concurrent workflow task limit
     wft_semaphore: MeteredPermitDealer<WorkflowSlotKind>,
-    local_act_mgr: Arc<LocalActivityManager>,
+    local_act_mgr: Option<Arc<LocalActivityManager>>,
     ever_polled: AtomicBool,
     default_versioning_behavior: Option<VersioningBehavior>,
+    metrics: MetricsContext,
 }
 
 pub(crate) struct WorkflowBasics {
@@ -159,16 +170,17 @@ impl Workflows {
         client: Arc<dyn WorkerClient>,
         wft_semaphore: MeteredPermitDealer<WorkflowSlotKind>,
         wft_stream: impl Stream<Item = WFTStreamIn> + Send + 'static,
-        local_activity_request_sink: impl LocalActivityRequestSink,
-        local_act_mgr: Arc<LocalActivityManager>,
-        heartbeat_timeout_rx: UnboundedReceiver<HeartbeatTimeoutMsg>,
+        local_activity_request_sink: Option<impl LocalActivityRequestSink>,
+        local_act_mgr: Option<Arc<LocalActivityManager>>,
+        heartbeat_timeout_rx: Option<UnboundedReceiver<HeartbeatTimeoutMsg>>,
         activity_tasks_handle: Option<ActivitiesFromWFTsHandle>,
-        telem_instance: Option<&TelemetryInstance>,
+        tracing_sub: Option<Arc<dyn Subscriber + Send + Sync>>,
     ) -> Self {
         let (local_tx, local_rx) = unbounded_channel();
         let (fetch_tx, fetch_rx) = unbounded_channel();
         let shutdown_tok = basics.shutdown_token.clone();
         let task_queue = basics.worker_config.task_queue.clone();
+        let metrics = basics.metrics.clone();
         let default_versioning_behavior = basics.default_versioning_behavior;
         let extracted_wft_stream = WFTExtractor::build(
             client.clone(),
@@ -176,15 +188,18 @@ impl Workflows {
             wft_stream,
             UnboundedReceiverStream::new(fetch_rx),
         );
-        let locals_stream = stream::select(
-            UnboundedReceiverStream::new(local_rx),
-            UnboundedReceiverStream::new(heartbeat_timeout_rx).map(Into::into),
-        );
+        let locals_stream = if let Some(hb_rx) = heartbeat_timeout_rx {
+            Either::Left(stream::select(
+                UnboundedReceiverStream::new(local_rx),
+                UnboundedReceiverStream::new(hb_rx).map(Into::into),
+            ))
+        } else {
+            Either::Right(UnboundedReceiverStream::new(local_rx))
+        };
         let (activation_tx, activation_rx) = unbounded_channel();
         let (start_polling_tx, start_polling_rx) = oneshot::channel();
         // We must spawn a task to constantly poll the activation stream, because otherwise
         // activation completions would not cause anything to happen until the next poll.
-        let tracing_sub = telem_instance.and_then(|ti| ti.trace_subscriber());
         let processing_task = thread::Builder::new()
             .name("workflow-processing".to_string())
             .spawn(move || {
@@ -197,13 +212,6 @@ impl Workflows {
                     .unwrap();
                 let local = LocalSet::new();
                 local.block_on(&rt, async move {
-                    let mut stream = WFStream::build(
-                        basics,
-                        extracted_wft_stream,
-                        locals_stream,
-                        local_activity_request_sink,
-                    );
-
                     // However, we want to avoid plowing ahead until we've been asked to poll at
                     // least once. This supports activity-only workers.
                     let do_poll = tokio::select! {
@@ -217,6 +225,14 @@ impl Workflows {
                     if !do_poll {
                         return;
                     }
+
+                    let mut stream = WFStream::build(
+                        basics,
+                        extracted_wft_stream,
+                        locals_stream,
+                        local_activity_request_sink,
+                    );
+
                     while let Some(output) = stream.next().await {
                         match output {
                             Ok(o) => {
@@ -256,6 +272,7 @@ impl Workflows {
             local_act_mgr,
             ever_polled: AtomicBool::new(false),
             default_versioning_behavior,
+            metrics,
         }
     }
 
@@ -336,18 +353,25 @@ impl Workflows {
                         force_new_wft,
                         sdk_metadata,
                         mut versioning_behavior,
+                        attempt,
                     },
             } => {
                 let reserved_act_permits =
                     self.reserve_activity_slots_for_outgoing_commands(commands.as_mut_slice());
                 debug!(commands=%commands.display(), query_responses=%query_responses.display(),
-                           messages=%messages.display(), force_new_wft,
-                           "Sending responses to server");
+                       messages=%messages.display(), force_new_wft,
+                       "Sending responses to server");
                 if let Some(default_vb) = self.default_versioning_behavior.as_ref()
                     && versioning_behavior == VersioningBehavior::Unspecified
                 {
                     versioning_behavior = *default_vb;
                 }
+                let nonfirst_local_activity_execution_attempts =
+                    if let Some(ref la_mgr) = self.local_act_mgr {
+                        la_mgr.get_nonfirst_attempt_count(run_id) as u32
+                    } else {
+                        0
+                    };
                 let mut completion = WorkflowTaskCompletion {
                     task_token: task_token.clone(),
                     commands,
@@ -358,10 +382,7 @@ impl Workflows {
                     force_create_new_workflow_task: force_new_wft,
                     sdk_metadata,
                     metering_metadata: MeteringMetadata {
-                        nonfirst_local_activity_execution_attempts: self
-                            .local_act_mgr
-                            .get_nonfirst_attempt_count(run_id)
-                            as u32,
+                        nonfirst_local_activity_execution_attempts,
                     },
                     versioning_behavior,
                 };
@@ -388,10 +409,14 @@ impl Workflows {
                                 response.activity_tasks,
                             );
                         }
-                        Err(e) if e.metadata().contains_key(MESSAGE_TOO_LARGE_KEY) => {
+                        // Reply with a task failure if we got grpc too large from server, but
+                        // not if this is a nonfirst attempt to avoid spamming.
+                        Err(e)
+                            if e.metadata().contains_key(MESSAGE_TOO_LARGE_KEY) && attempt < 2 =>
+                        {
                             let failure = Failure {
                                 failure: Some(
-                                    temporal_sdk_core_protos::temporal::api::failure::v1::Failure {
+                                    temporalio_common::protos::temporal::api::failure::v1::Failure {
                                         message: "GRPC Message too large".to_string(),
                                         failure_info: Some(FailureInfo::ApplicationFailureInfo(
                                             ApplicationFailureInfo {
@@ -412,6 +437,12 @@ impl Workflows {
                             );
                             self.handle_activation_failed(run_id, completion_time, new_outcome)
                                 .await;
+                            self.metrics
+                                .with_new_attrs([metrics::failure_reason(
+                                    FailureReason::GrpcMessageTooLarge,
+                                )])
+                                .wf_task_failed();
+                            return Err(e);
                         }
                         e => {
                             e?;
@@ -614,6 +645,12 @@ impl Workflows {
         rx
     }
 
+    /// Send a `BumpStream` message to the workflow stream. This ensures that any pending
+    /// workflow activation polls will resolve, like during shutdown, even if there are no other inputs.
+    pub(super) fn bump_stream(&self) {
+        self.send_local(LocalInputs::BumpStream);
+    }
+
     /// Query the state of workflow management. Can return `None` if workflow state is shut down.
     pub(super) fn get_state_info(&self) -> impl Future<Output = Option<WorkflowStateInfo>> {
         let rx = self.send_get_state_info_msg();
@@ -636,7 +673,7 @@ impl Workflows {
                 let mut interval = tokio::time::interval(Duration::from_millis(10));
                 loop {
                     interval.tick().await;
-                    let _ = self.get_state_info().await;
+                    self.bump_stream();
                 }
             });
             let (_, jh_res) = tokio::join!(
@@ -699,7 +736,7 @@ impl Workflows {
     fn send_local(&self, msg: impl Into<LocalInputs>) -> bool {
         let msg = msg.into();
         let print_err = match &msg {
-            LocalInputs::GetStateInfo(_) => false,
+            LocalInputs::GetStateInfo(_) | LocalInputs::BumpStream => false,
             LocalInputs::LocalResolution(lr) if lr.res.is_la_cancel_confirmation() => false,
             _ => true,
         };
@@ -987,6 +1024,7 @@ pub(crate) enum ActivationAction {
         force_new_wft: bool,
         sdk_metadata: WorkflowTaskCompletedMetadata,
         versioning_behavior: VersioningBehavior,
+        attempt: u32,
     },
     /// We should respond to a legacy query request
     RespondLegacyQuery { result: Box<QueryResult> },
@@ -1414,7 +1452,79 @@ pub(crate) enum WFMachinesError {
     Fatal(String),
 }
 
+/// Helper macro to create Nondeterminism errors with automatic assertion
+///
+/// Usage: `nondeterminism!("Activity id mismatch: {} vs {}", id1, id2)`
+macro_rules! nondeterminism {
+    ($($arg:tt)*) => {{
+        let msg = format!($($arg)*);
+        #[cfg(feature = "antithesis_assertions")]
+        $crate::antithesis::assert_unreachable!(
+            "Nondeterminism error detected",
+            ::serde_json::json!({
+                "error_message": &msg,
+                "error_type": "Nondeterminism",
+                "location": format!("{}:{}", file!(), line!())
+            })
+        );
+        WFMachinesError::Nondeterminism(msg)
+    }};
+}
+pub(crate) use nondeterminism;
+
+/// Helper macro to create Fatal errors with automatic assertion
+///
+/// Usage: `fatal!("Invalid state transition: {:?}", state)`
+macro_rules! fatal {
+    ($($arg:tt)*) => {{
+        let msg = format!($($arg)*);
+        #[cfg(feature = "antithesis_assertions")]
+        $crate::antithesis::assert_unreachable!(
+            "Fatal error detected",
+            ::serde_json::json!({
+                "error_message": &msg,
+                "error_type": "Fatal",
+                "location": format!("{}:{}", file!(), line!())
+            })
+        );
+        WFMachinesError::Fatal(msg)
+    }};
+}
+pub(crate) use fatal;
+
 impl WFMachinesError {
+    /// Create a new Nondeterminism error with assertion instrumentation
+    pub(crate) fn nondeterminism(message: impl Into<String>) -> Self {
+        let msg = message.into();
+
+        #[cfg(feature = "antithesis_assertions")]
+        crate::antithesis::assert_unreachable!(
+            "Nondeterminism error detected",
+            ::serde_json::json!({
+                "error_message": msg,
+                "error_type": "Nondeterminism"
+            })
+        );
+
+        WFMachinesError::Nondeterminism(msg)
+    }
+
+    /// Create a new Fatal error with assertion instrumentation
+    pub(crate) fn fatal(message: impl Into<String>) -> Self {
+        let msg = message.into();
+
+        #[cfg(feature = "antithesis_assertions")]
+        crate::antithesis::assert_unreachable!(
+            "Fatal error detected",
+            ::serde_json::json!({
+                "error_message": msg,
+                "error_type": "Fatal"
+            })
+        );
+
+        WFMachinesError::Fatal(msg)
+    }
+
     fn evict_reason(&self) -> EvictionReason {
         match self {
             WFMachinesError::Nondeterminism(_) => EvictionReason::Nondeterminism,
@@ -1425,7 +1535,7 @@ impl WFMachinesError {
     fn as_failure(&self) -> Failure {
         Failure {
             failure: Some(
-                temporal_sdk_core_protos::temporal::api::failure::v1::Failure::application_failure(
+                temporalio_common::protos::temporal::api::failure::v1::Failure::application_failure(
                     self.to_string(),
                     false,
                 ),
@@ -1440,7 +1550,7 @@ impl From<MachineError<WFMachinesError>> for WFMachinesError {
         match v {
             MachineError::InvalidTransition => {
                 // TODO: Get states back
-                WFMachinesError::Nondeterminism("Invalid transition in state machine".to_string())
+                WFMachinesError::nondeterminism("Invalid transition in state machine")
             }
             MachineError::Underlying(e) => e,
         }
@@ -1449,13 +1559,13 @@ impl From<MachineError<WFMachinesError>> for WFMachinesError {
 
 impl From<TimestampError> for WFMachinesError {
     fn from(_: TimestampError) -> Self {
-        Self::Fatal("Could not decode timestamp".to_string())
+        Self::fatal("Could not decode timestamp")
     }
 }
 
 impl From<anyhow::Error> for WFMachinesError {
     fn from(value: anyhow::Error) -> Self {
-        WFMachinesError::Fatal(value.to_string())
+        WFMachinesError::fatal(value.to_string())
     }
 }
 
@@ -1543,7 +1653,7 @@ fn prepare_to_ship_activation(wfa: &mut WorkflowActivation) {
 mod tests {
     use super::*;
     use itertools::Itertools;
-    use temporal_sdk_core_protos::coresdk::workflow_activation::SignalWorkflow;
+    use temporalio_common::protos::coresdk::workflow_activation::SignalWorkflow;
 
     #[test]
     fn jobs_sort() {

package/sdk-core/{core → crates/sdk-core}/src/worker/workflow/run_cache.rs

@@ -8,10 +8,12 @@ use crate::{
 };
 use lru::LruCache;
 use std::{num::NonZeroUsize, rc::Rc, sync::Arc};
-use temporal_sdk_core_api::worker::WorkerConfig;
-use temporal_sdk_core_protos::{
-    coresdk::workflow_activation::remove_from_cache::EvictionReason,
-    temporal::api::workflowservice::v1::get_system_info_response,
+use temporalio_common::{
+    protos::{
+        coresdk::workflow_activation::remove_from_cache::EvictionReason,
+        temporal::api::workflowservice::v1::get_system_info_response,
+    },
+    worker::WorkerConfig,
 };
 
 pub(super) struct RunCache {
@@ -20,7 +22,7 @@ pub(super) struct RunCache {
     server_capabilities: get_system_info_response::Capabilities,
     /// Run id -> Data
     runs: LruCache<String, ManagedRun>,
-    local_activity_request_sink: Rc<dyn LocalActivityRequestSink>,
+    local_activity_request_sink: Option<Rc<dyn LocalActivityRequestSink>>,
 
     metrics: MetricsContext,
 }
@@ -30,7 +32,7 @@ impl RunCache {
         worker_config: Arc<WorkerConfig>,
         sdk_name_and_version: (String, String),
         server_capabilities: get_system_info_response::Capabilities,
-        local_activity_request_sink: impl LocalActivityRequestSink,
+        local_activity_request_sink: Option<impl LocalActivityRequestSink>,
         metrics: MetricsContext,
     ) -> Self {
         // The cache needs room for at least one run, otherwise we couldn't do anything. In
@@ -47,7 +49,8 @@ impl RunCache {
             runs: LruCache::new(
                 NonZeroUsize::new(lru_size).expect("LRU size is guaranteed positive"),
             ),
-            local_activity_request_sink: Rc::new(local_activity_request_sink),
+            local_activity_request_sink: local_activity_request_sink
+                .map(|s| Rc::new(s) as Rc<dyn LocalActivityRequestSink>),
             metrics,
         }
     }