@temporalio/core-bridge 1.6.0 → 1.7.1

package/sdk-core/core/src/worker/workflow/mod.rs

@@ -22,11 +22,18 @@ pub(crate) use history_update::HistoryUpdate;
 pub(crate) use managed_run::ManagedWFFunc;
 
 use crate::{
-    abstractions::{stream_when_allowed, MeteredSemaphore, OwnedMeteredSemPermit},
+    abstractions::{
+        dbg_panic, stream_when_allowed, take_cell::TakeCell, MeteredSemaphore,
+        TrackedOwnedMeteredSemPermit, UsedMeteredSemPermit,
+    },
+    internal_flags::InternalFlags,
     protosext::{legacy_query_failure, ValidPollWFTQResponse},
-    telemetry::{metrics::workflow_worker_type, VecDisplayer},
+    telemetry::{
+        metrics::workflow_worker_type, set_trace_subscriber_for_current_thread, TelemetryInstance,
+        VecDisplayer,
+    },
     worker::{
-        activities::{ActivitiesFromWFTsHandle, LocalActivityManager, PermittedTqResp},
+        activities::{ActivitiesFromWFTsHandle, LocalActivityManager, TrackedPermittedTqResp},
         client::{WorkerClient, WorkflowTaskCompletion},
         workflow::{
             history_update::HistoryPaginator,
@@ -36,19 +43,26 @@ use crate::{
             workflow_stream::{LocalInput, LocalInputs, WFStream},
         },
         LocalActRequest, LocalActivityExecutionResult, LocalActivityResolution,
+        PostActivateHookData,
     },
     MetricsContext,
 };
+use anyhow::anyhow;
 use futures::{stream::BoxStream, Stream, StreamExt};
-use futures_util::stream;
+use futures_util::{future::abortable, stream};
 use prost_types::TimestampError;
 use std::{
+    cell::RefCell,
+    cmp::Ordering,
     collections::VecDeque,
     fmt::Debug,
     future::Future,
+    mem::discriminant,
     ops::DerefMut,
+    rc::Rc,
     result,
-    sync::Arc,
+    sync::{atomic, atomic::AtomicBool, Arc},
+    thread,
     time::{Duration, Instant},
 };
 use temporal_sdk_core_api::errors::{CompleteWfError, PollWfError};
@@ -66,11 +80,12 @@ use temporal_sdk_core_protos::{
     },
     temporal::api::{
         command::v1::{command::Attributes, Command as ProtoCommand, Command},
-        common::v1::{Memo, RetryPolicy, SearchAttributes, WorkflowExecution},
+        common::v1::{Memo, MeteringMetadata, RetryPolicy, SearchAttributes, WorkflowExecution},
         enums::v1::WorkflowTaskFailedCause,
         query::v1::WorkflowQuery,
+        sdk::v1::WorkflowTaskCompletedMetadata,
         taskqueue::v1::StickyExecutionAttributes,
-        workflowservice::v1::PollActivityTaskQueueResponse,
+        workflowservice::v1::{get_system_info_response, PollActivityTaskQueueResponse},
     },
     TaskToken,
 };
@@ -79,8 +94,7 @@ use tokio::{
         mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
         oneshot,
     },
-    task,
-    task::{JoinError, JoinHandle},
+    task::{spawn_blocking, LocalSet},
 };
 use tokio_stream::wrappers::UnboundedReceiverStream;
 use tokio_util::sync::CancellationToken;
@@ -94,12 +108,13 @@ const MAX_EAGER_ACTIVITY_RESERVATIONS_PER_WORKFLOW_TASK: usize = 3;
 
 type Result<T, E = WFMachinesError> = result::Result<T, E>;
 type BoxedActivationStream = BoxStream<'static, Result<ActivationOrAuto, PollWfError>>;
+type InternalFlagsRef = Rc<RefCell<InternalFlags>>;
 
 /// Centralizes all state related to workflows and workflow tasks
 pub(crate) struct Workflows {
     task_queue: String,
     local_tx: UnboundedSender<LocalInput>,
-    processing_task: tokio::sync::Mutex<Option<JoinHandle<()>>>,
+    processing_task: TakeCell<thread::JoinHandle<()>>,
     activation_stream: tokio::sync::Mutex<(
         BoxedActivationStream,
         // Used to indicate polling may begin
@@ -113,6 +128,8 @@ pub(crate) struct Workflows {
     activity_tasks_handle: Option<ActivitiesFromWFTsHandle>,
     /// Ensures we stay at or below this worker's maximum concurrent workflow task limit
     wft_semaphore: MeteredSemaphore,
+    local_act_mgr: Arc<LocalActivityManager>,
+    ever_polled: AtomicBool,
 }
 
 pub(crate) struct WorkflowBasics {
@@ -124,19 +141,33 @@ pub(crate) struct WorkflowBasics {
     pub task_queue: String,
     pub ignore_evicts_on_shutdown: bool,
     pub fetching_concurrency: usize,
+    pub server_capabilities: get_system_info_response::Capabilities,
     #[cfg(feature = "save_wf_inputs")]
     pub wf_state_inputs: Option<UnboundedSender<Vec<u8>>>,
 }
 
+pub(crate) struct RunBasics<'a> {
+    pub namespace: String,
+    pub workflow_id: String,
+    pub workflow_type: String,
+    pub run_id: String,
+    pub history: HistoryUpdate,
+    pub metrics: MetricsContext,
+    pub capabilities: &'a get_system_info_response::Capabilities,
+}
+
 impl Workflows {
+    #[allow(clippy::too_many_arguments)] // Not much worth combining here
     pub(super) fn new(
         basics: WorkflowBasics,
         sticky_attrs: Option<StickyExecutionAttributes>,
         client: Arc<dyn WorkerClient>,
         wft_stream: impl Stream<Item = Result<ValidPollWFTQResponse, tonic::Status>> + Send + 'static,
         local_activity_request_sink: impl LocalActivityRequestSink,
+        local_act_mgr: Arc<LocalActivityManager>,
         heartbeat_timeout_rx: UnboundedReceiver<HeartbeatTimeoutMsg>,
         activity_tasks_handle: Option<ActivitiesFromWFTsHandle>,
+        telem_instance: Option<&TelemetryInstance>,
     ) -> Self {
         let (local_tx, local_rx) = unbounded_channel();
         let (fetch_tx, fetch_rx) = unbounded_channel();
@@ -162,54 +193,67 @@ impl Workflows {
             UnboundedReceiverStream::new(local_rx),
             UnboundedReceiverStream::new(heartbeat_timeout_rx).map(Into::into),
         );
-        let mut stream = WFStream::build(
-            basics,
-            extracted_wft_stream,
-            locals_stream,
-            local_activity_request_sink,
-        );
         let (activation_tx, activation_rx) = unbounded_channel();
         let (start_polling_tx, start_polling_rx) = oneshot::channel();
         // We must spawn a task to constantly poll the activation stream, because otherwise
         // activation completions would not cause anything to happen until the next poll.
-        let processing_task = task::spawn(async move {
-            // However, we want to avoid plowing ahead until we've been asked to poll at least once.
-            // This supports activity-only workers.
-            let do_poll = tokio::select! {
-                sp = start_polling_rx => {
-                    sp.is_ok()
-                }
-                _ = shutdown_tok.cancelled() => {
-                    false
-                }
-            };
-            if !do_poll {
-                return;
+        let tracing_sub = telem_instance.map(|ti| ti.trace_subscriber());
+        let processing_task = thread::spawn(move || {
+            if let Some(ts) = tracing_sub {
+                set_trace_subscriber_for_current_thread(ts);
             }
-            while let Some(output) = stream.next().await {
-                match output {
-                    Ok(o) => {
-                        for fetchreq in o.fetch_histories {
-                            fetch_tx
-                                .send(fetchreq)
-                                .expect("Fetch channel must not be dropped");
-                        }
-                        for act in o.activations {
-                            activation_tx
-                                .send(Ok(act))
-                                .expect("Activation processor channel not dropped");
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .thread_name("workflow-processing")
+                .build()
+                .unwrap();
+            let local = LocalSet::new();
+            local.block_on(&rt, async move {
+                let mut stream = WFStream::build(
+                    basics,
+                    extracted_wft_stream,
+                    locals_stream,
+                    local_activity_request_sink,
+                );
+
+                // However, we want to avoid plowing ahead until we've been asked to poll at least
+                // once. This supports activity-only workers.
+                let do_poll = tokio::select! {
+                    sp = start_polling_rx => {
+                        sp.is_ok()
+                    }
+                    _ = shutdown_tok.cancelled() => {
+                        false
+                    }
+                };
+                if !do_poll {
+                    return;
+                }
+                while let Some(output) = stream.next().await {
+                    match output {
+                        Ok(o) => {
+                            for fetchreq in o.fetch_histories {
+                                fetch_tx
+                                    .send(fetchreq)
+                                    .expect("Fetch channel must not be dropped");
+                            }
+                            for act in o.activations {
+                                activation_tx
+                                    .send(Ok(act))
+                                    .expect("Activation processor channel not dropped");
+                            }
                         }
+                        Err(e) => activation_tx
+                            .send(Err(e))
+                            .expect("Activation processor channel not dropped"),
                     }
-                    Err(e) => activation_tx
-                        .send(Err(e))
-                        .expect("Activation processor channel not dropped"),
                 }
-            }
+            });
         });
         Self {
             task_queue,
             local_tx,
-            processing_task: tokio::sync::Mutex::new(Some(processing_task)),
+            processing_task: TakeCell::new(processing_task),
             activation_stream: tokio::sync::Mutex::new((
                 UnboundedReceiverStream::new(activation_rx).boxed(),
                 Some(start_polling_tx),
@@ -218,10 +262,13 @@ impl Workflows {
             sticky_attrs,
             activity_tasks_handle,
             wft_semaphore,
+            local_act_mgr,
+            ever_polled: AtomicBool::new(false),
         }
     }
 
-    pub async fn next_workflow_activation(&self) -> Result<WorkflowActivation, PollWfError> {
+    pub(super) async fn next_workflow_activation(&self) -> Result<WorkflowActivation, PollWfError> {
+        self.ever_polled.store(true, atomic::Ordering::Release);
         loop {
             let al = {
                 let mut lock = self.activation_stream.lock().await;
@@ -233,15 +280,39 @@ impl Workflows {
             };
             Span::current().record("run_id", al.run_id());
             match al {
-                ActivationOrAuto::LangActivation(act) | ActivationOrAuto::ReadyForQueries(act) => {
+                ActivationOrAuto::LangActivation(mut act)
+                | ActivationOrAuto::ReadyForQueries(mut act) => {
+                    sort_act_jobs(&mut act);
                     debug!(activation=%act, "Sending activation to lang");
                     break Ok(act);
                 }
                 ActivationOrAuto::Autocomplete { run_id } => {
-                    self.activation_completed(WorkflowActivationCompletion {
-                        run_id,
-                        status: Some(workflow_completion::Success::from_variants(vec![]).into()),
-                    })
+                    self.activation_completed(
+                        WorkflowActivationCompletion {
+                            run_id,
+                            status: Some(
+                                workflow_completion::Success::from_variants(vec![]).into(),
+                            ),
+                        },
+                        true,
+                        // We need to say a type, but the type is irrelevant, so imagine some
+                        // boxed function we'll never call.
+                        Option::<Box<dyn Fn(PostActivateHookData) + Send>>::None,
+                    )
+                    .await?;
+                }
+                ActivationOrAuto::AutoFail {
+                    run_id,
+                    machines_err,
+                } => {
+                    self.activation_completed(
+                        WorkflowActivationCompletion {
+                            run_id,
+                            status: Some(auto_fail_to_complete_status(machines_err)),
+                        },
+                        true,
+                        Option::<Box<dyn Fn(PostActivateHookData) + Send>>::None,
+                    )
                     .await?;
                 }
             }
@@ -252,10 +323,12 @@ impl Workflows {
     /// the outcome of that completion. See [ActivationCompletedOutcome].
     ///
     /// Returns the most-recently-processed event number for the run.
-    pub async fn activation_completed(
+    pub(super) async fn activation_completed(
         &self,
         completion: WorkflowActivationCompletion,
-    ) -> Result<usize, CompleteWfError> {
+        is_autocomplete: bool,
+        post_activate_hook: Option<impl Fn(PostActivateHookData)>,
+    ) -> Result<(), CompleteWfError> {
         let is_empty_completion = completion.is_empty();
         let completion = validate_completion(completion)?;
         let run_id = completion.run_id().to_string();
@@ -267,7 +340,7 @@ impl Workflows {
         if !was_sent {
             if is_empty_completion {
                 // Empty complete which is likely an evict reply, we can just ignore.
-                return Ok(0);
+                return Ok(());
             }
             panic!(
                 "A non-empty completion was not processed. Workflow processing may have \
@@ -275,9 +348,18 @@ impl Workflows {
             );
         }
 
-        let completion_outcome = rx
-            .await
-            .expect("Send half of activation complete response not dropped");
+        let completion_outcome = if let Ok(c) = rx.await {
+            c
+        } else {
+            dbg_panic!("Send half of activation complete response channel went missing");
+            self.request_eviction(
+                run_id,
+                "Send half of activation complete response channel went missing",
+                EvictionReason::Fatal,
+            );
+            return Ok(());
+        };
+
         let mut wft_from_complete = None;
         let wft_report_status = match completion_outcome.outcome {
             ActivationCompleteOutcome::ReportWFTSuccess(report) => match report {
@@ -288,6 +370,7 @@ impl Workflows {
                             mut commands,
                             query_responses,
                             force_new_wft,
+                            sdk_metadata,
                         },
                 } => {
                     let reserved_act_permits =
@@ -301,6 +384,13 @@ impl Workflows {
                         sticky_attributes: None,
                         return_new_workflow_task: true,
                         force_create_new_workflow_task: force_new_wft,
+                        sdk_metadata,
+                        metering_metadata: MeteringMetadata {
+                            nonfirst_local_activity_execution_attempts: self
+                                .local_act_mgr
+                                .get_nonfirst_attempt_count(&run_id)
+                                as u32,
+                        },
                     };
                     let sticky_attrs = self.sticky_attrs.clone();
                     // Do not return new WFT if we would not cache, because returned new WFTs are
@@ -370,17 +460,30 @@ impl Workflows {
             None
         };
 
+        if let Some(h) = post_activate_hook {
+            h(PostActivateHookData {
+                run_id: &run_id,
+                most_recent_event: completion_outcome.most_recently_processed_event,
+                replaying: completion_outcome.replaying,
+            });
+        }
+
         self.post_activation(PostActivationMsg {
             run_id,
             wft_report_status,
             wft_from_complete: maybe_pwft,
+            is_autocomplete,
         });
 
-        Ok(completion_outcome.most_recently_processed_event)
+        Ok(())
     }
 
     /// Tell workflow that a local activity has finished with the provided result
-    pub fn notify_of_local_result(&self, run_id: impl Into<String>, resolved: LocalResolution) {
+    pub(super) fn notify_of_local_result(
+        &self,
+        run_id: impl Into<String>,
+        resolved: LocalResolution,
+    ) {
         self.send_local(LocalResolutionMsg {
             run_id: run_id.into(),
             res: resolved,
@@ -388,7 +491,7 @@ impl Workflows {
     }
 
     /// Request eviction of a workflow
-    pub fn request_eviction(
+    pub(super) fn request_eviction(
         &self,
         run_id: impl Into<String>,
         message: impl Into<String>,
@@ -398,30 +501,47 @@ impl Workflows {
             run_id: run_id.into(),
             message: message.into(),
             reason,
+            auto_reply_fail_tt: None,
         });
     }
 
     /// Query the state of workflow management. Can return `None` if workflow state is shut down.
-    pub fn get_state_info(&self) -> impl Future<Output = Option<WorkflowStateInfo>> {
+    pub(super) fn get_state_info(&self) -> impl Future<Output = Option<WorkflowStateInfo>> {
         let (tx, rx) = oneshot::channel();
         self.send_local(GetStateInfoMsg { response_tx: tx });
         async move { rx.await.ok() }
     }
 
-    pub fn available_wft_permits(&self) -> usize {
+    pub(super) fn available_wft_permits(&self) -> usize {
         self.wft_semaphore.available_permits()
     }
 
-    pub async fn shutdown(&self) -> Result<(), JoinError> {
-        let maybe_jh = self.processing_task.lock().await.take();
-        if let Some(jh) = maybe_jh {
-            // This acts as a final wake up in case the stream is still alive and wouldn't otherwise
-            // receive another message. It allows it to shut itself down.
-            let _ = self.get_state_info().await;
-            jh.await
-        } else {
-            Ok(())
+    pub(super) async fn shutdown(&self) -> Result<(), anyhow::Error> {
+        if let Some(jh) = self.processing_task.take_once() {
+            // This serves to drive the stream if it is still alive and wouldn't otherwise receive
+            // another message. It allows it to shut itself down.
+            let (waker, stop_waker) = abortable(async {
+                let mut interval = tokio::time::interval(Duration::from_millis(10));
+                loop {
+                    interval.tick().await;
+                    let _ = self.get_state_info().await;
+                }
+            });
+            let (_, jh_res) = tokio::join!(
+                waker,
+                spawn_blocking(move || {
+                    let r = jh.join();
+                    stop_waker.abort();
+                    r
+                })
+            );
+            jh_res?.map_err(|e| anyhow!("Error joining workflow processing thread: {e:?}"))?;
         }
+        Ok(())
+    }
+
+    pub(super) fn ever_polled(&self) -> bool {
+        self.ever_polled.load(atomic::Ordering::Acquire)
     }
 
     /// Must be called after every activation completion has finished
@@ -488,7 +608,7 @@ impl Workflows {
     /// Process eagerly returned activities from WFT completion
     fn handle_eager_activities(
         &self,
-        reserved_act_permits: Vec<OwnedMeteredSemPermit>,
+        reserved_act_permits: Vec<TrackedOwnedMeteredSemPermit>,
         eager_acts: Vec<PollActivityTaskQueueResponse>,
     ) {
         if let Some(at_handle) = self.activity_tasks_handle.as_ref() {
@@ -509,7 +629,7 @@ impl Workflows {
             let with_permits = reserved_act_permits
                 .into_iter()
                 .zip(eager_acts.into_iter())
-                .map(|(permit, resp)| PermittedTqResp { permit, resp });
+                .map(|(permit, resp)| TrackedPermittedTqResp { permit, resp });
             if with_permits.len() > 0 {
                 debug!(
                     "Adding {} activity tasks received from WFT complete",
@@ -532,7 +652,7 @@ impl Workflows {
     fn reserve_activity_slots_for_outgoing_commands(
         &self,
         commands: &mut [Command],
-    ) -> Vec<OwnedMeteredSemPermit> {
+    ) -> Vec<TrackedOwnedMeteredSemPermit> {
         let mut reserved = vec![];
         for cmd in commands {
             if let Some(Attributes::ScheduleActivityTaskCommandAttributes(attrs)) =
@@ -619,6 +739,11 @@ enum ActivationOrAuto {
     Autocomplete {
         run_id: String,
     },
+    #[display(fmt = "AutoFail(run_id={run_id})")]
+    AutoFail {
+        run_id: String,
+        machines_err: WFMachinesError,
+    },
 }
 impl ActivationOrAuto {
     pub fn run_id(&self) -> &str {
@@ -626,6 +751,7 @@ impl ActivationOrAuto {
             ActivationOrAuto::LangActivation(act) => &act.run_id,
             ActivationOrAuto::Autocomplete { run_id, .. } => run_id,
             ActivationOrAuto::ReadyForQueries(act) => &act.run_id,
+            ActivationOrAuto::AutoFail { run_id, .. } => run_id,
         }
     }
 }
@@ -641,9 +767,9 @@ pub(crate) struct PermittedWFT {
     work: PreparedWFT,
     #[cfg_attr(
         feature = "save_wf_inputs",
-        serde(skip, default = "OwnedMeteredSemPermit::fake_deserialized")
+        serde(skip, default = "UsedMeteredSemPermit::fake_deserialized")
     )]
-    permit: OwnedMeteredSemPermit,
+    permit: UsedMeteredSemPermit,
     #[cfg_attr(
         feature = "save_wf_inputs",
         serde(skip, default = "HistoryPaginator::fake_deserialized")
@@ -683,7 +809,7 @@ pub(crate) struct OutstandingTask {
     pub start_time: Instant,
     /// The WFT permit owned by this task, ensures we don't exceed max concurrent WFT, and makes
     /// sure the permit is automatically freed when we delete the task.
-    pub permit: OwnedMeteredSemPermit,
+    pub permit: UsedMeteredSemPermit,
 }
 
 impl OutstandingTask {
@@ -762,6 +888,7 @@ pub(crate) enum ActivationAction {
         commands: Vec<ProtoCommand>,
         query_responses: Vec<QueryResult>,
         force_new_wft: bool,
+        sdk_metadata: WorkflowTaskCompletedMetadata,
     },
     /// We should respond to a legacy query request
     RespondLegacyQuery { result: Box<QueryResult> },
@@ -818,6 +945,7 @@ struct PostActivationMsg {
     run_id: String,
     wft_report_status: WFTReportStatus,
     wft_from_complete: Option<(PreparedWFT, HistoryPaginator)>,
+    is_autocomplete: bool,
 }
 #[derive(Debug, Clone)]
 #[cfg_attr(
@@ -828,6 +956,10 @@ struct RequestEvictMsg {
     run_id: String,
     message: String,
     reason: EvictionReason,
+    /// If set, we requested eviction because something went wrong processing a brand new poll task,
+    /// which means we won't have stored the WFT and we need to track the task token separately so
+    /// we can reply with a failure to server after the evict goes through.
+    auto_reply_fail_tt: Option<TaskToken>,
 }
 #[derive(Debug)]
 pub(crate) struct HeartbeatTimeoutMsg {
@@ -843,6 +975,7 @@ struct GetStateInfoMsg {
 #[derive(Debug)]
 struct ActivationCompleteResult {
     most_recently_processed_event: usize,
+    replaying: bool,
     outcome: ActivationCompleteOutcome,
 }
 /// What needs to be done after calling [Workflows::activation_completed]
@@ -881,7 +1014,7 @@ fn validate_completion(
     match completion.status {
         Some(workflow_activation_completion::Status::Successful(success)) => {
             // Convert to wf commands
-            let commands = success
+            let mut commands = success
                 .commands
                 .into_iter()
                 .map(|c| c.try_into())
@@ -908,9 +1041,20 @@ fn validate_completion(
                 });
             }
 
+            // Any non-query-response commands after a terminal command should be ignored
+            if let Some(term_cmd_pos) = commands.iter().position(|c| c.is_terminal()) {
+                // Query responses are just fine, so keep them.
+                let queries = commands
+                    .split_off(term_cmd_pos + 1)
+                    .into_iter()
+                    .filter(|c| matches!(c, WFCommand::QueryResponse(_)));
+                commands.extend(queries);
+            }
+
             Ok(ValidatedCompletion::Success {
                 run_id: completion.run_id,
                 commands,
+                used_flags: success.used_internal_flags,
             })
         }
         Some(workflow_activation_completion::Status::Failed(failure)) => {
@@ -936,6 +1080,7 @@ enum ValidatedCompletion {
     Success {
         run_id: String,
         commands: Vec<WFCommand>,
+        used_flags: Vec<u32>,
     },
     Fail {
         run_id: String,
@@ -1058,6 +1203,23 @@ impl TryFrom<WorkflowCommand> for WFCommand {
     }
 }
 
+impl WFCommand {
+    /// Returns true if the command is one which ends the workflow:
+    /// * Completed
+    /// * Failed
+    /// * Cancelled
+    /// * Continue-as-new
+    pub fn is_terminal(&self) -> bool {
+        matches!(
+            self,
+            WFCommand::CompleteWorkflow(_)
+                | WFCommand::FailWorkflow(_)
+                | WFCommand::CancelWorkflow(_)
+                | WFCommand::ContinueAsNew(_)
+        )
+    }
+}
+
 #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
 enum CommandID {
     Timer(u32),
@@ -1128,6 +1290,28 @@ impl From<TimestampError> for WFMachinesError {
     }
 }
 
+impl From<anyhow::Error> for WFMachinesError {
+    fn from(value: anyhow::Error) -> Self {
+        WFMachinesError::Fatal(value.to_string())
+    }
+}
+
+fn auto_fail_to_complete_status(err: WFMachinesError) -> workflow_activation_completion::Status {
+    workflow_activation_completion::Status::Failed(Failure {
+        failure: Some(
+            temporal_sdk_core_protos::temporal::api::failure::v1::Failure {
+                message: "Error while processing workflow task".to_string(),
+                source: err.to_string(),
+                stack_trace: "".to_string(),
+                encoded_attributes: None,
+                cause: None,
+                failure_info: None,
+            },
+        ),
+        force_cause: WorkflowTaskFailedCause::from(err.evict_reason()) as i32,
+    })
+}
+
 pub(crate) trait LocalActivityRequestSink: Send + Sync + 'static {
     fn sink_reqs(&self, reqs: Vec<LocalActRequest>) -> Vec<LocalActivityResolution>;
 }
@@ -1158,3 +1342,81 @@ impl LocalActivityRequestSink for LAReqSink {
         res
     }
 }
+
+/// Sorts jobs in an activation to be in the order lang expects:
+/// `patches -> signals -> other -> queries`
+fn sort_act_jobs(wfa: &mut WorkflowActivation) {
+    wfa.jobs.sort_by(|j1, j2| {
+        // Unwrapping is fine here since we'll never issue empty variants
+        let j1v = j1.variant.as_ref().unwrap();
+        let j2v = j2.variant.as_ref().unwrap();
+        if discriminant(j1v) == discriminant(j2v) {
+            return Ordering::Equal;
+        }
+        fn variant_ordinal(v: &workflow_activation_job::Variant) -> u8 {
+            match v {
+                workflow_activation_job::Variant::NotifyHasPatch(_) => 1,
+                workflow_activation_job::Variant::SignalWorkflow(_) => 2,
+                workflow_activation_job::Variant::QueryWorkflow(_) => 4,
+                _ => 3,
+            }
+        }
+        variant_ordinal(j1v).cmp(&variant_ordinal(j2v))
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use itertools::Itertools;
+
+    #[test]
+    fn jobs_sort() {
+        let mut act = WorkflowActivation {
+            jobs: vec![
+                WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::SignalWorkflow(
+                        Default::default(),
+                    )),
+                },
+                WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::NotifyHasPatch(
+                        Default::default(),
+                    )),
+                },
+                WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::QueryWorkflow(
+                        Default::default(),
+                    )),
+                },
+                WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::FireTimer(
+                        Default::default(),
+                    )),
+                },
+                WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::ResolveActivity(
+                        Default::default(),
+                    )),
+                },
+            ],
+            ..Default::default()
+        };
+        sort_act_jobs(&mut act);
+        let variants = act
+            .jobs
+            .into_iter()
+            .map(|j| j.variant.unwrap())
+            .collect_vec();
+        assert_matches!(
+            variants.as_slice(),
+            &[
+                workflow_activation_job::Variant::NotifyHasPatch(_),
+                workflow_activation_job::Variant::SignalWorkflow(_),
+                workflow_activation_job::Variant::FireTimer(_),
+                workflow_activation_job::Variant::ResolveActivity(_),
+                workflow_activation_job::Variant::QueryWorkflow(_)
+            ]
+        )
+    }
+}