@temporalio/core-bridge 1.5.2 → 1.6.0

package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs

@@ -1,7 +1,5 @@
 mod local_acts;
 
-pub(crate) use temporal_sdk_core_api::errors::WFMachinesError;
-
 use super::{
     activity_state_machine::new_activity, cancel_external_state_machine::new_external_cancel,
     cancel_workflow_state_machine::cancel_workflow,
@@ -20,9 +18,10 @@ use crate::{
     telemetry::{metrics::MetricsContext, VecDisplayer},
     worker::{
         workflow::{
+            history_update::NextWFT,
             machines::modify_workflow_properties_state_machine::modify_workflow_properties,
             CommandID, DrivenWorkflow, HistoryUpdate, LocalResolution, OutgoingJob, WFCommand,
-            WorkflowFetcher, WorkflowStartedInfo,
+            WFMachinesError, WorkflowFetcher, WorkflowStartedInfo,
         },
         ExecutingLAId, LocalActRequest, LocalActivityExecutionResult, LocalActivityResolution,
     },
@@ -133,7 +132,7 @@ pub(crate) struct WorkflowMachines {
 }
 
 #[derive(Debug, derive_more::Display)]
-#[display(fmt = "Cmd&Machine({})", "command")]
+#[display(fmt = "Cmd&Machine({command})")]
 struct CommandAndMachine {
     command: MachineAssociatedCommand,
     machine: MachineKey,
@@ -142,7 +141,7 @@ struct CommandAndMachine {
 #[derive(Debug, derive_more::Display)]
 enum MachineAssociatedCommand {
     Real(Box<ProtoCommand>),
-    #[display(fmt = "FakeLocalActivityMarker({})", "_0")]
+    #[display(fmt = "FakeLocalActivityMarker({_0})")]
     FakeLocalActivityMarker(u32),
 }
 
@@ -156,7 +155,7 @@ struct ChangeInfo {
 #[must_use]
 #[allow(clippy::large_enum_variant)]
 pub(super) enum MachineResponse {
-    #[display(fmt = "PushWFJob({})", "_0")]
+    #[display(fmt = "PushWFJob({_0})")]
     PushWFJob(OutgoingJob),
 
     /// Pushes a new command into the list that will be sent to server once we respond with the
@@ -165,31 +164,31 @@ pub(super) enum MachineResponse {
     /// The machine requests the creation of another *different* machine. This acts as if lang
     /// had replied to the activation with a command, but we use a special set of IDs to avoid
     /// collisions.
-    #[display(fmt = "NewCoreOriginatedCommand({:?})", "_0")]
+    #[display(fmt = "NewCoreOriginatedCommand({_0:?})")]
     NewCoreOriginatedCommand(ProtoCmdAttrs),
-    #[display(fmt = "IssueFakeLocalActivityMarker({})", "_0")]
+    #[display(fmt = "IssueFakeLocalActivityMarker({_0})")]
     IssueFakeLocalActivityMarker(u32),
     #[display(fmt = "TriggerWFTaskStarted")]
     TriggerWFTaskStarted {
         task_started_event_id: i64,
         time: SystemTime,
     },
-    #[display(fmt = "UpdateRunIdOnWorkflowReset({})", run_id)]
+    #[display(fmt = "UpdateRunIdOnWorkflowReset({run_id})")]
     UpdateRunIdOnWorkflowReset { run_id: String },
 
     /// Queue a local activity to be processed by the worker
     #[display(fmt = "QueueLocalActivity")]
     QueueLocalActivity(ValidScheduleLA),
     /// Request cancellation of an executing local activity
-    #[display(fmt = "RequestCancelLocalActivity({})", "_0")]
+    #[display(fmt = "RequestCancelLocalActivity({_0})")]
     RequestCancelLocalActivity(u32),
     /// Indicates we are abandoning the indicated LA, so we can remove it from "outstanding" LAs
     /// and we will not try to WFT heartbeat because of it.
-    #[display(fmt = "AbandonLocalActivity({:?})", "_0")]
+    #[display(fmt = "AbandonLocalActivity({_0:?})")]
     AbandonLocalActivity(u32),
 
     /// Set the workflow time to the provided time
-    #[display(fmt = "UpdateWFTime({:?})", "_0")]
+    #[display(fmt = "UpdateWFTime({_0:?})")]
     UpdateWFTime(Option<SystemTime>),
 }
 
@@ -212,7 +211,7 @@ impl WorkflowMachines {
         driven_wf: DrivenWorkflow,
         metrics: MetricsContext,
     ) -> Self {
-        let replaying = history.previous_started_event_id > 0;
+        let replaying = history.previous_wft_started_id > 0;
         Self {
             last_history_from_server: history,
             namespace,
@@ -255,10 +254,10 @@ impl WorkflowMachines {
             .and_then(|(st, et)| et.duration_since(st).ok())
     }
 
-    pub(crate) async fn new_history_from_server(&mut self, update: HistoryUpdate) -> Result<()> {
+    pub(crate) fn new_history_from_server(&mut self, update: HistoryUpdate) -> Result<()> {
         self.last_history_from_server = update;
-        self.replaying = self.last_history_from_server.previous_started_event_id > 0;
-        self.apply_next_wft_from_history().await?;
+        self.replaying = self.last_history_from_server.previous_wft_started_id > 0;
+        self.apply_next_wft_from_history()?;
         Ok(())
     }
 
@@ -290,9 +289,8 @@ impl WorkflowMachines {
                     self.process_machine_responses(mk, resps)?;
                 } else {
                     return Err(WFMachinesError::Nondeterminism(format!(
-                        "Command matching activity with seq num {} existed but was not a \
-                        local activity!",
-                        seq
+                        "Command matching activity with seq num {seq} existed but was not a \
+                        local activity!"
                     )));
                 }
                 self.local_activity_data.done_executing(seq);
@@ -366,8 +364,8 @@ impl WorkflowMachines {
 
     /// Iterate the state machines, which consists of grabbing any pending outgoing commands from
     /// the workflow code, handling them, and preparing them to be sent off to the server.
-    pub(crate) async fn iterate_machines(&mut self) -> Result<()> {
-        let results = self.drive_me.fetch_workflow_iteration_output().await;
+    pub(crate) fn iterate_machines(&mut self) -> Result<()> {
+        let results = self.drive_me.fetch_workflow_iteration_output();
         self.handle_driven_results(results)?;
         self.prepare_commands()?;
         if self.workflow_is_finished() {
@@ -378,10 +376,16 @@ impl WorkflowMachines {
         Ok(())
     }
 
+    /// Returns true if machines are ready to apply the next WFT sequence, false if events will need
+    /// to be fetched in order to create a complete update with the entire next WFT sequence.
+    pub(crate) fn ready_to_apply_next_wft(&self) -> bool {
+        self.last_history_from_server
+            .can_take_next_wft_sequence(self.current_started_event_id)
+    }
+
     /// Apply the next (unapplied) entire workflow task from history to these machines. Will replay
-    /// any events that need to be replayed until caught up to the newest WFT. May also fetch
-    /// history from server if needed.
-    pub(crate) async fn apply_next_wft_from_history(&mut self) -> Result<usize> {
+    /// any events that need to be replayed until caught up to the newest WFT.
+    pub(crate) fn apply_next_wft_from_history(&mut self) -> Result<usize> {
         // If we have already seen the terminal event for the entire workflow in a previous WFT,
         // then we don't need to do anything here, and in fact we need to avoid re-applying the
         // final WFT.
@@ -390,16 +394,25 @@ impl WorkflowMachines {
         }
 
         let last_handled_wft_started_id = self.current_started_event_id;
-        let events = {
-            let mut evts = self
+        let events =
+            match self
                 .last_history_from_server
                 .take_next_wft_sequence(last_handled_wft_started_id)
-                .await
-                .map_err(WFMachinesError::HistoryFetchingError)?;
-            // Do not re-process events we have already processed
-            evts.retain(|e| e.event_id > self.last_processed_event);
-            evts
-        };
+            {
+                NextWFT::ReplayOver => {
+                    vec![]
+                }
+                NextWFT::WFT(mut evts) => {
+                    // Do not re-process events we have already processed
+                    evts.retain(|e| e.event_id > self.last_processed_event);
+                    evts
+                }
+                NextWFT::NeedFetch => return Err(WFMachinesError::Fatal(
+                    "Need to fetch history events to continue applying workflow task, but this \
+                     should be prevented ahead of time! This is a Core SDK bug."
+                        .to_string(),
+                )),
+            };
         let num_events_to_process = events.len();
 
         // We're caught up on reply if there are no new events to process
@@ -434,7 +447,11 @@ impl WorkflowMachines {
 
         // Scan through to the next WFT, searching for any patch / la markers, so that we can
         // pre-resolve them.
-        for e in self.last_history_from_server.peek_next_wft_sequence() {
+        let mut wake_las = vec![];
+        for e in self
+            .last_history_from_server
+            .peek_next_wft_sequence(last_handled_wft_started_id)
+        {
             if let Some((patch_id, _)) = e.get_patch_marker_details() {
                 self.encountered_change_markers.insert(
                     patch_id.clone(),
@@ -448,7 +465,30 @@ impl WorkflowMachines {
                         .into(),
                 );
             } else if e.is_local_activity_marker() {
-                self.local_activity_data.process_peekahead_marker(e)?;
+                if let Some(la_dat) = e.clone().into_local_activity_marker_details() {
+                    if let Ok(mk) =
+                        self.get_machine_key(CommandID::LocalActivity(la_dat.marker_dat.seq))
+                    {
+                        wake_las.push((mk, la_dat));
+                    } else {
+                        self.local_activity_data.insert_peeked_marker(la_dat);
+                    }
+                } else {
+                    return Err(WFMachinesError::Fatal(format!(
+                        "Local activity marker was unparsable: {e:?}"
+                    )));
+                }
+            }
+        }
+        for (mk, la_dat) in wake_las {
+            let mach = self.machine_mut(mk);
+            if let Machines::LocalActivityMachine(ref mut lam) = *mach {
+                if lam.will_accept_resolve_marker() {
+                    let resps = lam.try_resolve_with_dat(la_dat.into())?;
+                    self.process_machine_responses(mk, resps)?;
+                } else {
+                    self.local_activity_data.insert_peeked_marker(la_dat);
+                }
             }
         }
 
@@ -488,7 +528,7 @@ impl WorkflowMachines {
         }
         if self.replaying
             && self.current_started_event_id
-                >= self.last_history_from_server.previous_started_event_id
+                >= self.last_history_from_server.previous_wft_started_id
             && event.event_type() != EventType::WorkflowTaskCompleted
         {
             // Replay is finished
@@ -497,8 +537,7 @@ impl WorkflowMachines {
         if event.event_type() == EventType::Unspecified || event.attributes.is_none() {
             return if !event.worker_may_ignore {
                 Err(WFMachinesError::Fatal(format!(
-                    "Event type is unspecified! This history is invalid. Event detail: {:?}",
-                    event
+                    "Event type is unspecified! This history is invalid. Event detail: {event:?}"
                 )))
             } else {
                 debug!("Event is ignorable");
@@ -526,8 +565,7 @@ impl WorkflowMachines {
                 None => {
                     return Err(WFMachinesError::Nondeterminism(format!(
                         "During event handling, this event had an initial command ID but we \
-                            could not find a matching command for it: {:?}",
-                        event
+                            could not find a matching command for it: {event:?}"
                     )));
                 }
             }
@@ -550,7 +588,7 @@ impl WorkflowMachines {
     fn handle_command_event(&mut self, event: HistoryEvent) -> Result<()> {
         if event.is_local_activity_marker() {
             let deets = event.extract_local_activity_marker_data().ok_or_else(|| {
-                WFMachinesError::Fatal(format!("Local activity marker was unparsable: {:?}", event))
+                WFMachinesError::Fatal(format!("Local activity marker was unparsable: {event:?}"))
             })?;
             let cmdid = CommandID::LocalActivity(deets.seq);
             let mkey = self.get_machine_key(cmdid)?;
@@ -562,8 +600,7 @@ impl WorkflowMachines {
             } else {
                 return Err(WFMachinesError::Fatal(format!(
                     "Encountered local activity marker but the associated machine was of the \
-                     wrong type! {:?}",
-                    event
+                     wrong type! {event:?}"
                 )));
             }
         }
@@ -588,8 +625,7 @@ impl WorkflowMachines {
                 c
             } else {
                 return Err(WFMachinesError::Nondeterminism(format!(
-                    "No command scheduled for event {}",
-                    event
+                    "No command scheduled for event {event}"
                 )));
             };
 
@@ -645,8 +681,7 @@ impl WorkflowMachines {
                     );
                 } else {
                     return Err(WFMachinesError::Fatal(format!(
-                        "WorkflowExecutionStarted event did not have appropriate attributes: {}",
-                        event
+                        "WorkflowExecutionStarted event did not have appropriate attributes: {event}"
                     )));
                 }
             }
@@ -682,8 +717,7 @@ impl WorkflowMachines {
             }
             _ => {
                 return Err(WFMachinesError::Fatal(format!(
-                    "The event is not a non-stateful event, but we tried to handle it as one: {}",
-                    event
+                    "The event is not a non-stateful event, but we tried to handle it as one: {event}"
                 )));
             }
         }
@@ -756,7 +790,7 @@ impl WorkflowMachines {
     ) -> Result<()> {
         let sm = self.machine(smk);
         if !machine_responses.is_empty() {
-            debug!(responses = %machine_responses.display(), machine_name = %sm.name(),
+            trace!(responses = %machine_responses.display(), machine_name = %sm.name(),
                    "Machine produced responses");
         }
         self.process_machine_resps_impl(smk, machine_responses)
@@ -811,8 +845,7 @@ impl WorkflowMachines {
                     }
                     c => {
                         return Err(WFMachinesError::Fatal(format!(
-                        "A machine requested to create a new command of an unsupported type: {:?}",
-                        c
+                        "A machine requested to create a new command of an unsupported type: {c:?}"
                     )))
                     }
                 },
@@ -849,7 +882,7 @@ impl WorkflowMachines {
                                 Duration::from_secs(0),
                                 removed_act.attempt,
                                 None,
-                                None,
+                                removed_act.original_schedule_time,
                             )?;
                             self.process_machine_responses(smk, more_responses)?;
                         } else {
@@ -937,8 +970,7 @@ impl WorkflowMachines {
                     )
                     .map_err(|e| {
                         WFMachinesError::Fatal(format!(
-                            "Invalid schedule local activity request (seq {}): {}",
-                            seq, e
+                            "Invalid schedule local activity request (seq {seq}): {e}"
                         ))
                     })?;
                     let (la, mach_resp) = new_local_activity(
@@ -1075,7 +1107,7 @@ impl WorkflowMachines {
 
     fn get_machine_key(&self, id: CommandID) -> Result<MachineKey> {
         Ok(*self.id_to_machine.get(&id).ok_or_else(|| {
-            WFMachinesError::Fatal(format!("Missing associated machine for {:?}", id))
+            WFMachinesError::Fatal(format!("Missing associated machine for {id:?}"))
         })?)
     }
 
@@ -1173,9 +1205,8 @@ fn change_marker_handling(event: &HistoryEvent, mach: &Machines) -> Result<Chang
                 return Ok(ChangeMarkerOutcome::SkipEvent);
             }
             return Err(WFMachinesError::Nondeterminism(format!(
-                "Non-deprecated patch marker encountered for change {}, \
-                            but there is no corresponding change command!",
-                patch_name
+                "Non-deprecated patch marker encountered for change {patch_name}, \
+                            but there is no corresponding change command!"
             )));
         }
         // Patch machines themselves may also not *have* matching markers, where non-deprecated

package/sdk-core/core/src/worker/workflow/machines/workflow_task_state_machine.rs

@@ -48,7 +48,7 @@ pub(super) enum WFTaskMachineCommand {
         task_started_event_id: i64,
         time: SystemTime,
     },
-    #[display(fmt = "RunIdOnWorkflowResetUpdate({})", run_id)]
+    #[display(fmt = "RunIdOnWorkflowResetUpdate({run_id})")]
     RunIdOnWorkflowResetUpdate { run_id: String },
 }
 
@@ -121,8 +121,7 @@ impl TryFrom<HistoryEvent> for WorkflowTaskMachineEvents {
                     }
                 } else {
                     return Err(WFMachinesError::Fatal(format!(
-                        "Workflow task started event must contain timestamp: {}",
-                        e
+                        "Workflow task started event must contain timestamp: {e}"
                     )));
                 };
                 WFTStartedDat {
@@ -150,15 +149,13 @@ impl TryFrom<HistoryEvent> for WorkflowTaskMachineEvents {
                     })
                 } else {
                     return Err(WFMachinesError::Fatal(format!(
-                        "Workflow task failed is missing attributes: {}",
-                        e
+                        "Workflow task failed is missing attributes: {e}"
                     )));
                 }
             }
             _ => {
                 return Err(WFMachinesError::Nondeterminism(format!(
-                    "Event does not apply to a wf task machine: {}",
-                    e
+                    "Event does not apply to a wf task machine: {e}"
                 )))
             }
         })

package/sdk-core/core/src/worker/workflow/managed_run/managed_wf_test.rs

@@ -4,11 +4,13 @@ use crate::{
     test_help::TEST_Q,
     worker::{
         workflow::{
-            history_update::TestHBExt, machines::WorkflowMachines, WFCommand, WorkflowFetcher,
+            history_update::tests::TestHBExt, machines::WorkflowMachines, WFCommand,
+            WorkflowFetcher,
         },
         LocalActRequest, LocalActivityResolution,
     },
 };
+use crossbeam::channel::bounded;
 use std::{convert::TryInto, time::Duration};
 use temporal_sdk::{WorkflowFunction, WorkflowResult};
 use temporal_sdk_core_protos::{
@@ -27,13 +29,12 @@ use tokio::{
 };
 
 pub(crate) struct WFFutureDriver {
-    completions_rx: UnboundedReceiver<WorkflowActivationCompletion>,
+    completions_q: crossbeam::channel::Receiver<WorkflowActivationCompletion>,
 }
 
-#[async_trait::async_trait]
 impl WorkflowFetcher for WFFutureDriver {
-    async fn fetch_workflow_iteration_output(&mut self) -> Vec<WFCommand> {
-        if let Some(completion) = self.completions_rx.recv().await {
+    fn fetch_workflow_iteration_output(&mut self) -> Vec<WFCommand> {
+        if let Ok(completion) = self.completions_q.try_recv() {
             debug!("Managed wf completion: {}", completion);
             completion
                 .status
@@ -57,6 +58,8 @@ impl WorkflowFetcher for WFFutureDriver {
 pub struct ManagedWFFunc {
     mgr: WorkflowManager,
     activation_tx: UnboundedSender<WorkflowActivation>,
+    completions_rx: UnboundedReceiver<WorkflowActivationCompletion>,
+    completions_sync_tx: crossbeam::channel::Sender<WorkflowActivationCompletion>,
     future_handle: Option<JoinHandle<WorkflowResult<()>>>,
     was_shutdown: bool,
 }
@@ -79,7 +82,10 @@ impl ManagedWFFunc {
             completions_tx,
         );
         let spawned = tokio::spawn(wff);
-        let driver = WFFutureDriver { completions_rx };
+        let (completions_sync_tx, completions_sync_rx) = bounded(1);
+        let driver = WFFutureDriver {
+            completions_q: completions_sync_rx,
+        };
         let state_machines = WorkflowMachines::new(
             "test_namespace".to_string(),
             "wfid".to_string(),
@@ -93,6 +99,8 @@ impl ManagedWFFunc {
         Self {
             mgr,
             activation_tx: activations,
+            completions_rx,
+            completions_sync_tx,
             future_handle: Some(spawned),
             was_shutdown: false,
         }
@@ -100,7 +108,7 @@ impl ManagedWFFunc {
 
     #[instrument(skip(self))]
     pub(crate) async fn get_next_activation(&mut self) -> Result<WorkflowActivation> {
-        let res = self.mgr.get_next_activation().await?;
+        let res = self.mgr.get_next_activation()?;
         debug!("Managed wf next activation: {}", &res);
         self.push_activation_to_wf(&res).await?;
         Ok(res)
@@ -121,7 +129,7 @@ impl ManagedWFFunc {
         &mut self,
         update: HistoryUpdate,
     ) -> Result<WorkflowActivation> {
-        let res = self.mgr.feed_history_from_server(update).await?;
+        let res = self.mgr.feed_history_from_server(update)?;
         self.push_activation_to_wf(&res).await?;
         Ok(res)
     }
@@ -183,7 +191,11 @@ impl ManagedWFFunc {
         self.activation_tx
             .send(res.clone())
             .expect("Workflow should not be dropped if we are still sending activations");
-        self.mgr.machines.iterate_machines().await?;
+        // Move the completion response to the sync workflow bridge
+        self.completions_sync_tx
+            .send(self.completions_rx.recv().await.unwrap())
+            .unwrap();
+        self.mgr.machines.iterate_machines()?;
         Ok(())
     }
 }