@temporalio/core-bridge 0.16.4 → 0.18.0

package/sdk-core/{src → core/src/workflow}/machines/workflow_machines.rs

@@ -1,20 +1,29 @@
+mod local_acts;
+
+pub(crate) use temporal_sdk_core_api::errors::WFMachinesError;
+
+use super::{
+    activity_state_machine::new_activity, cancel_external_state_machine::new_external_cancel,
+    cancel_workflow_state_machine::cancel_workflow,
+    child_workflow_state_machine::new_child_workflow,
+    complete_workflow_state_machine::complete_workflow,
+    continue_as_new_workflow_state_machine::continue_as_new,
+    fail_workflow_state_machine::fail_workflow, local_activity_state_machine::new_local_activity,
+    patch_state_machine::has_change, signal_external_state_machine::new_external_signal,
+    timer_state_machine::new_timer, workflow_machines::local_acts::LocalActivityData,
+    workflow_task_state_machine::WorkflowTaskMachine, MachineKind, Machines, NewMachineWithCommand,
+    TemporalStateMachine,
+};
 use crate::{
-    machines::{
-        activity_state_machine::new_activity, cancel_external_state_machine::new_external_cancel,
-        cancel_workflow_state_machine::cancel_workflow,
-        child_workflow_state_machine::new_child_workflow,
-        complete_workflow_state_machine::complete_workflow,
-        continue_as_new_workflow_state_machine::continue_as_new,
-        fail_workflow_state_machine::fail_workflow, patch_state_machine::has_change,
-        signal_external_state_machine::new_external_signal, timer_state_machine::new_timer,
-        workflow_task_state_machine::WorkflowTaskMachine, MachineKind, NewMachineWithCommand,
-        ProtoCommand, TemporalStateMachine, WFCommand,
-    },
-    protosext::HistoryEventExt,
+    protosext::{HistoryEventExt, ValidScheduleLA},
     telemetry::{metrics::MetricsContext, VecDisplayer},
-    workflow::{CommandID, DrivenWorkflow, HistoryUpdate, WorkflowFetcher},
+    worker::{
+        ExecutingLAId, LocalActRequest, LocalActivityExecutionResult, LocalActivityResolution,
+    },
+    workflow::{
+        CommandID, DrivenWorkflow, HistoryUpdate, LocalResolution, WFCommand, WorkflowFetcher,
+    },
 };
-use prost_types::TimestampOutOfSystemRangeError;
 use slotmap::SlotMap;
 use std::{
     borrow::{Borrow, BorrowMut},
@@ -27,8 +36,8 @@ use temporal_sdk_core_protos::{
     coresdk::{
         common::{NamespacedWorkflowExecution, Payload},
         workflow_activation::{
-            wf_activation_job::{self, Variant},
-            NotifyHasPatch, StartWorkflow, UpdateRandomSeed, WfActivation,
+            workflow_activation_job::{self, Variant},
+            NotifyHasPatch, StartWorkflow, UpdateRandomSeed, WorkflowActivation,
         },
         workflow_commands::{
             request_cancel_external_workflow_execution as cancel_we,
@@ -37,12 +46,13 @@ use temporal_sdk_core_protos::{
         FromPayloadsExt,
     },
     temporal::api::{
+        command::v1::Command as ProtoCommand,
         common::v1::Header,
         enums::v1::EventType,
-        history::v1::{history_event, HistoryEvent},
+        history::v1::{history_event, HistoryEvent, WorkflowExecutionStartedEventAttributes},
     },
+    utilities::TryIntoOrNone,
 };
-use tracing::Level;
 
 type Result<T, E = WFMachinesError> = std::result::Result<T, E>;
 
@@ -61,12 +71,18 @@ pub(crate) struct WorkflowMachines {
     /// Eventually, this number should reach the started id in the latest history update, but
     /// we must incrementally apply the history while communicating with lang.
     next_started_event_id: i64,
+    /// The event id of the most recent event processed. It's possible in some situations (ex legacy
+    /// queries) to receive a history with no new workflow tasks. If the last history we processed
+    /// also had no new tasks, we need a way to know not to apply the same events over again.
+    pub last_processed_event: i64,
     /// True if the workflow is replaying from history
     pub replaying: bool,
     /// Namespace this workflow exists in
     pub namespace: String,
     /// Workflow identifier
     pub workflow_id: String,
+    /// Workflow type identifier. (Function name, class, etc)
+    pub workflow_type: String,
     /// Identifies the current run
     pub run_id: String,
     /// The time the workflow execution began, as told by the WEStarted event
@@ -74,17 +90,18 @@ pub(crate) struct WorkflowMachines {
     /// The time the workflow execution finished, as determined by when the machines handled
     /// a terminal workflow command. If this is `Some`, you know the workflow is ended.
     workflow_end_time: Option<SystemTime>,
-    /// The current workflow time if it has been established
+    /// The WFT start time if it has been established
+    wft_start_time: Option<SystemTime>,
+    /// The current workflow time if it has been established. This may differ from the WFT start
+    /// time since local activities may advance the clock
     current_wf_time: Option<SystemTime>,
 
-    // TODO: Nothing gets deleted from here
-    all_machines: SlotMap<MachineKey, Box<dyn TemporalStateMachine + 'static>>,
+    all_machines: SlotMap<MachineKey, Machines>,
 
     /// A mapping for accessing machines associated to a particular event, where the key is the id
     /// of the initiating event for that machine.
     machines_by_event_id: HashMap<i64, MachineKey>,
 
-    // TODO: Nothing gets deleted from here
     /// Maps command ids as created by workflow authors to their associated machines.
     id_to_machine: HashMap<CommandID, MachineKey>,
 
@@ -93,19 +110,20 @@ pub(crate) struct WorkflowMachines {
     commands: VecDeque<CommandAndMachine>,
     /// Commands generated by the currently processing workflow task, which will eventually be
     /// transferred to `commands` (and hence eventually sent to the server)
-    ///
-    /// Old note: It is a queue as commands can be added (due to marker based commands) while
-    /// iterating over already added commands.
     current_wf_task_commands: VecDeque<CommandAndMachine>,
+
     /// Information about patch markers we have already seen while replaying history
     encountered_change_markers: HashMap<String, ChangeInfo>,
 
+    /// Contains extra local-activity related data
+    local_activity_data: LocalActivityData,
+
     /// The workflow that is being driven by this instance of the machines
     drive_me: DrivenWorkflow,
 
     /// Is set to true once we've seen the final event in workflow history, to avoid accidentally
     /// re-applying the final workflow task.
-    have_seen_terminal_event: bool,
+    pub have_seen_terminal_event: bool,
 
     /// Metrics context
     pub metrics: MetricsContext,
@@ -114,13 +132,19 @@ pub(crate) struct WorkflowMachines {
 #[derive(Debug, derive_more::Display)]
 #[display(fmt = "Cmd&Machine({})", "command")]
 struct CommandAndMachine {
-    command: ProtoCommand,
+    command: MachineAssociatedCommand,
     machine: MachineKey,
 }
 
+#[derive(Debug, derive_more::Display)]
+enum MachineAssociatedCommand {
+    Real(Box<ProtoCommand>),
+    #[display(fmt = "FakeLocalActivityMarker({})", "_0")]
+    FakeLocalActivityMarker(u32),
+}
+
 #[derive(Debug, Clone, Copy)]
 struct ChangeInfo {
-    deprecated: bool,
     created_command: bool,
 }
 
@@ -129,10 +153,12 @@ struct ChangeInfo {
 #[must_use]
 #[allow(clippy::large_enum_variant)]
 pub enum MachineResponse {
-    #[display(fmt = "PushWFJob")]
-    PushWFJob(wf_activation_job::Variant),
+    #[display(fmt = "PushWFJob({})", "_0")]
+    PushWFJob(workflow_activation_job::Variant),
 
     IssueNewCommand(ProtoCommand),
+    #[display(fmt = "IssueFakeLocalActivityMarker({})", "_0")]
+    IssueFakeLocalActivityMarker(u32),
     #[display(fmt = "TriggerWFTaskStarted")]
     TriggerWFTaskStarted {
         task_started_event_id: i64,
@@ -142,43 +168,37 @@ pub enum MachineResponse {
     UpdateRunIdOnWorkflowReset {
         run_id: String,
     },
+
+    /// Queue a local activity to be processed by the worker
+    #[display(fmt = "QueueLocalActivity")]
+    QueueLocalActivity(ValidScheduleLA),
+    /// Request cancellation of an executing local activity
+    #[display(fmt = "RequestCancelLocalActivity({})", "_0")]
+    RequestCancelLocalActivity(u32),
+    /// Indicates we are abandoning the indicated LA, so we can remove it from "outstanding" LAs
+    /// and we will not try to WFT heartbeat because of it.
+    #[display(fmt = "AbandonLocalActivity({:?})", "_0")]
+    AbandonLocalActivity(u32),
+
+    /// Set the workflow time to the provided time
+    #[display(fmt = "UpdateWFTime({:?})", "_0")]
+    UpdateWFTime(Option<SystemTime>),
 }
 
-// Must use `From` b/c ofZZ
 impl<T> From<T> for MachineResponse
 where
-    T: Into<wf_activation_job::Variant>,
+    T: Into<workflow_activation_job::Variant>,
 {
     fn from(v: T) -> Self {
         Self::PushWFJob(v.into())
     }
 }
 
-#[derive(thiserror::Error, Debug)]
-pub(crate) enum WFMachinesError {
-    #[error("Nondeterminism error: {0}")]
-    Nondeterminism(String),
-    #[error("Fatal error in workflow machines: {0}")]
-    Fatal(String),
-
-    #[error("Unrecoverable network error while fetching history: {0}")]
-    HistoryFetchingError(tonic::Status),
-
-    /// Should always be caught internally and turned into a workflow task failure
-    #[error("Unable to process partial event history because workflow is no longer cached.")]
-    CacheMiss,
-}
-
-impl From<TimestampOutOfSystemRangeError> for WFMachinesError {
-    fn from(_: TimestampOutOfSystemRangeError) -> Self {
-        Self::Fatal("Could not decode timestamp".to_string())
-    }
-}
-
 impl WorkflowMachines {
     pub(crate) fn new(
         namespace: String,
         workflow_id: String,
+        workflow_type: String,
         run_id: String,
         history: HistoryUpdate,
         driven_wf: DrivenWorkflow,
@@ -189,6 +209,7 @@ impl WorkflowMachines {
             last_history_from_server: history,
             namespace,
             workflow_id,
+            workflow_type,
             run_id,
             drive_me: driven_wf,
             replaying,
@@ -196,8 +217,10 @@ impl WorkflowMachines {
             // In an ideal world one could say ..Default::default() here and it'd still work.
             current_started_event_id: 0,
             next_started_event_id: 0,
+            last_processed_event: 0,
             workflow_start_time: None,
             workflow_end_time: None,
+            wft_start_time: None,
             current_wf_time: None,
             all_machines: Default::default(),
             machines_by_event_id: Default::default(),
@@ -205,6 +228,7 @@ impl WorkflowMachines {
             commands: Default::default(),
             current_wf_task_commands: Default::default(),
             encountered_change_markers: Default::default(),
+            local_activity_data: LocalActivityData::default(),
             have_seen_terminal_event: false,
         }
     }
@@ -229,16 +253,60 @@ impl WorkflowMachines {
         Ok(())
     }
 
+    /// Let this workflow know that something we've been waiting locally on has resolved, like a
+    /// local activity or side effect
+    pub(crate) fn local_resolution(&mut self, resolution: LocalResolution) -> Result<()> {
+        match resolution {
+            LocalResolution::LocalActivity(LocalActivityResolution {
+                seq,
+                result,
+                runtime,
+                attempt,
+                backoff,
+                original_schedule_time,
+            }) => {
+                let act_id = CommandID::LocalActivity(seq);
+                let mk = self.get_machine_key(act_id)?;
+                let mach = self.machine_mut(mk);
+                if let Machines::LocalActivityMachine(ref mut lam) = *mach {
+                    let resps =
+                        lam.try_resolve(result, runtime, attempt, backoff, original_schedule_time)?;
+                    self.process_machine_responses(mk, resps)?;
+                } else {
+                    return Err(WFMachinesError::Nondeterminism(format!(
+                        "Command matching activity with seq num {} existed but was not a \
+                        local activity!",
+                        seq
+                    )));
+                }
+                self.local_activity_data.done_executing(seq);
+            }
+        }
+        Ok(())
+    }
+
+    /// Drain all queued local activities that need executing or cancellation
+    pub(crate) fn drain_queued_local_activities(&mut self) -> Vec<LocalActRequest> {
+        self.local_activity_data
+            .take_all_reqs(&self.workflow_type, &self.workflow_id, &self.run_id)
+    }
+
+    /// Returns the number of local activities we know we need to execute but have not yet finished
+    pub(crate) fn outstanding_local_activity_count(&self) -> usize {
+        self.local_activity_data.outstanding_la_count()
+    }
+
+    /// Returns the start attributes for the workflow if it has started
+    pub(crate) fn started_attrs(&self) -> Option<&WorkflowExecutionStartedEventAttributes> {
+        self.drive_me.get_started_attrs()
+    }
+
     /// Handle a single event from the workflow history. `has_next_event` should be false if `event`
     /// is the last event in the history.
     ///
     /// TODO: Describe what actually happens in here
     #[instrument(level = "debug", skip(self, event), fields(event=%event))]
-    pub(crate) fn handle_event(
-        &mut self,
-        event: &HistoryEvent,
-        has_next_event: bool,
-    ) -> Result<()> {
+    fn handle_event(&mut self, event: &HistoryEvent, has_next_event: bool) -> Result<()> {
         if event.is_final_wf_execution_event() {
             self.have_seen_terminal_event = true;
         }
@@ -287,21 +355,24 @@ impl WorkflowMachines {
     /// Called when a workflow task started event has triggered. Ensures we are tracking the ID
     /// of the current started event as well as workflow time properly.
     fn task_started(&mut self, task_started_event_id: i64, time: SystemTime) -> Result<()> {
-        let s = span!(Level::DEBUG, "Task started trigger");
-        let _enter = s.enter();
-
-        // TODO: Local activity machines
-        // // Give local activities a chance to recreate their requests if they were lost due
-        // // to the last workflow task failure. The loss could happen only the last workflow task
-        // // was forcibly created by setting forceCreate on RespondWorkflowTaskCompletedRequest.
-        // if (nonProcessedWorkflowTask) {
-        //     for (LocalActivityStateMachine value : localActivityMap.values()) {
-        //         value.nonReplayWorkflowTaskStarted();
-        //     }
-        // }
-
         self.current_started_event_id = task_started_event_id;
+        self.wft_start_time = Some(time);
         self.set_current_time(time);
+
+        // Notify local activity machines that we started a non-replay WFT, which will allow any
+        // which were waiting for a marker to instead decide to execute the LA since it clearly
+        // will not be resolved via marker.
+        if !self.replaying {
+            let mut resps = vec![];
+            for (k, mach) in self.all_machines.iter_mut() {
+                if let Machines::LocalActivityMachine(lam) = mach {
+                    resps.push((k, lam.encountered_non_replay_wft()?));
+                }
+            }
+            for (mkey, resp_set) in resps {
+                self.process_machine_responses(mkey, resp_set)?;
+            }
+        }
         Ok(())
     }
 
@@ -315,10 +386,25 @@ impl WorkflowMachines {
     /// with a state machine, which is then notified about the event and the command is removed from
     /// the commands queue.
     fn handle_command_event(&mut self, event: &HistoryEvent) -> Result<()> {
-        // TODO: Local activity handling stuff
-        //     if (handleLocalActivityMarker(event)) {
-        //       return;
-        //     }
+        if event.is_local_activity_marker() {
+            let deets = event.extract_local_activity_marker_data().ok_or_else(|| {
+                WFMachinesError::Fatal(format!("Local activity marker was unparsable: {:?}", event))
+            })?;
+            let cmdid = CommandID::LocalActivity(deets.seq);
+            let mkey = self.get_machine_key(cmdid)?;
+            if let Machines::LocalActivityMachine(lam) = self.machine(mkey) {
+                if lam.marker_should_get_special_handling()? {
+                    self.submachine_handle_event(mkey, event, false)?;
+                    return Ok(());
+                }
+            } else {
+                return Err(WFMachinesError::Fatal(format!(
+                    "Encountered local activity marker but the associated machine was of the \
+                     wrong type! {:?}",
+                    event
+                )));
+            }
+        }
 
         let consumed_cmd = loop {
             if let Some(peek_machine) = self.commands.front() {
@@ -343,19 +429,17 @@ impl WorkflowMachines {
                 )));
             };
 
-            // Feed the machine the event
             let canceled_before_sent = self
                 .machine(command.machine)
                 .was_cancelled_before_sent_to_server();
 
             if !canceled_before_sent {
+                // Feed the machine the event
                 self.submachine_handle_event(command.machine, event, true)?;
                 break command;
             }
         };
 
-        // TODO: validate command
-
         if !self.machine(consumed_cmd.machine).is_final_state() {
             self.machines_by_event_id
                 .insert(event.event_id, consumed_cmd.machine);
@@ -417,7 +501,7 @@ impl WorkflowMachines {
             }
             Some(EventType::WorkflowTaskScheduled) => {
                 let wf_task_sm = WorkflowTaskMachine::new(self.next_started_event_id);
-                let key = self.all_machines.insert(Box::new(wf_task_sm));
+                let key = self.all_machines.insert(wf_task_sm.into());
                 self.submachine_handle_event(key, event, has_next_event)?;
                 self.machines_by_event_id.insert(event.event_id, key);
             }
@@ -461,7 +545,10 @@ impl WorkflowMachines {
             .iter()
             .filter_map(|c| {
                 if !self.machine(c.machine).is_final_state() {
-                    Some(c.command.clone())
+                    match &c.command {
+                        MachineAssociatedCommand::Real(cmd) => Some((**cmd).clone()),
+                        MachineAssociatedCommand::FakeLocalActivityMarker(_) => None,
+                    }
                 } else {
                     None
                 }
@@ -475,9 +562,9 @@ impl WorkflowMachines {
     ///
     /// The job list may be empty, in which case it is expected the caller handles what to do in a
     /// "no work" situation. Possibly, it may know about some work the machines don't, like queries.
-    pub(crate) fn get_wf_activation(&mut self) -> WfActivation {
+    pub(crate) fn get_wf_activation(&mut self) -> WorkflowActivation {
         let jobs = self.drive_me.drain_jobs();
-        WfActivation {
+        WorkflowActivation {
             timestamp: self.current_wf_time.map(Into::into),
             is_replaying: self.replaying,
             run_id: self.run_id.clone(),
@@ -485,6 +572,10 @@ impl WorkflowMachines {
         }
     }
 
+    pub(crate) fn has_pending_jobs(&self) -> bool {
+        self.drive_me.has_pending_jobs()
+    }
+
     fn set_current_time(&mut self, time: SystemTime) -> SystemTime {
         if self.current_wf_time.map_or(true, |t| t < time) {
             self.current_wf_time = Some(time);
@@ -495,14 +586,9 @@ impl WorkflowMachines {
 
     /// Iterate the state machines, which consists of grabbing any pending outgoing commands from
     /// the workflow code, handling them, and preparing them to be sent off to the server.
-    ///
-    /// Returns a boolean flag which indicates whether or not new activations were produced by the
-    /// state machine. If true, pending activation should be created by the caller making jobs
-    /// available to the lang side.
-    pub(crate) async fn iterate_machines(&mut self) -> Result<bool> {
+    pub(crate) async fn iterate_machines(&mut self) -> Result<()> {
         let results = self.drive_me.fetch_workflow_iteration_output().await;
         let jobs = self.handle_driven_results(results)?;
-        let has_new_lang_jobs = !jobs.is_empty();
         for job in jobs {
             self.drive_me.send_job(job);
         }
@@ -512,12 +598,12 @@ impl WorkflowMachines {
                 self.metrics.wf_e2e_latency(rt);
             }
         }
-        Ok(has_new_lang_jobs)
+        Ok(())
     }
 
     /// Apply the next (unapplied) entire workflow task from history to these machines. Will replay
     /// any events that need to be replayed until caught up to the newest WFT.
-    pub(crate) async fn apply_next_wft_from_history(&mut self) -> Result<()> {
+    pub(crate) async fn apply_next_wft_from_history(&mut self) -> Result<usize> {
         // A much higher-up span (ex: poll) may want this field filled
         tracing::Span::current().record("run_id", &self.run_id.as_str());
 
@@ -525,15 +611,21 @@ impl WorkflowMachines {
         // then we don't need to do anything here, and in fact we need to avoid re-applying the
         // final WFT.
         if self.have_seen_terminal_event {
-            return Ok(());
+            return Ok(0);
         }
 
         let last_handled_wft_started_id = self.current_started_event_id;
-        let events = self
-            .last_history_from_server
-            .take_next_wft_sequence(last_handled_wft_started_id)
-            .await
-            .map_err(WFMachinesError::HistoryFetchingError)?;
+        let events = {
+            let mut evts = self
+                .last_history_from_server
+                .take_next_wft_sequence(last_handled_wft_started_id)
+                .await
+                .map_err(WFMachinesError::HistoryFetchingError)?;
+            // Do not re-process events we have already processed
+            evts.retain(|e| e.event_id > self.last_processed_event);
+            evts
+        };
+        let num_events_to_process = events.len();
 
         // We're caught up on reply if there are no new events to process
         // TODO: Probably this is unneeded if we evict whenever history is from non-sticky queue
@@ -564,31 +656,30 @@ impl WorkflowMachines {
 
         while let Some(event) = history.next() {
             let next_event = history.peek();
-
+            self.handle_event(event, next_event.is_some())?;
+            self.last_processed_event = event.event_id;
             if event.event_type == EventType::WorkflowTaskStarted as i32 && next_event.is_none() {
-                self.handle_event(event, false)?;
                 break;
             }
-
-            self.handle_event(event, next_event.is_some())?;
         }
 
         // Scan through to the next WFT, searching for any patch markers, so that we can
         // pre-resolve them.
         for e in self.last_history_from_server.peek_next_wft_sequence() {
-            if let Some((patch_id, deprecated)) = e.get_changed_marker_details() {
+            if let Some((patch_id, _)) = e.get_patch_marker_details() {
                 self.encountered_change_markers.insert(
                     patch_id.clone(),
                     ChangeInfo {
-                        deprecated,
                         created_command: false,
                     },
                 );
                 // Found a patch marker
                 self.drive_me
-                    .send_job(wf_activation_job::Variant::NotifyHasPatch(NotifyHasPatch {
-                        patch_id,
-                    }));
+                    .send_job(workflow_activation_job::Variant::NotifyHasPatch(
+                        NotifyHasPatch { patch_id },
+                    ));
+            } else if e.is_local_activity_marker() {
+                self.local_activity_data.process_peekahead_marker(e)?;
             }
         }
 
@@ -596,7 +687,7 @@ impl WorkflowMachines {
             self.metrics.wf_task_replay_latency(replay_start.elapsed());
         }
 
-        Ok(())
+        Ok(num_events_to_process)
     }
 
     /// Wrapper for calling [TemporalStateMachine::handle_event] which appropriately takes action
@@ -622,10 +713,15 @@ impl WorkflowMachines {
                 .machine(c.machine)
                 .was_cancelled_before_sent_to_server()
             {
-                let machine_responses = self
-                    .machine_mut(c.machine)
-                    .handle_command(c.command.command_type())?;
-                self.process_machine_responses(c.machine, machine_responses)?;
+                match &c.command {
+                    MachineAssociatedCommand::Real(cmd) => {
+                        let machine_responses = self
+                            .machine_mut(c.machine)
+                            .handle_command(cmd.command_type())?;
+                        self.process_machine_responses(c.machine, machine_responses)?;
+                    }
+                    MachineAssociatedCommand::FakeLocalActivityMarker(_) => {}
+                }
                 self.commands.push_back(c);
             }
         }
@@ -637,10 +733,10 @@ impl WorkflowMachines {
     /// this function uses to drive sending jobs to lang, triggering new workflow tasks, etc.
     fn process_machine_responses(
         &mut self,
-        sm: MachineKey,
+        smk: MachineKey,
         machine_responses: Vec<MachineResponse>,
     ) -> Result<()> {
-        let sm = self.machine_mut(sm);
+        let sm = self.machine(smk);
         if !machine_responses.is_empty() {
             debug!(responses = %machine_responses.display(), machine_name = %sm.kind(),
                    "Machine produced responses");
@@ -660,14 +756,40 @@ impl WorkflowMachines {
                     // TODO: Should this also update self.run_id? Should we track orig/current
                     //   separately?
                     self.drive_me
-                        .send_job(wf_activation_job::Variant::UpdateRandomSeed(
+                        .send_job(workflow_activation_job::Variant::UpdateRandomSeed(
                             UpdateRandomSeed {
                                 randomness_seed: str_to_randomness_seed(&new_run_id),
                             },
                         ));
                 }
-                MachineResponse::IssueNewCommand(_) => {
-                    panic!("Issue new command machine response not expected here")
+                MachineResponse::IssueNewCommand(c) => {
+                    self.current_wf_task_commands.push_back(CommandAndMachine {
+                        command: MachineAssociatedCommand::Real(Box::new(c)),
+                        machine: smk,
+                    })
+                }
+                MachineResponse::IssueFakeLocalActivityMarker(seq) => {
+                    self.current_wf_task_commands.push_back(CommandAndMachine {
+                        command: MachineAssociatedCommand::FakeLocalActivityMarker(seq),
+                        machine: smk,
+                    });
+                }
+                MachineResponse::QueueLocalActivity(act) => {
+                    self.local_activity_data.enqueue(act);
+                }
+                MachineResponse::RequestCancelLocalActivity(_) => {
+                    panic!(
+                        "Request cancel local activity should not be returned from \
+                         anything other than explicit cancellation"
+                    )
+                }
+                MachineResponse::AbandonLocalActivity(seq) => {
+                    self.local_activity_data.done_executing(seq);
+                }
+                MachineResponse::UpdateWFTime(t) => {
+                    if let Some(t) = t {
+                        self.set_current_time(t);
+                    }
                 }
             }
         }
@@ -683,30 +805,53 @@ impl WorkflowMachines {
     fn handle_driven_results(
         &mut self,
         results: Vec<WFCommand>,
-    ) -> Result<Vec<wf_activation_job::Variant>> {
+    ) -> Result<Vec<workflow_activation_job::Variant>> {
         let mut jobs = vec![];
         for cmd in results {
             match cmd {
                 WFCommand::AddTimer(attrs) => {
                     let seq = attrs.seq;
-                    let timer = self.add_new_command_machine(new_timer(attrs));
-                    self.id_to_machine
-                        .insert(CommandID::Timer(seq), timer.machine);
-                    self.current_wf_task_commands.push_back(timer);
+                    self.add_cmd_to_wf_task(new_timer(attrs), Some(CommandID::Timer(seq)));
                 }
                 WFCommand::CancelTimer(attrs) => {
                     jobs.extend(self.process_cancellation(CommandID::Timer(attrs.seq))?);
                 }
                 WFCommand::AddActivity(attrs) => {
                     let seq = attrs.seq;
-                    let activity = self.add_new_command_machine(new_activity(attrs));
+                    self.add_cmd_to_wf_task(new_activity(attrs), Some(CommandID::Activity(seq)));
+                }
+                WFCommand::AddLocalActivity(attrs) => {
+                    let seq = attrs.seq;
+                    let attrs: ValidScheduleLA = ValidScheduleLA::from_schedule_la(
+                        attrs,
+                        self.started_attrs()
+                            .as_ref()
+                            .map(|x| x.workflow_execution_timeout.clone().try_into_or_none())
+                            .flatten(),
+                    )
+                    .map_err(|e| {
+                        WFMachinesError::Fatal(format!(
+                            "Invalid schedule local activity request (seq {}): {}",
+                            seq, e
+                        ))
+                    })?;
+                    let (la, mach_resp) = new_local_activity(
+                        attrs,
+                        self.replaying,
+                        self.local_activity_data.take_preresolution(seq),
+                        self.current_wf_time,
+                    )?;
+                    let machkey = self.all_machines.insert(la.into());
                     self.id_to_machine
-                        .insert(CommandID::Activity(seq), activity.machine);
-                    self.current_wf_task_commands.push_back(activity);
+                        .insert(CommandID::LocalActivity(seq), machkey);
+                    self.process_machine_responses(machkey, mach_resp)?;
                 }
                 WFCommand::RequestCancelActivity(attrs) => {
                     jobs.extend(self.process_cancellation(CommandID::Activity(attrs.seq))?);
                 }
+                WFCommand::RequestCancelLocalActivity(attrs) => {
+                    jobs.extend(self.process_cancellation(CommandID::LocalActivity(attrs.seq))?);
+                }
                 WFCommand::CompleteWorkflow(attrs) => {
                     self.metrics.wf_completed();
                     self.add_terminal_command(complete_workflow(attrs));
@@ -727,15 +872,12 @@ impl WorkflowMachines {
                     // Do not create commands for change IDs that we have already created commands
                     // for.
                     if !matches!(self.encountered_change_markers.get(&attrs.patch_id),
-                                Some(ChangeInfo {created_command, ..})
-                                    if *created_command)
+                                 Some(ChangeInfo {created_command}) if *created_command)
                     {
-                        let verm = self.add_new_command_machine(has_change(
-                            attrs.patch_id.clone(),
-                            self.replaying,
-                            attrs.deprecated,
-                        ));
-                        self.current_wf_task_commands.push_back(verm);
+                        self.add_cmd_to_wf_task(
+                            has_change(attrs.patch_id.clone(), self.replaying, attrs.deprecated),
+                            None,
+                        );
 
                         if let Some(ci) = self.encountered_change_markers.get_mut(&attrs.patch_id) {
                             ci.created_command = true;
@@ -743,7 +885,6 @@ impl WorkflowMachines {
                             self.encountered_change_markers.insert(
                                 attrs.patch_id,
                                 ChangeInfo {
-                                    deprecated: attrs.deprecated,
                                     created_command: true,
                                 },
                             );
@@ -752,10 +893,10 @@ impl WorkflowMachines {
                 }
                 WFCommand::AddChildWorkflow(attrs) => {
                     let seq = attrs.seq;
-                    let child_workflow = self.add_new_command_machine(new_child_workflow(attrs));
-                    self.id_to_machine
-                        .insert(CommandID::ChildWorkflowStart(seq), child_workflow.machine);
-                    self.current_wf_task_commands.push_back(child_workflow);
+                    self.add_cmd_to_wf_task(
+                        new_child_workflow(attrs),
+                        Some(CommandID::ChildWorkflowStart(seq)),
+                    );
                 }
                 WFCommand::CancelUnstartedChild(attrs) => jobs.extend(self.process_cancellation(
                     CommandID::ChildWorkflowStart(attrs.child_workflow_seq),
@@ -778,11 +919,10 @@ impl WorkflowMachines {
                         ),
                         Some(cancel_we::Target::WorkflowExecution(we)) => (we, false),
                     };
-                    let mach = self
-                        .add_new_command_machine(new_external_cancel(attrs.seq, we, only_child));
-                    self.id_to_machine
-                        .insert(CommandID::CancelExternal(attrs.seq), mach.machine);
-                    self.current_wf_task_commands.push_back(mach);
+                    self.add_cmd_to_wf_task(
+                        new_external_cancel(attrs.seq, we, only_child),
+                        Some(CommandID::CancelExternal(attrs.seq)),
+                    );
                 }
                 WFCommand::SignalExternalWorkflow(attrs) => {
                     let (we, only_child) = match attrs.target {
@@ -803,16 +943,16 @@ impl WorkflowMachines {
                         Some(sig_we::Target::WorkflowExecution(we)) => (we, false),
                     };
 
-                    let sigm = self.add_new_command_machine(new_external_signal(
-                        attrs.seq,
-                        we,
-                        attrs.signal_name,
-                        attrs.args,
-                        only_child,
-                    ));
-                    self.id_to_machine
-                        .insert(CommandID::SignalExternal(attrs.seq), sigm.machine);
-                    self.current_wf_task_commands.push_back(sigm);
+                    self.add_cmd_to_wf_task(
+                        new_external_signal(
+                            attrs.seq,
+                            we,
+                            attrs.signal_name,
+                            attrs.args,
+                            only_child,
+                        ),
+                        Some(CommandID::SignalExternal(attrs.seq)),
+                    );
                 }
                 WFCommand::CancelSignalWorkflow(attrs) => {
                     jobs.extend(self.process_cancellation(CommandID::SignalExternal(attrs.seq))?);
@@ -832,19 +972,62 @@ impl WorkflowMachines {
     fn process_cancellation(&mut self, id: CommandID) -> Result<Vec<Variant>> {
         let mut jobs = vec![];
         let m_key = self.get_machine_key(id)?;
-        let res = self.machine_mut(m_key).cancel()?;
-        debug!(machine_responses = ?res, cmd_id = ?id, "Cancel request responses");
-        for r in res {
+        let machine_resps = self.machine_mut(m_key).cancel()?;
+        debug!(machine_responses = ?machine_resps, cmd_id = ?id, "Cancel request responses");
+        for r in machine_resps {
             match r {
                 MachineResponse::IssueNewCommand(c) => {
                     self.current_wf_task_commands.push_back(CommandAndMachine {
-                        command: c,
+                        command: MachineAssociatedCommand::Real(Box::new(c)),
                         machine: m_key,
                     });
                 }
                 MachineResponse::PushWFJob(j) => {
                     jobs.push(j);
                 }
+                MachineResponse::RequestCancelLocalActivity(seq) => {
+                    // We might already know about the status from a pre-resolution. Apply it if so.
+                    // We need to do this because otherwise we might need to perform additional
+                    // activations during replay that didn't happen during execution, just like
+                    // we sometimes pre-resolve activities when first requested.
+                    if let Some(preres) = self.local_activity_data.take_preresolution(seq) {
+                        if let Machines::LocalActivityMachine(lam) = self.machine_mut(m_key) {
+                            let more_responses = lam.try_resolve_with_dat(preres)?;
+                            self.process_machine_responses(m_key, more_responses)?;
+                        } else {
+                            panic!("A non local-activity machine returned a request cancel LA response");
+                        }
+                    }
+                    // If it's in the request queue, just rip it out.
+                    else if let Some(removed_act) =
+                        self.local_activity_data.remove_from_queue(seq)
+                    {
+                        // We removed it. Notify the machine that the activity cancelled.
+                        if let Machines::LocalActivityMachine(lam) = self.machine_mut(m_key) {
+                            let more_responses = lam.try_resolve(
+                                LocalActivityExecutionResult::empty_cancel(),
+                                Duration::from_secs(0),
+                                removed_act.attempt,
+                                None,
+                                None,
+                            )?;
+                            self.process_machine_responses(m_key, more_responses)?;
+                        } else {
+                            panic!("A non local-activity machine returned a request cancel LA response");
+                        }
+                    } else {
+                        // Finally, if we know about the LA at all, it's currently running, so
+                        // queue the cancel request to be given to the LA manager.
+                        self.local_activity_data.enqueue_cancel(ExecutingLAId {
+                            run_id: self.run_id.clone(),
+                            seq_num: seq,
+                        });
+                    }
+                }
+                MachineResponse::AbandonLocalActivity(seq) => {
+                    self.local_activity_data.done_executing(seq);
+                }
+                MachineResponse::UpdateWFTime(None) => {}
                 v => {
                     return Err(WFMachinesError::Fatal(format!(
                         "Unexpected machine response {:?} when cancelling {:?}",
@@ -862,34 +1045,37 @@ impl WorkflowMachines {
         })?)
     }
 
-    fn add_terminal_command<T: TemporalStateMachine + 'static>(
-        &mut self,
-        machine: NewMachineWithCommand<T>,
-    ) {
+    fn add_terminal_command(&mut self, machine: NewMachineWithCommand) {
         let cwfm = self.add_new_command_machine(machine);
         self.workflow_end_time = Some(SystemTime::now());
         self.current_wf_task_commands.push_back(cwfm);
     }
 
-    fn add_new_command_machine<T: TemporalStateMachine + 'static>(
-        &mut self,
-        machine: NewMachineWithCommand<T>,
-    ) -> CommandAndMachine {
-        let k = self.all_machines.insert(Box::new(machine.machine));
+    /// Add a new command/machines for that command to the current workflow task
+    fn add_cmd_to_wf_task(&mut self, machine: NewMachineWithCommand, id: Option<CommandID>) {
+        let mach = self.add_new_command_machine(machine);
+        if let Some(id) = id {
+            self.id_to_machine.insert(id, mach.machine);
+        }
+        self.current_wf_task_commands.push_back(mach);
+    }
+
+    fn add_new_command_machine(&mut self, machine: NewMachineWithCommand) -> CommandAndMachine {
+        let k = self.all_machines.insert(machine.machine);
         CommandAndMachine {
-            command: machine.command,
+            command: MachineAssociatedCommand::Real(Box::new(machine.command)),
             machine: k,
         }
     }
 
-    fn machine(&self, m: MachineKey) -> &dyn TemporalStateMachine {
+    fn machine(&self, m: MachineKey) -> &Machines {
         self.all_machines
             .get(m)
             .expect("Machine must exist")
             .borrow()
     }
 
-    fn machine_mut(&mut self, m: MachineKey) -> &mut (dyn TemporalStateMachine + 'static) {
+    fn machine_mut(&mut self, m: MachineKey) -> &mut Machines {
         self.all_machines
             .get_mut(m)
             .expect("Machine must exist")
@@ -917,23 +1103,23 @@ fn change_marker_handling(
 ) -> Result<ChangeMarkerOutcome> {
     if !mach.matches_event(event) {
         // Version markers can be skipped in the event they are deprecated
-        if let Some(changed_info) = event.get_changed_marker_details() {
+        if let Some((patch_name, deprecated)) = event.get_patch_marker_details() {
             // Is deprecated. We can simply ignore this event, as deprecated change
             // markers are allowed without matching changed calls.
-            if changed_info.1 {
+            if deprecated {
                 debug!("Deprecated patch marker tried against wrong machine, skipping.");
                 return Ok(ChangeMarkerOutcome::SkipEvent);
             }
             return Err(WFMachinesError::Nondeterminism(format!(
                 "Non-deprecated patch marker encountered for change {}, \
                             but there is no corresponding change command!",
-                changed_info.0
+                patch_name
             )));
         }
         // Version machines themselves may also not *have* matching markers, where non-deprecated
         // calls take the old path, and deprecated calls assume history is produced by a new-code
         // worker.
-        if mach.kind() == MachineKind::Version {
+        if mach.kind() == MachineKind::Patch {
             debug!("Skipping non-matching event against version machine");
             return Ok(ChangeMarkerOutcome::SkipCommand);
         }