temporalio 0.0.1 → 0.1.0

data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs

@@ -1,26 +1,32 @@
 mod local_acts;
 
-pub(crate) use temporal_sdk_core_api::errors::WFMachinesError;
-
 use super::{
-    activity_state_machine::new_activity, cancel_external_state_machine::new_external_cancel,
+    cancel_external_state_machine::new_external_cancel,
     cancel_workflow_state_machine::cancel_workflow,
-    child_workflow_state_machine::new_child_workflow,
     complete_workflow_state_machine::complete_workflow,
     continue_as_new_workflow_state_machine::continue_as_new,
     fail_workflow_state_machine::fail_workflow, local_activity_state_machine::new_local_activity,
     patch_state_machine::has_change, signal_external_state_machine::new_external_signal,
     timer_state_machine::new_timer, upsert_search_attributes_state_machine::upsert_search_attrs,
     workflow_machines::local_acts::LocalActivityData,
-    workflow_task_state_machine::WorkflowTaskMachine, MachineKind, Machines, NewMachineWithCommand,
+    workflow_task_state_machine::WorkflowTaskMachine, Machines, NewMachineWithCommand,
     TemporalStateMachine,
 };
 use crate::{
+    internal_flags::InternalFlags,
     protosext::{HistoryEventExt, ValidScheduleLA},
     telemetry::{metrics::MetricsContext, VecDisplayer},
     worker::{
         workflow::{
-            CommandID, DrivenWorkflow, HistoryUpdate, LocalResolution, WFCommand, WorkflowFetcher,
+            history_update::NextWFT,
+            machines::{
+                activity_state_machine::ActivityMachine,
+                child_workflow_state_machine::ChildWorkflowMachine,
+                modify_workflow_properties_state_machine::modify_workflow_properties,
+                HistEventData,
+            },
+            CommandID, DrivenWorkflow, HistoryUpdate, InternalFlagsRef, LocalResolution,
+            OutgoingJob, RunBasics, WFCommand, WFMachinesError, WorkflowFetcher,
             WorkflowStartedInfo,
         },
         ExecutingLAId, LocalActRequest, LocalActivityExecutionResult, LocalActivityResolution,
@@ -30,14 +36,17 @@ use siphasher::sip::SipHasher13;
 use slotmap::{SlotMap, SparseSecondaryMap};
 use std::{
     borrow::{Borrow, BorrowMut},
+    cell::RefCell,
     collections::{HashMap, VecDeque},
     convert::TryInto,
     hash::{Hash, Hasher},
+    rc::Rc,
     time::{Duration, Instant, SystemTime},
 };
 use temporal_sdk_core_protos::{
     coresdk::{
         common::NamespacedWorkflowExecution,
+        workflow_activation,
         workflow_activation::{
             workflow_activation_job, NotifyHasPatch, UpdateRandomSeed, WorkflowActivation,
         },
@@ -49,6 +58,7 @@ use temporal_sdk_core_protos::{
         command::v1::{command::Attributes as ProtoCmdAttrs, Command as ProtoCommand},
         enums::v1::EventType,
         history::v1::{history_event, HistoryEvent},
+        sdk::v1::WorkflowTaskCompletedMetadata,
     },
 };
 
@@ -93,6 +103,9 @@ pub(crate) struct WorkflowMachines {
     /// The current workflow time if it has been established. This may differ from the WFT start
     /// time since local activities may advance the clock
     current_wf_time: Option<SystemTime>,
+    /// The internal flags which have been seen so far during this run's execution and thus are
+    /// usable during replay.
+    observed_internal_flags: InternalFlagsRef,
 
     all_machines: SlotMap<MachineKey, Machines>,
     /// If a machine key is in this map, that machine was created internally by core, not as a
@@ -131,7 +144,7 @@ pub(crate) struct WorkflowMachines {
 }
 
 #[derive(Debug, derive_more::Display)]
-#[display(fmt = "Cmd&Machine({})", "command")]
+#[display(fmt = "Cmd&Machine({command})")]
 struct CommandAndMachine {
     command: MachineAssociatedCommand,
     machine: MachineKey,
@@ -140,7 +153,7 @@ struct CommandAndMachine {
 #[derive(Debug, derive_more::Display)]
 enum MachineAssociatedCommand {
     Real(Box<ProtoCommand>),
-    #[display(fmt = "FakeLocalActivityMarker({})", "_0")]
+    #[display(fmt = "FakeLocalActivityMarker({_0})")]
     FakeLocalActivityMarker(u32),
 }
 
@@ -153,9 +166,9 @@ struct ChangeInfo {
 #[derive(Debug, derive_more::Display)]
 #[must_use]
 #[allow(clippy::large_enum_variant)]
-pub enum MachineResponse {
-    #[display(fmt = "PushWFJob({})", "_0")]
-    PushWFJob(workflow_activation_job::Variant),
+pub(super) enum MachineResponse {
+    #[display(fmt = "PushWFJob({_0})")]
+    PushWFJob(OutgoingJob),
 
     /// Pushes a new command into the list that will be sent to server once we respond with the
     /// workflow task completion
@@ -163,31 +176,31 @@ pub enum MachineResponse {
     /// The machine requests the creation of another *different* machine. This acts as if lang
     /// had replied to the activation with a command, but we use a special set of IDs to avoid
     /// collisions.
-    #[display(fmt = "NewCoreOriginatedCommand({:?})", "_0")]
+    #[display(fmt = "NewCoreOriginatedCommand({_0:?})")]
     NewCoreOriginatedCommand(ProtoCmdAttrs),
-    #[display(fmt = "IssueFakeLocalActivityMarker({})", "_0")]
+    #[display(fmt = "IssueFakeLocalActivityMarker({_0})")]
     IssueFakeLocalActivityMarker(u32),
     #[display(fmt = "TriggerWFTaskStarted")]
     TriggerWFTaskStarted {
         task_started_event_id: i64,
         time: SystemTime,
     },
-    #[display(fmt = "UpdateRunIdOnWorkflowReset({})", run_id)]
+    #[display(fmt = "UpdateRunIdOnWorkflowReset({run_id})")]
     UpdateRunIdOnWorkflowReset { run_id: String },
 
     /// Queue a local activity to be processed by the worker
     #[display(fmt = "QueueLocalActivity")]
     QueueLocalActivity(ValidScheduleLA),
     /// Request cancellation of an executing local activity
-    #[display(fmt = "RequestCancelLocalActivity({})", "_0")]
+    #[display(fmt = "RequestCancelLocalActivity({_0})")]
     RequestCancelLocalActivity(u32),
     /// Indicates we are abandoning the indicated LA, so we can remove it from "outstanding" LAs
     /// and we will not try to WFT heartbeat because of it.
-    #[display(fmt = "AbandonLocalActivity({:?})", "_0")]
+    #[display(fmt = "AbandonLocalActivity({_0:?})")]
     AbandonLocalActivity(u32),
 
     /// Set the workflow time to the provided time
-    #[display(fmt = "UpdateWFTime({:?})", "_0")]
+    #[display(fmt = "UpdateWFTime({_0:?})")]
     UpdateWFTime(Option<SystemTime>),
 }
 
@@ -196,30 +209,27 @@ where
     T: Into<workflow_activation_job::Variant>,
 {
     fn from(v: T) -> Self {
-        Self::PushWFJob(v.into())
+        Self::PushWFJob(v.into().into())
     }
 }
 
 impl WorkflowMachines {
-    pub(crate) fn new(
-        namespace: String,
-        workflow_id: String,
-        workflow_type: String,
-        run_id: String,
-        history: HistoryUpdate,
-        driven_wf: DrivenWorkflow,
-        metrics: MetricsContext,
-    ) -> Self {
-        let replaying = history.previous_started_event_id > 0;
+    pub(crate) fn new(basics: RunBasics, driven_wf: DrivenWorkflow) -> Self {
+        let replaying = basics.history.previous_wft_started_id > 0;
+        let mut observed_internal_flags = InternalFlags::new(basics.capabilities);
+        // Peek ahead to determine used patches in the first WFT.
+        if let Some(attrs) = basics.history.peek_next_wft_completed(0) {
+            observed_internal_flags.add_from_complete(attrs);
+        };
         Self {
-            last_history_from_server: history,
-            namespace,
-            workflow_id,
-            workflow_type,
-            run_id,
+            last_history_from_server: basics.history,
+            namespace: basics.namespace,
+            workflow_id: basics.workflow_id,
+            workflow_type: basics.workflow_type,
+            run_id: basics.run_id,
             drive_me: driven_wf,
             replaying,
-            metrics,
+            metrics: basics.metrics,
             // In an ideal world one could say ..Default::default() here and it'd still work.
             current_started_event_id: 0,
             next_started_event_id: 0,
@@ -228,6 +238,7 @@ impl WorkflowMachines {
             workflow_end_time: None,
             wft_start_time: None,
             current_wf_time: None,
+            observed_internal_flags: Rc::new(RefCell::new(observed_internal_flags)),
             all_machines: Default::default(),
             machine_is_core_created: Default::default(),
             machines_by_event_id: Default::default(),
@@ -253,10 +264,10 @@ impl WorkflowMachines {
             .and_then(|(st, et)| et.duration_since(st).ok())
     }
 
-    pub(crate) async fn new_history_from_server(&mut self, update: HistoryUpdate) -> Result<()> {
+    pub(crate) fn new_history_from_server(&mut self, update: HistoryUpdate) -> Result<()> {
         self.last_history_from_server = update;
-        self.replaying = self.last_history_from_server.previous_started_event_id > 0;
-        self.apply_next_wft_from_history().await?;
+        self.replaying = self.last_history_from_server.previous_wft_started_id > 0;
+        self.apply_next_wft_from_history()?;
         Ok(())
     }
 
@@ -288,9 +299,8 @@ impl WorkflowMachines {
                     self.process_machine_responses(mk, resps)?;
                 } else {
                     return Err(WFMachinesError::Nondeterminism(format!(
-                        "Command matching activity with seq num {} existed but was not a \
-                        local activity!",
-                        seq
+                        "Command matching activity with seq num {seq} existed but was not a \
+                        local activity!"
                     )));
                 }
                 self.local_activity_data.done_executing(seq);
@@ -315,6 +325,237 @@ impl WorkflowMachines {
         self.drive_me.get_started_info()
     }
 
+    /// Fetches commands which are ready for processing from the state machines, generally to be
+    /// sent off to the server. They are not removed from the internal queue, that happens when
+    /// corresponding history events from the server are being handled.
+    pub(crate) fn get_commands(&self) -> Vec<ProtoCommand> {
+        self.commands
+            .iter()
+            .filter_map(|c| {
+                if !self.machine(c.machine).is_final_state() {
+                    match &c.command {
+                        MachineAssociatedCommand::Real(cmd) => Some((**cmd).clone()),
+                        MachineAssociatedCommand::FakeLocalActivityMarker(_) => None,
+                    }
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+
+    /// Returns the next activation that needs to be performed by the lang sdk. Things like unblock
+    /// timer, etc. This does *not* cause any advancement of the state machines, it merely drains
+    /// from the outgoing queue of activation jobs.
+    ///
+    /// The job list may be empty, in which case it is expected the caller handles what to do in a
+    /// "no work" situation. Possibly, it may know about some work the machines don't, like queries.
+    pub(crate) fn get_wf_activation(&mut self) -> WorkflowActivation {
+        let jobs = self.drive_me.drain_jobs();
+        WorkflowActivation {
+            timestamp: self.current_wf_time.map(Into::into),
+            is_replaying: self.replaying,
+            run_id: self.run_id.clone(),
+            history_length: self.last_processed_event as u32,
+            jobs,
+            available_internal_flags: (*self.observed_internal_flags)
+                .borrow()
+                .all_lang()
+                .iter()
+                .copied()
+                .collect(),
+        }
+    }
+
+    pub(crate) fn has_pending_jobs(&self) -> bool {
+        !self.drive_me.peek_pending_jobs().is_empty()
+    }
+
+    pub(crate) fn has_pending_la_resolutions(&self) -> bool {
+        self.drive_me
+            .peek_pending_jobs()
+            .iter()
+            .any(|v| v.is_la_resolution)
+    }
+
+    pub(crate) fn get_metadata_for_wft_complete(&self) -> WorkflowTaskCompletedMetadata {
+        (*self.observed_internal_flags)
+            .borrow_mut()
+            .gather_for_wft_complete()
+    }
+
+    pub(crate) fn add_lang_used_flags(&self, flags: Vec<u32>) {
+        (*self.observed_internal_flags)
+            .borrow_mut()
+            .add_lang_used(flags);
+    }
+
+    /// Iterate the state machines, which consists of grabbing any pending outgoing commands from
+    /// the workflow code, handling them, and preparing them to be sent off to the server.
+    pub(crate) fn iterate_machines(&mut self) -> Result<()> {
+        let results = self.drive_me.fetch_workflow_iteration_output();
+        self.handle_driven_results(results)?;
+        self.prepare_commands()?;
+        if self.workflow_is_finished() {
+            if let Some(rt) = self.total_runtime() {
+                self.metrics.wf_e2e_latency(rt);
+            }
+        }
+        Ok(())
+    }
+
+    /// Returns true if machines are ready to apply the next WFT sequence, false if events will need
+    /// to be fetched in order to create a complete update with the entire next WFT sequence.
+    pub(crate) fn ready_to_apply_next_wft(&self) -> bool {
+        self.last_history_from_server
+            .can_take_next_wft_sequence(self.current_started_event_id)
+    }
+
+    /// Apply the next (unapplied) entire workflow task from history to these machines. Will replay
+    /// any events that need to be replayed until caught up to the newest WFT.
+    pub(crate) fn apply_next_wft_from_history(&mut self) -> Result<usize> {
+        // If we have already seen the terminal event for the entire workflow in a previous WFT,
+        // then we don't need to do anything here, and in fact we need to avoid re-applying the
+        // final WFT.
+        if self.have_seen_terminal_event {
+            return Ok(0);
+        }
+
+        // Update observed patches with any that were used in the task
+        if let Some(next_complete) = self
+            .last_history_from_server
+            .peek_next_wft_completed(self.last_processed_event)
+        {
+            (*self.observed_internal_flags)
+                .borrow_mut()
+                .add_from_complete(next_complete);
+        }
+
+        let last_handled_wft_started_id = self.current_started_event_id;
+        let (events, has_final_event) = match self
+            .last_history_from_server
+            .take_next_wft_sequence(last_handled_wft_started_id)
+        {
+            NextWFT::ReplayOver => (vec![], true),
+            NextWFT::WFT(mut evts, has_final_event) => {
+                // Do not re-process events we have already processed
+                evts.retain(|e| e.event_id > self.last_processed_event);
+                (evts, has_final_event)
+            }
+            NextWFT::NeedFetch => {
+                return Err(WFMachinesError::Fatal(
+                    "Need to fetch history events to continue applying workflow task, but this \
+                     should be prevented ahead of time! This is a Core SDK bug."
+                        .to_string(),
+                ));
+            }
+        };
+        let num_events_to_process = events.len();
+
+        // We're caught up on reply if there are no new events to process
+        if events.is_empty() {
+            self.replaying = false;
+        }
+        let replay_start = Instant::now();
+
+        if let Some(last_event) = events.last() {
+            if last_event.event_type == EventType::WorkflowTaskStarted as i32 {
+                self.next_started_event_id = last_event.event_id;
+            }
+        }
+
+        let mut saw_completed = false;
+        let mut history = events.into_iter().peekable();
+        while let Some(event) = history.next() {
+            if event.event_id != self.last_processed_event + 1 {
+                return Err(WFMachinesError::Fatal(format!(
+                    "History is out of order. Last processed event: {}, event id: {}",
+                    self.last_processed_event, event.event_id
+                )));
+            }
+            let next_event = history.peek();
+            let eid = event.event_id;
+
+            // This definition of replaying here is that we are no longer replaying as soon as we
+            // see new events that have never been seen or produced by the SDK.
+            //
+            // Specifically, replay ends once we have seen the last command-event which was produced
+            // as a result of the last completed WFT. Thus, replay would be false for things like
+            // signals which were received and after the last completion, and thus generated the
+            // current WFT being handled.
+            if self.replaying && has_final_event && saw_completed && !event.is_command_event() {
+                // Replay is finished
+                self.replaying = false;
+            }
+            if event.event_type() == EventType::WorkflowTaskCompleted {
+                saw_completed = true;
+            }
+
+            self.handle_event(
+                HistEventData {
+                    event,
+                    replaying: self.replaying,
+                    current_task_is_last_in_history: has_final_event,
+                },
+                next_event.is_some() || !has_final_event,
+            )?;
+            self.last_processed_event = eid;
+        }
+
+        // Scan through to the next WFT, searching for any patch / la markers, so that we can
+        // pre-resolve them.
+        let mut wake_las = vec![];
+        for e in self
+            .last_history_from_server
+            .peek_next_wft_sequence(last_handled_wft_started_id)
+        {
+            if let Some((patch_id, _)) = e.get_patch_marker_details() {
+                self.encountered_change_markers.insert(
+                    patch_id.clone(),
+                    ChangeInfo {
+                        created_command: false,
+                    },
+                );
+                // Found a patch marker
+                self.drive_me.send_job(
+                    workflow_activation_job::Variant::NotifyHasPatch(NotifyHasPatch { patch_id })
+                        .into(),
+                );
+            } else if e.is_local_activity_marker() {
+                if let Some(la_dat) = e.clone().into_local_activity_marker_details() {
+                    if let Ok(mk) =
+                        self.get_machine_key(CommandID::LocalActivity(la_dat.marker_dat.seq))
+                    {
+                        wake_las.push((mk, la_dat));
+                    } else {
+                        self.local_activity_data.insert_peeked_marker(la_dat);
+                    }
+                } else {
+                    return Err(WFMachinesError::Fatal(format!(
+                        "Local activity marker was unparsable: {e:?}"
+                    )));
+                }
+            }
+        }
+        for (mk, la_dat) in wake_las {
+            let mach = self.machine_mut(mk);
+            if let Machines::LocalActivityMachine(ref mut lam) = *mach {
+                if lam.will_accept_resolve_marker() {
+                    let resps = lam.try_resolve_with_dat(la_dat.into())?;
+                    self.process_machine_responses(mk, resps)?;
+                } else {
+                    self.local_activity_data.insert_peeked_marker(la_dat);
+                }
+            }
+        }
+
+        if !self.replaying {
+            self.metrics.wf_task_replay_latency(replay_start.elapsed());
+        }
+
+        Ok(num_events_to_process)
+    }
+
     /// Handle a single event from the workflow history. `has_next_event` should be false if `event`
     /// is the last event in the history.
     ///
@@ -323,14 +564,9 @@ impl WorkflowMachines {
     /// event is applied to the machine, which may also return a nondeterminism error if the machine
     /// does not match the expected type. A fatal error may be returned if the machine is in an
     /// invalid state.
-    #[instrument(level = "debug", skip(self, event), fields(event=%event))]
-    fn handle_event(&mut self, event: HistoryEvent, has_next_event: bool) -> Result<()> {
-        if event.event_type() == EventType::Unspecified {
-            return Err(WFMachinesError::Fatal(format!(
-                "Event type is unspecified! This history is invalid. Event detail: {:?}",
-                event
-            )));
-        }
+    #[instrument(skip(self, event_dat), fields(event=%event_dat))]
+    fn handle_event(&mut self, event_dat: HistEventData, has_next_event: bool) -> Result<()> {
+        let event = &event_dat.event;
         if event.is_final_wf_execution_event() {
             self.have_seen_terminal_event = true;
         }
@@ -348,69 +584,45 @@ impl WorkflowMachines {
                 Ok(())
             };
         }
+        if event.event_type() == EventType::Unspecified || event.attributes.is_none() {
+            return if !event.worker_may_ignore {
+                Err(WFMachinesError::Fatal(format!(
+                    "Event type is unspecified! This history is invalid. Event detail: {event:?}"
+                )))
+            } else {
+                debug!("Event is ignorable");
+                Ok(())
+            };
+        }
 
         if event.is_command_event() {
-            self.handle_command_event(event)?;
+            self.handle_command_event(event_dat)?;
             return Ok(());
         }
-        if self.replaying
-            && self.current_started_event_id
-                >= self.last_history_from_server.previous_started_event_id
-            && event.event_type() != EventType::WorkflowTaskCompleted
-        {
-            // Replay is finished
-            self.replaying = false;
-        }
 
-        match event.get_initial_command_event_id() {
-            Some(initial_cmd_id) => {
-                // We remove the machine while we it handles events, then return it, to avoid
-                // borrowing from ourself mutably.
-                let maybe_machine = self.machines_by_event_id.remove(&initial_cmd_id);
-                match maybe_machine {
-                    Some(sm) => {
-                        self.submachine_handle_event(sm, event, has_next_event)?;
-                        // Restore machine if not in it's final state
-                        if !self.machine(sm).is_final_state() {
-                            self.machines_by_event_id.insert(initial_cmd_id, sm);
-                        }
-                    }
-                    None => {
-                        return Err(WFMachinesError::Nondeterminism(format!(
-                            "During event handling, this event had an initial command ID but we \
-                            could not find a matching command for it: {:?}",
-                            event
-                        )));
+        if let Some(initial_cmd_id) = event.get_initial_command_event_id() {
+            // We remove the machine while we it handles events, then return it, to avoid
+            // borrowing from ourself mutably.
+            let maybe_machine = self.machines_by_event_id.remove(&initial_cmd_id);
+            match maybe_machine {
+                Some(sm) => {
+                    self.submachine_handle_event(sm, event_dat)?;
+                    // Restore machine if not in it's final state
+                    if !self.machine(sm).is_final_state() {
+                        self.machines_by_event_id.insert(initial_cmd_id, sm);
                     }
                 }
-            }
-            None => self.handle_non_stateful_event(event, has_next_event)?,
-        }
-
-        Ok(())
-    }
-
-    /// Called when a workflow task started event has triggered. Ensures we are tracking the ID
-    /// of the current started event as well as workflow time properly.
-    fn task_started(&mut self, task_started_event_id: i64, time: SystemTime) -> Result<()> {
-        self.current_started_event_id = task_started_event_id;
-        self.wft_start_time = Some(time);
-        self.set_current_time(time);
-
-        // Notify local activity machines that we started a non-replay WFT, which will allow any
-        // which were waiting for a marker to instead decide to execute the LA since it clearly
-        // will not be resolved via marker.
-        if !self.replaying {
-            let mut resps = vec![];
-            for (k, mach) in self.all_machines.iter_mut() {
-                if let Machines::LocalActivityMachine(lam) = mach {
-                    resps.push((k, lam.encountered_non_replay_wft()?));
+                None => {
+                    return Err(WFMachinesError::Nondeterminism(format!(
+                        "During event handling, this event had an initial command ID but we \
+                            could not find a matching command for it: {event:?}"
+                    )));
                 }
             }
-            for (mkey, resp_set) in resps {
-                self.process_machine_responses(mkey, resp_set)?;
-            }
+        } else {
+            self.handle_non_stateful_event(event_dat)?;
         }
+
         Ok(())
     }
 
@@ -423,23 +635,23 @@ impl WorkflowMachines {
     /// The handling consists of verifying that the next command in the commands queue is associated
     /// with a state machine, which is then notified about the event and the command is removed from
     /// the commands queue.
-    fn handle_command_event(&mut self, event: HistoryEvent) -> Result<()> {
+    fn handle_command_event(&mut self, event_dat: HistEventData) -> Result<()> {
+        let event = &event_dat.event;
         if event.is_local_activity_marker() {
             let deets = event.extract_local_activity_marker_data().ok_or_else(|| {
-                WFMachinesError::Fatal(format!("Local activity marker was unparsable: {:?}", event))
+                WFMachinesError::Fatal(format!("Local activity marker was unparsable: {event:?}"))
             })?;
             let cmdid = CommandID::LocalActivity(deets.seq);
             let mkey = self.get_machine_key(cmdid)?;
             if let Machines::LocalActivityMachine(lam) = self.machine(mkey) {
                 if lam.marker_should_get_special_handling()? {
-                    self.submachine_handle_event(mkey, event, false)?;
+                    self.submachine_handle_event(mkey, event_dat)?;
                     return Ok(());
                 }
             } else {
                 return Err(WFMachinesError::Fatal(format!(
                     "Encountered local activity marker but the associated machine was of the \
-                     wrong type! {:?}",
-                    event
+                     wrong type! {event:?}"
                 )));
             }
         }
@@ -449,7 +661,7 @@ impl WorkflowMachines {
         let consumed_cmd = loop {
             if let Some(peek_machine) = self.commands.front() {
                 let mach = self.machine(peek_machine.machine);
-                match change_marker_handling(&event, mach)? {
+                match change_marker_handling(event, mach)? {
                     ChangeMarkerOutcome::SkipEvent => return Ok(()),
                     ChangeMarkerOutcome::SkipCommand => {
                         self.commands.pop_front();
@@ -464,8 +676,7 @@ impl WorkflowMachines {
                 c
             } else {
                 return Err(WFMachinesError::Nondeterminism(format!(
-                    "No command scheduled for event {}",
-                    event
+                    "No command scheduled for event {event}"
                 )));
             };
 
@@ -475,7 +686,7 @@ impl WorkflowMachines {
 
             if !canceled_before_sent {
                 // Feed the machine the event
-                self.submachine_handle_event(command.machine, event, true)?;
+                self.submachine_handle_event(command.machine, event_dat)?;
                 break command;
             }
         };
@@ -488,23 +699,19 @@ impl WorkflowMachines {
         Ok(())
     }
 
-    fn handle_non_stateful_event(
-        &mut self,
-        event: HistoryEvent,
-        has_next_event: bool,
-    ) -> Result<()> {
+    fn handle_non_stateful_event(&mut self, event_dat: HistEventData) -> Result<()> {
         trace!(
-            event = %event,
+            event = %event_dat.event,
             "handling non-stateful event"
         );
-        let event_id = event.event_id;
-        match EventType::from_i32(event.event_type) {
+        let event_id = event_dat.event.event_id;
+        match EventType::from_i32(event_dat.event.event_type) {
             Some(EventType::WorkflowExecutionStarted) => {
                 if let Some(history_event::Attributes::WorkflowExecutionStartedEventAttributes(
                     attrs,
-                )) = event.attributes
+                )) = event_dat.event.attributes
                 {
-                    if let Some(st) = event.event_time {
+                    if let Some(st) = event_dat.event.event_time.clone() {
                         let as_systime: SystemTime = st.try_into()?;
                         self.workflow_start_time = Some(as_systime);
                         // Set the workflow time to be the event time of the first event, so that
@@ -516,27 +723,28 @@ impl WorkflowMachines {
                     self.drive_me.start(
                         self.workflow_id.clone(),
                         str_to_randomness_seed(&attrs.original_execution_run_id),
+                        event_dat.event.event_time.unwrap_or_default(),
                         attrs,
                     );
                 } else {
                     return Err(WFMachinesError::Fatal(format!(
-                        "WorkflowExecutionStarted event did not have appropriate attributes: {}",
-                        event
+                        "WorkflowExecutionStarted event did not have appropriate attributes: {event_dat}"
                     )));
                 }
             }
             Some(EventType::WorkflowTaskScheduled) => {
                 let wf_task_sm = WorkflowTaskMachine::new(self.next_started_event_id);
                 let key = self.all_machines.insert(wf_task_sm.into());
-                self.submachine_handle_event(key, event, has_next_event)?;
+                self.submachine_handle_event(key, event_dat)?;
                 self.machines_by_event_id.insert(event_id, key);
             }
             Some(EventType::WorkflowExecutionSignaled) => {
                 if let Some(history_event::Attributes::WorkflowExecutionSignaledEventAttributes(
                     attrs,
-                )) = event.attributes
+                )) = event_dat.event.attributes
                 {
-                    self.drive_me.signal(attrs.into());
+                    self.drive_me
+                        .send_job(workflow_activation::SignalWorkflow::from(attrs).into());
                 } else {
                     // err
                 }
@@ -546,63 +754,23 @@ impl WorkflowMachines {
                     history_event::Attributes::WorkflowExecutionCancelRequestedEventAttributes(
                         attrs,
                     ),
-                ) = event.attributes
+                ) = event_dat.event.attributes
                 {
-                    self.drive_me.cancel(attrs.into());
+                    self.drive_me
+                        .send_job(workflow_activation::CancelWorkflow::from(attrs).into());
                 } else {
                     // err
                 }
             }
             _ => {
                 return Err(WFMachinesError::Fatal(format!(
-                    "The event is not a non-stateful event, but we tried to handle it as one: {}",
-                    event
+                    "The event is not a non-stateful event, but we tried to handle it as one: {event_dat}"
                 )));
             }
         }
         Ok(())
     }
 
-    /// Fetches commands which are ready for processing from the state machines, generally to be
-    /// sent off to the server. They are not removed from the internal queue, that happens when
-    /// corresponding history events from the server are being handled.
-    pub(crate) fn get_commands(&self) -> Vec<ProtoCommand> {
-        self.commands
-            .iter()
-            .filter_map(|c| {
-                if !self.machine(c.machine).is_final_state() {
-                    match &c.command {
-                        MachineAssociatedCommand::Real(cmd) => Some((**cmd).clone()),
-                        MachineAssociatedCommand::FakeLocalActivityMarker(_) => None,
-                    }
-                } else {
-                    None
-                }
-            })
-            .collect()
-    }
-
-    /// Returns the next activation that needs to be performed by the lang sdk. Things like unblock
-    /// timer, etc. This does *not* cause any advancement of the state machines, it merely drains
-    /// from the outgoing queue of activation jobs.
-    ///
-    /// The job list may be empty, in which case it is expected the caller handles what to do in a
-    /// "no work" situation. Possibly, it may know about some work the machines don't, like queries.
-    pub(crate) fn get_wf_activation(&mut self) -> WorkflowActivation {
-        let jobs = self.drive_me.drain_jobs();
-        WorkflowActivation {
-            timestamp: self.current_wf_time.map(Into::into),
-            is_replaying: self.replaying,
-            run_id: self.run_id.clone(),
-            history_length: self.last_processed_event as u32,
-            jobs,
-        }
-    }
-
-    pub(crate) fn has_pending_jobs(&self) -> bool {
-        !self.drive_me.peek_pending_jobs().is_empty()
-    }
-
     fn set_current_time(&mut self, time: SystemTime) -> SystemTime {
         if self.current_wf_time.map_or(true, |t| t < time) {
             self.current_wf_time = Some(time);
@@ -611,111 +779,10 @@ impl WorkflowMachines {
             .expect("We have just ensured this is populated")
     }
 
-    /// Iterate the state machines, which consists of grabbing any pending outgoing commands from
-    /// the workflow code, handling them, and preparing them to be sent off to the server.
-    pub(crate) async fn iterate_machines(&mut self) -> Result<()> {
-        let results = self.drive_me.fetch_workflow_iteration_output().await;
-        self.handle_driven_results(results)?;
-        self.prepare_commands()?;
-        if self.workflow_is_finished() {
-            if let Some(rt) = self.total_runtime() {
-                self.metrics.wf_e2e_latency(rt);
-            }
-        }
-        Ok(())
-    }
-
-    /// Apply the next (unapplied) entire workflow task from history to these machines. Will replay
-    /// any events that need to be replayed until caught up to the newest WFT. May also fetch
-    /// history from server if needed.
-    pub(crate) async fn apply_next_wft_from_history(&mut self) -> Result<usize> {
-        // If we have already seen the terminal event for the entire workflow in a previous WFT,
-        // then we don't need to do anything here, and in fact we need to avoid re-applying the
-        // final WFT.
-        if self.have_seen_terminal_event {
-            return Ok(0);
-        }
-
-        let last_handled_wft_started_id = self.current_started_event_id;
-        let events = {
-            let mut evts = self
-                .last_history_from_server
-                .take_next_wft_sequence(last_handled_wft_started_id)
-                .await
-                .map_err(WFMachinesError::HistoryFetchingError)?;
-            // Do not re-process events we have already processed
-            evts.retain(|e| e.event_id > self.last_processed_event);
-            evts
-        };
-        let num_events_to_process = events.len();
-
-        // We're caught up on reply if there are no new events to process
-        // TODO: Probably this is unneeded if we evict whenever history is from non-sticky queue
-        if events.is_empty() {
-            self.replaying = false;
-        }
-        let replay_start = Instant::now();
-
-        if let Some(last_event) = events.last() {
-            if last_event.event_type == EventType::WorkflowTaskStarted as i32 {
-                self.next_started_event_id = last_event.event_id;
-            }
-        }
-
-        let mut history = events.into_iter().peekable();
-        while let Some(event) = history.next() {
-            if event.event_id != self.last_processed_event + 1 {
-                return Err(WFMachinesError::Fatal(format!(
-                    "History is out of order. Last processed event: {}, event id: {}",
-                    self.last_processed_event, event.event_id
-                )));
-            }
-            let next_event = history.peek();
-            let eid = event.event_id;
-            let etype = event.event_type;
-            self.handle_event(event, next_event.is_some())?;
-            self.last_processed_event = eid;
-            if etype == EventType::WorkflowTaskStarted as i32 && next_event.is_none() {
-                break;
-            }
-        }
-
-        // Scan through to the next WFT, searching for any patch markers, so that we can
-        // pre-resolve them.
-        for e in self.last_history_from_server.peek_next_wft_sequence() {
-            if let Some((patch_id, _)) = e.get_patch_marker_details() {
-                self.encountered_change_markers.insert(
-                    patch_id.clone(),
-                    ChangeInfo {
-                        created_command: false,
-                    },
-                );
-                // Found a patch marker
-                self.drive_me
-                    .send_job(workflow_activation_job::Variant::NotifyHasPatch(
-                        NotifyHasPatch { patch_id },
-                    ));
-            } else if e.is_local_activity_marker() {
-                self.local_activity_data.process_peekahead_marker(e)?;
-            }
-        }
-
-        if !self.replaying {
-            self.metrics.wf_task_replay_latency(replay_start.elapsed());
-        }
-
-        Ok(num_events_to_process)
-    }
-
     /// Wrapper for calling [TemporalStateMachine::handle_event] which appropriately takes action
     /// on the returned machine responses
-    fn submachine_handle_event(
-        &mut self,
-        sm: MachineKey,
-        event: HistoryEvent,
-        has_next_event: bool,
-    ) -> Result<()> {
-        let machine_responses = self.machine_mut(sm).handle_event(event, has_next_event)?;
+    fn submachine_handle_event(&mut self, sm: MachineKey, event: HistEventData) -> Result<()> {
+        let machine_responses = self.machine_mut(sm).handle_event(event)?;
         self.process_machine_responses(sm, machine_responses)?;
         Ok(())
     }
@@ -765,7 +832,7 @@ impl WorkflowMachines {
     ) -> Result<()> {
         let sm = self.machine(smk);
         if !machine_responses.is_empty() {
-            debug!(responses = %machine_responses.display(), machine_name = %sm.kind(),
+            trace!(responses = %machine_responses.display(), machine_name = %sm.name(),
                    "Machine produced responses");
         }
         self.process_machine_resps_impl(smk, machine_responses)
@@ -791,14 +858,12 @@ impl WorkflowMachines {
                     self.task_started(task_started_event_id, time)?;
                 }
                 MachineResponse::UpdateRunIdOnWorkflowReset { run_id: new_run_id } => {
-                    // TODO: Should this also update self.run_id? Should we track orig/current
-                    //   separately?
-                    self.drive_me
-                        .send_job(workflow_activation_job::Variant::UpdateRandomSeed(
-                            UpdateRandomSeed {
-                                randomness_seed: str_to_randomness_seed(&new_run_id),
-                            },
-                        ));
+                    self.drive_me.send_job(
+                        workflow_activation_job::Variant::UpdateRandomSeed(UpdateRandomSeed {
+                            randomness_seed: str_to_randomness_seed(&new_run_id),
+                        })
+                        .into(),
+                    );
                 }
                 MachineResponse::IssueNewCommand(c) => {
                     self.current_wf_task_commands.push_back(CommandAndMachine {
@@ -822,8 +887,7 @@ impl WorkflowMachines {
                     }
                     c => {
                         return Err(WFMachinesError::Fatal(format!(
-                        "A machine requested to create a new command of an unsupported type: {:?}",
-                        c
+                        "A machine requested to create a new command of an unsupported type: {c:?}"
                     )))
                     }
                 },
@@ -860,7 +924,7 @@ impl WorkflowMachines {
                                 Duration::from_secs(0),
                                 removed_act.attempt,
                                 None,
-                                None,
+                                removed_act.original_schedule_time,
                             )?;
                             self.process_machine_responses(smk, more_responses)?;
                         } else {
@@ -888,6 +952,30 @@ impl WorkflowMachines {
         Ok(())
     }
 
+    /// Called when a workflow task started event has triggered. Ensures we are tracking the ID
+    /// of the current started event as well as workflow time properly.
+    fn task_started(&mut self, task_started_event_id: i64, time: SystemTime) -> Result<()> {
+        self.current_started_event_id = task_started_event_id;
+        self.wft_start_time = Some(time);
+        self.set_current_time(time);
+
+        // Notify local activity machines that we started a non-replay WFT, which will allow any
+        // which were waiting for a marker to instead decide to execute the LA since it clearly
+        // will not be resolved via marker.
+        if !self.replaying {
+            let mut resps = vec![];
+            for (k, mach) in self.all_machines.iter_mut() {
+                if let Machines::LocalActivityMachine(lam) = mach {
+                    resps.push((k, lam.encountered_non_replay_wft()?));
+                }
+            }
+            for (mkey, resp_set) in resps {
+                self.process_machine_responses(mkey, resp_set)?;
+            }
+        }
+        Ok(())
+    }
+
     /// Handles results of the workflow activation, delegating work to the appropriate state
     /// machine. Returns a list of workflow jobs that should be queued in the pending activation for
     /// the next poll. This list will be populated only if state machine produced lang activations
@@ -912,7 +1000,10 @@ impl WorkflowMachines {
                 }
                 WFCommand::AddActivity(attrs) => {
                     let seq = attrs.seq;
-                    self.add_cmd_to_wf_task(new_activity(attrs), CommandID::Activity(seq).into());
+                    self.add_cmd_to_wf_task(
+                        ActivityMachine::new_scheduled(attrs, self.observed_internal_flags.clone()),
+                        CommandID::Activity(seq).into(),
+                    );
                 }
                 WFCommand::AddLocalActivity(attrs) => {
                     let seq = attrs.seq;
@@ -924,8 +1015,7 @@ impl WorkflowMachines {
                     )
                     .map_err(|e| {
                         WFMachinesError::Fatal(format!(
-                            "Invalid schedule local activity request (seq {}): {}",
-                            seq, e
+                            "Invalid schedule local activity request (seq {seq}): {e}"
                         ))
                     })?;
                     let (la, mach_resp) = new_local_activity(
@@ -933,6 +1023,7 @@ impl WorkflowMachines {
                         self.replaying,
                         self.local_activity_data.take_preresolution(seq),
                         self.current_wf_time,
+                        self.observed_internal_flags.clone(),
                     )?;
                     let machkey = self.all_machines.insert(la.into());
                     self.id_to_machine
@@ -988,7 +1079,10 @@ impl WorkflowMachines {
                 WFCommand::AddChildWorkflow(attrs) => {
                     let seq = attrs.seq;
                     self.add_cmd_to_wf_task(
-                        new_child_workflow(attrs),
+                        ChildWorkflowMachine::new_scheduled(
+                            attrs,
+                            self.observed_internal_flags.clone(),
+                        ),
                         CommandID::ChildWorkflowStart(seq).into(),
                     );
                 }
@@ -1037,6 +1131,12 @@ impl WorkflowMachines {
                     // Nothing to do here, queries are handled above the machine level
                     unimplemented!("Query responses should not make it down into the machines")
                 }
+                WFCommand::ModifyWorkflowProperties(attrs) => {
+                    self.add_cmd_to_wf_task(
+                        modify_workflow_properties(attrs),
+                        CommandIdKind::NeverResolves,
+                    );
+                }
                 WFCommand::NoCommandsFromLang => (),
             }
         }
@@ -1056,7 +1156,7 @@ impl WorkflowMachines {
 
     fn get_machine_key(&self, id: CommandID) -> Result<MachineKey> {
         Ok(*self.id_to_machine.get(&id).ok_or_else(|| {
-            WFMachinesError::Fatal(format!("Missing associated machine for {:?}", id))
+            WFMachinesError::Fatal(format!("Missing associated machine for {id:?}"))
         })?)
     }
 
@@ -1143,10 +1243,7 @@ enum ChangeMarkerOutcome {
 
 /// Special handling for patch markers, when handling command events as in
 /// [WorkflowMachines::handle_command_event]
-fn change_marker_handling(
-    event: &HistoryEvent,
-    mach: &dyn TemporalStateMachine,
-) -> Result<ChangeMarkerOutcome> {
+fn change_marker_handling(event: &HistoryEvent, mach: &Machines) -> Result<ChangeMarkerOutcome> {
     if !mach.matches_event(event) {
         // Version markers can be skipped in the event they are deprecated
         if let Some((patch_name, deprecated)) = event.get_patch_marker_details() {
@@ -1157,16 +1254,15 @@ fn change_marker_handling(
                 return Ok(ChangeMarkerOutcome::SkipEvent);
             }
             return Err(WFMachinesError::Nondeterminism(format!(
-                "Non-deprecated patch marker encountered for change {}, \
-                            but there is no corresponding change command!",
-                patch_name
+                "Non-deprecated patch marker encountered for change {patch_name}, \
+                            but there is no corresponding change command!"
             )));
         }
-        // Version machines themselves may also not *have* matching markers, where non-deprecated
+        // Patch machines themselves may also not *have* matching markers, where non-deprecated
         // calls take the old path, and deprecated calls assume history is produced by a new-code
         // worker.
-        if mach.kind() == MachineKind::Patch {
-            debug!("Skipping non-matching event against version machine");
+        if matches!(mach, Machines::PatchMachine(_)) {
+            debug!("Skipping non-matching event against patch machine");
             return Ok(ChangeMarkerOutcome::SkipCommand);
         }
     }