@temporalio/core-bridge 1.5.2 → 1.7.0

package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs

@@ -1,11 +1,8 @@
 mod local_acts;
 
-pub(crate) use temporal_sdk_core_api::errors::WFMachinesError;
-
 use super::{
-    activity_state_machine::new_activity, cancel_external_state_machine::new_external_cancel,
+    cancel_external_state_machine::new_external_cancel,
     cancel_workflow_state_machine::cancel_workflow,
-    child_workflow_state_machine::new_child_workflow,
     complete_workflow_state_machine::complete_workflow,
     continue_as_new_workflow_state_machine::continue_as_new,
     fail_workflow_state_machine::fail_workflow, local_activity_state_machine::new_local_activity,
@@ -16,13 +13,23 @@ use super::{
     TemporalStateMachine,
 };
 use crate::{
+    internal_flags::InternalFlags,
     protosext::{HistoryEventExt, ValidScheduleLA},
     telemetry::{metrics::MetricsContext, VecDisplayer},
     worker::{
         workflow::{
-            machines::modify_workflow_properties_state_machine::modify_workflow_properties,
-            CommandID, DrivenWorkflow, HistoryUpdate, LocalResolution, OutgoingJob, WFCommand,
-            WorkflowFetcher, WorkflowStartedInfo,
+            history_update::NextWFT,
+            machines::{
+                activity_state_machine::ActivityMachine,
+                child_workflow_state_machine::ChildWorkflowMachine,
+                modify_workflow_properties_state_machine::modify_workflow_properties,
+                patch_state_machine::VERSION_SEARCH_ATTR_KEY,
+                upsert_search_attributes_state_machine::upsert_search_attrs_internal,
+                HistEventData,
+            },
+            CommandID, DrivenWorkflow, HistoryUpdate, InternalFlagsRef, LocalResolution,
+            OutgoingJob, RunBasics, WFCommand, WFMachinesError, WorkflowFetcher,
+            WorkflowStartedInfo,
         },
         ExecutingLAId, LocalActRequest, LocalActivityExecutionResult, LocalActivityResolution,
     },
@@ -31,9 +38,11 @@ use siphasher::sip::SipHasher13;
 use slotmap::{SlotMap, SparseSecondaryMap};
 use std::{
     borrow::{Borrow, BorrowMut},
+    cell::RefCell,
     collections::{HashMap, VecDeque},
     convert::TryInto,
     hash::{Hash, Hasher},
+    rc::Rc,
     time::{Duration, Instant, SystemTime},
 };
 use temporal_sdk_core_protos::{
@@ -50,7 +59,8 @@ use temporal_sdk_core_protos::{
     temporal::api::{
         command::v1::{command::Attributes as ProtoCmdAttrs, Command as ProtoCommand},
         enums::v1::EventType,
-        history::v1::{history_event, HistoryEvent},
+        history::v1::{history_event, history_event::Attributes, HistoryEvent},
+        sdk::v1::WorkflowTaskCompletedMetadata,
     },
 };
 
@@ -95,6 +105,9 @@ pub(crate) struct WorkflowMachines {
     /// The current workflow time if it has been established. This may differ from the WFT start
     /// time since local activities may advance the clock
     current_wf_time: Option<SystemTime>,
+    /// The internal flags which have been seen so far during this run's execution and thus are
+    /// usable during replay.
+    observed_internal_flags: InternalFlagsRef,
 
     all_machines: SlotMap<MachineKey, Machines>,
     /// If a machine key is in this map, that machine was created internally by core, not as a
@@ -133,7 +146,7 @@ pub(crate) struct WorkflowMachines {
 }
 
 #[derive(Debug, derive_more::Display)]
-#[display(fmt = "Cmd&Machine({})", "command")]
+#[display(fmt = "Cmd&Machine({command})")]
 struct CommandAndMachine {
     command: MachineAssociatedCommand,
     machine: MachineKey,
@@ -142,7 +155,7 @@ struct CommandAndMachine {
 #[derive(Debug, derive_more::Display)]
 enum MachineAssociatedCommand {
     Real(Box<ProtoCommand>),
-    #[display(fmt = "FakeLocalActivityMarker({})", "_0")]
+    #[display(fmt = "FakeLocalActivityMarker({_0})")]
     FakeLocalActivityMarker(u32),
 }
 
@@ -156,7 +169,7 @@ struct ChangeInfo {
 #[must_use]
 #[allow(clippy::large_enum_variant)]
 pub(super) enum MachineResponse {
-    #[display(fmt = "PushWFJob({})", "_0")]
+    #[display(fmt = "PushWFJob({_0})")]
     PushWFJob(OutgoingJob),
 
     /// Pushes a new command into the list that will be sent to server once we respond with the
@@ -165,31 +178,31 @@ pub(super) enum MachineResponse {
     /// The machine requests the creation of another *different* machine. This acts as if lang
     /// had replied to the activation with a command, but we use a special set of IDs to avoid
     /// collisions.
-    #[display(fmt = "NewCoreOriginatedCommand({:?})", "_0")]
+    #[display(fmt = "NewCoreOriginatedCommand({_0:?})")]
     NewCoreOriginatedCommand(ProtoCmdAttrs),
-    #[display(fmt = "IssueFakeLocalActivityMarker({})", "_0")]
+    #[display(fmt = "IssueFakeLocalActivityMarker({_0})")]
     IssueFakeLocalActivityMarker(u32),
     #[display(fmt = "TriggerWFTaskStarted")]
     TriggerWFTaskStarted {
         task_started_event_id: i64,
         time: SystemTime,
     },
-    #[display(fmt = "UpdateRunIdOnWorkflowReset({})", run_id)]
+    #[display(fmt = "UpdateRunIdOnWorkflowReset({run_id})")]
     UpdateRunIdOnWorkflowReset { run_id: String },
 
     /// Queue a local activity to be processed by the worker
     #[display(fmt = "QueueLocalActivity")]
     QueueLocalActivity(ValidScheduleLA),
     /// Request cancellation of an executing local activity
-    #[display(fmt = "RequestCancelLocalActivity({})", "_0")]
+    #[display(fmt = "RequestCancelLocalActivity({_0})")]
     RequestCancelLocalActivity(u32),
     /// Indicates we are abandoning the indicated LA, so we can remove it from "outstanding" LAs
     /// and we will not try to WFT heartbeat because of it.
-    #[display(fmt = "AbandonLocalActivity({:?})", "_0")]
+    #[display(fmt = "AbandonLocalActivity({_0:?})")]
     AbandonLocalActivity(u32),
 
     /// Set the workflow time to the provided time
-    #[display(fmt = "UpdateWFTime({:?})", "_0")]
+    #[display(fmt = "UpdateWFTime({_0:?})")]
     UpdateWFTime(Option<SystemTime>),
 }
 
@@ -203,25 +216,22 @@ where
 }
 
 impl WorkflowMachines {
-    pub(crate) fn new(
-        namespace: String,
-        workflow_id: String,
-        workflow_type: String,
-        run_id: String,
-        history: HistoryUpdate,
-        driven_wf: DrivenWorkflow,
-        metrics: MetricsContext,
-    ) -> Self {
-        let replaying = history.previous_started_event_id > 0;
+    pub(crate) fn new(basics: RunBasics, driven_wf: DrivenWorkflow) -> Self {
+        let replaying = basics.history.previous_wft_started_id > 0;
+        let mut observed_internal_flags = InternalFlags::new(basics.capabilities);
+        // Peek ahead to determine used patches in the first WFT.
+        if let Some(attrs) = basics.history.peek_next_wft_completed(0) {
+            observed_internal_flags.add_from_complete(attrs);
+        };
         Self {
-            last_history_from_server: history,
-            namespace,
-            workflow_id,
-            workflow_type,
-            run_id,
+            last_history_from_server: basics.history,
+            namespace: basics.namespace,
+            workflow_id: basics.workflow_id,
+            workflow_type: basics.workflow_type,
+            run_id: basics.run_id,
             drive_me: driven_wf,
             replaying,
-            metrics,
+            metrics: basics.metrics,
             // In an ideal world one could say ..Default::default() here and it'd still work.
             current_started_event_id: 0,
             next_started_event_id: 0,
@@ -230,6 +240,7 @@ impl WorkflowMachines {
             workflow_end_time: None,
             wft_start_time: None,
             current_wf_time: None,
+            observed_internal_flags: Rc::new(RefCell::new(observed_internal_flags)),
             all_machines: Default::default(),
             machine_is_core_created: Default::default(),
             machines_by_event_id: Default::default(),
@@ -255,10 +266,10 @@ impl WorkflowMachines {
             .and_then(|(st, et)| et.duration_since(st).ok())
     }
 
-    pub(crate) async fn new_history_from_server(&mut self, update: HistoryUpdate) -> Result<()> {
+    pub(crate) fn new_history_from_server(&mut self, update: HistoryUpdate) -> Result<()> {
         self.last_history_from_server = update;
-        self.replaying = self.last_history_from_server.previous_started_event_id > 0;
-        self.apply_next_wft_from_history().await?;
+        self.replaying = self.last_history_from_server.previous_wft_started_id > 0;
+        self.apply_next_wft_from_history()?;
         Ok(())
     }
 
@@ -290,9 +301,8 @@ impl WorkflowMachines {
                     self.process_machine_responses(mk, resps)?;
                 } else {
                     return Err(WFMachinesError::Nondeterminism(format!(
-                        "Command matching activity with seq num {} existed but was not a \
-                        local activity!",
-                        seq
+                        "Command matching activity with seq num {seq} existed but was not a \
+                        local activity!"
                     )));
                 }
                 self.local_activity_data.done_executing(seq);
@@ -350,6 +360,12 @@ impl WorkflowMachines {
             run_id: self.run_id.clone(),
             history_length: self.last_processed_event as u32,
             jobs,
+            available_internal_flags: (*self.observed_internal_flags)
+                .borrow()
+                .all_lang()
+                .iter()
+                .copied()
+                .collect(),
         }
     }
 
@@ -364,10 +380,22 @@ impl WorkflowMachines {
             .any(|v| v.is_la_resolution)
     }
 
+    pub(crate) fn get_metadata_for_wft_complete(&self) -> WorkflowTaskCompletedMetadata {
+        (*self.observed_internal_flags)
+            .borrow_mut()
+            .gather_for_wft_complete()
+    }
+
+    pub(crate) fn add_lang_used_flags(&self, flags: Vec<u32>) {
+        (*self.observed_internal_flags)
+            .borrow_mut()
+            .add_lang_used(flags);
+    }
+
     /// Iterate the state machines, which consists of grabbing any pending outgoing commands from
     /// the workflow code, handling them, and preparing them to be sent off to the server.
-    pub(crate) async fn iterate_machines(&mut self) -> Result<()> {
-        let results = self.drive_me.fetch_workflow_iteration_output().await;
+    pub(crate) fn iterate_machines(&mut self) -> Result<()> {
+        let results = self.drive_me.fetch_workflow_iteration_output();
         self.handle_driven_results(results)?;
         self.prepare_commands()?;
         if self.workflow_is_finished() {
@@ -378,27 +406,60 @@ impl WorkflowMachines {
         Ok(())
     }
 
+    /// Returns true if machines are ready to apply the next WFT sequence, false if events will need
+    /// to be fetched in order to create a complete update with the entire next WFT sequence.
+    pub(crate) fn ready_to_apply_next_wft(&self) -> bool {
+        self.last_history_from_server
+            .can_take_next_wft_sequence(self.current_started_event_id)
+    }
+
     /// Apply the next (unapplied) entire workflow task from history to these machines. Will replay
-    /// any events that need to be replayed until caught up to the newest WFT. May also fetch
-    /// history from server if needed.
-    pub(crate) async fn apply_next_wft_from_history(&mut self) -> Result<usize> {
+    /// any events that need to be replayed until caught up to the newest WFT.
+    pub(crate) fn apply_next_wft_from_history(&mut self) -> Result<usize> {
         // If we have already seen the terminal event for the entire workflow in a previous WFT,
         // then we don't need to do anything here, and in fact we need to avoid re-applying the
         // final WFT.
         if self.have_seen_terminal_event {
+            // Replay clearly counts as done now, since we return here and never do anything else.
+            self.replaying = false;
             return Ok(0);
         }
 
-        let last_handled_wft_started_id = self.current_started_event_id;
-        let events = {
-            let mut evts = self
+        fn update_internal_flags(me: &mut WorkflowMachines) {
+            // Update observed patches with any that were used in the task
+            if let Some(next_complete) = me
                 .last_history_from_server
-                .take_next_wft_sequence(last_handled_wft_started_id)
-                .await
-                .map_err(WFMachinesError::HistoryFetchingError)?;
-            // Do not re-process events we have already processed
-            evts.retain(|e| e.event_id > self.last_processed_event);
-            evts
+                .peek_next_wft_completed(me.last_processed_event)
+            {
+                (*me.observed_internal_flags)
+                    .borrow_mut()
+                    .add_from_complete(next_complete);
+            }
+        }
+
+        // We update the internal flags before applying the current task (peeking to the completion
+        // of this task), and also at the end (peeking to the completion of the task that lang is
+        // about to generate commands for, and for which we will want those flags active).
+        update_internal_flags(self);
+
+        let last_handled_wft_started_id = self.current_started_event_id;
+        let (events, has_final_event) = match self
+            .last_history_from_server
+            .take_next_wft_sequence(last_handled_wft_started_id)
+        {
+            NextWFT::ReplayOver => (vec![], true),
+            NextWFT::WFT(mut evts, has_final_event) => {
+                // Do not re-process events we have already processed
+                evts.retain(|e| e.event_id > self.last_processed_event);
+                (evts, has_final_event)
+            }
+            NextWFT::NeedFetch => {
+                return Err(WFMachinesError::Fatal(
+                    "Need to fetch history events to continue applying workflow task, but this \
+                     should be prevented ahead of time! This is a Core SDK bug."
+                        .to_string(),
+                ));
+            }
         };
         let num_events_to_process = events.len();
 
@@ -414,6 +475,8 @@ impl WorkflowMachines {
             }
         }
 
+        let mut saw_completed = false;
+        let mut do_handle_event = true;
         let mut history = events.into_iter().peekable();
         while let Some(event) = history.next() {
             if event.event_id != self.last_processed_event + 1 {
@@ -424,17 +487,52 @@ impl WorkflowMachines {
             }
             let next_event = history.peek();
             let eid = event.event_id;
-            let etype = event.event_type();
-            self.handle_event(event, next_event.is_some())?;
-            self.last_processed_event = eid;
-            if etype == EventType::WorkflowTaskStarted && next_event.is_none() {
-                break;
+
+            // This definition of replaying here is that we are no longer replaying as soon as we
+            // see new events that have never been seen or produced by the SDK.
+            //
+            // Specifically, replay ends once we have seen the last command-event which was produced
+            // as a result of the last completed WFT. Thus, replay would be false for things like
+            // signals which were received and after the last completion, and thus generated the
+            // current WFT being handled.
+            if self.replaying && has_final_event && saw_completed && !event.is_command_event() {
+                // Replay is finished
+                self.replaying = false;
+            }
+            if event.event_type() == EventType::WorkflowTaskCompleted {
+                saw_completed = true;
             }
+
+            if do_handle_event {
+                let eho = self.handle_event(
+                    HistEventData {
+                        event,
+                        replaying: self.replaying,
+                        current_task_is_last_in_history: has_final_event,
+                    },
+                    next_event,
+                )?;
+                if matches!(
+                    eho,
+                    EventHandlingOutcome::SkipEvent {
+                        skip_next_event: true
+                    }
+                ) {
+                    do_handle_event = false;
+                }
+            } else {
+                do_handle_event = true;
+            }
+            self.last_processed_event = eid;
         }
 
         // Scan through to the next WFT, searching for any patch / la markers, so that we can
         // pre-resolve them.
-        for e in self.last_history_from_server.peek_next_wft_sequence() {
+        let mut wake_las = vec![];
+        for e in self
+            .last_history_from_server
+            .peek_next_wft_sequence(last_handled_wft_started_id)
+        {
             if let Some((patch_id, _)) = e.get_patch_marker_details() {
                 self.encountered_change_markers.insert(
                     patch_id.clone(),
@@ -448,10 +546,35 @@ impl WorkflowMachines {
                         .into(),
                 );
             } else if e.is_local_activity_marker() {
-                self.local_activity_data.process_peekahead_marker(e)?;
+                if let Some(la_dat) = e.clone().into_local_activity_marker_details() {
+                    if let Ok(mk) =
+                        self.get_machine_key(CommandID::LocalActivity(la_dat.marker_dat.seq))
+                    {
+                        wake_las.push((mk, la_dat));
+                    } else {
+                        self.local_activity_data.insert_peeked_marker(la_dat);
+                    }
+                } else {
+                    return Err(WFMachinesError::Fatal(format!(
+                        "Local activity marker was unparsable: {e:?}"
+                    )));
+                }
+            }
+        }
+        for (mk, la_dat) in wake_las {
+            let mach = self.machine_mut(mk);
+            if let Machines::LocalActivityMachine(ref mut lam) = *mach {
+                if lam.will_accept_resolve_marker() {
+                    let resps = lam.try_resolve_with_dat(la_dat.into())?;
+                    self.process_machine_responses(mk, resps)?;
+                } else {
+                    self.local_activity_data.insert_peeked_marker(la_dat);
+                }
             }
         }
 
+        update_internal_flags(self);
+
         if !self.replaying {
             self.metrics.wf_task_replay_latency(replay_start.elapsed());
         }
@@ -459,16 +582,20 @@ impl WorkflowMachines {
         Ok(num_events_to_process)
     }
 
-    /// Handle a single event from the workflow history. `has_next_event` should be false if `event`
-    /// is the last event in the history.
+    /// Handle a single event from the workflow history.
     ///
     /// This function will attempt to apply the event to the workflow state machines. If there is
     /// not a matching machine for the event, a nondeterminism error is returned. Otherwise, the
     /// event is applied to the machine, which may also return a nondeterminism error if the machine
     /// does not match the expected type. A fatal error may be returned if the machine is in an
     /// invalid state.
-    #[instrument(skip(self, event), fields(event=%event))]
-    fn handle_event(&mut self, event: HistoryEvent, has_next_event: bool) -> Result<()> {
+    #[instrument(skip(self, event_dat), fields(event=%event_dat))]
+    fn handle_event(
+        &mut self,
+        event_dat: HistEventData,
+        next_event: Option<&HistoryEvent>,
+    ) -> Result<EventHandlingOutcome> {
+        let event = &event_dat.event;
         if event.is_final_wf_execution_event() {
             self.have_seen_terminal_event = true;
         }
@@ -476,39 +603,33 @@ impl WorkflowMachines {
             event.event_type(),
             EventType::WorkflowExecutionTerminated | EventType::WorkflowExecutionTimedOut
         ) {
-            return if has_next_event {
+            let are_more_events =
+                next_event.is_some() || !event_dat.current_task_is_last_in_history;
+            return if are_more_events {
                 Err(WFMachinesError::Fatal(
                     "Machines were fed a history which has an event after workflow execution was \
                      terminated!"
                         .to_string(),
                 ))
             } else {
-                Ok(())
+                Ok(EventHandlingOutcome::Normal)
             };
         }
-        if self.replaying
-            && self.current_started_event_id
-                >= self.last_history_from_server.previous_started_event_id
-            && event.event_type() != EventType::WorkflowTaskCompleted
-        {
-            // Replay is finished
-            self.replaying = false;
-        }
         if event.event_type() == EventType::Unspecified || event.attributes.is_none() {
             return if !event.worker_may_ignore {
                 Err(WFMachinesError::Fatal(format!(
-                    "Event type is unspecified! This history is invalid. Event detail: {:?}",
-                    event
+                    "Event type is unspecified! This history is invalid. Event detail: {event:?}"
                 )))
             } else {
                 debug!("Event is ignorable");
-                Ok(())
+                Ok(EventHandlingOutcome::SkipEvent {
+                    skip_next_event: false,
+                })
             };
         }
 
         if event.is_command_event() {
-            self.handle_command_event(event)?;
-            return Ok(());
+            return self.handle_command_event(event_dat, next_event);
         }
 
         if let Some(initial_cmd_id) = event.get_initial_command_event_id() {
@@ -517,7 +638,7 @@ impl WorkflowMachines {
             let maybe_machine = self.machines_by_event_id.remove(&initial_cmd_id);
             match maybe_machine {
                 Some(sm) => {
-                    self.submachine_handle_event(sm, event, has_next_event)?;
+                    self.submachine_handle_event(sm, event_dat)?;
                     // Restore machine if not in it's final state
                     if !self.machine(sm).is_final_state() {
                         self.machines_by_event_id.insert(initial_cmd_id, sm);
@@ -526,16 +647,15 @@ impl WorkflowMachines {
                 None => {
                     return Err(WFMachinesError::Nondeterminism(format!(
                         "During event handling, this event had an initial command ID but we \
-                            could not find a matching command for it: {:?}",
-                        event
+                            could not find a matching command for it: {event:?}"
                     )));
                 }
             }
         } else {
-            self.handle_non_stateful_event(event, has_next_event)?;
+            self.handle_non_stateful_event(event_dat)?;
         }
 
-        Ok(())
+        Ok(EventHandlingOutcome::Normal)
     }
 
     /// A command event is an event which is generated from a command emitted as a result of
@@ -547,23 +667,28 @@ impl WorkflowMachines {
     /// The handling consists of verifying that the next command in the commands queue is associated
     /// with a state machine, which is then notified about the event and the command is removed from
     /// the commands queue.
-    fn handle_command_event(&mut self, event: HistoryEvent) -> Result<()> {
+    fn handle_command_event(
+        &mut self,
+        event_dat: HistEventData,
+        next_event: Option<&HistoryEvent>,
+    ) -> Result<EventHandlingOutcome> {
+        let event = &event_dat.event;
+
         if event.is_local_activity_marker() {
             let deets = event.extract_local_activity_marker_data().ok_or_else(|| {
-                WFMachinesError::Fatal(format!("Local activity marker was unparsable: {:?}", event))
+                WFMachinesError::Fatal(format!("Local activity marker was unparsable: {event:?}"))
             })?;
             let cmdid = CommandID::LocalActivity(deets.seq);
             let mkey = self.get_machine_key(cmdid)?;
             if let Machines::LocalActivityMachine(lam) = self.machine(mkey) {
                 if lam.marker_should_get_special_handling()? {
-                    self.submachine_handle_event(mkey, event, false)?;
-                    return Ok(());
+                    self.submachine_handle_event(mkey, event_dat)?;
+                    return Ok(EventHandlingOutcome::Normal);
                 }
             } else {
                 return Err(WFMachinesError::Fatal(format!(
                     "Encountered local activity marker but the associated machine was of the \
-                     wrong type! {:?}",
-                    event
+                     wrong type! {event:?}"
                 )));
             }
         }
@@ -573,13 +698,13 @@ impl WorkflowMachines {
         let consumed_cmd = loop {
             if let Some(peek_machine) = self.commands.front() {
                 let mach = self.machine(peek_machine.machine);
-                match change_marker_handling(&event, mach)? {
-                    ChangeMarkerOutcome::SkipEvent => return Ok(()),
-                    ChangeMarkerOutcome::SkipCommand => {
+                match change_marker_handling(event, mach, next_event)? {
+                    EventHandlingOutcome::SkipCommand => {
                         self.commands.pop_front();
                         continue;
                     }
-                    ChangeMarkerOutcome::Normal => {}
+                    eho @ EventHandlingOutcome::SkipEvent { .. } => return Ok(eho),
+                    EventHandlingOutcome::Normal => {}
                 }
             }
 
@@ -588,8 +713,7 @@ impl WorkflowMachines {
                 c
             } else {
                 return Err(WFMachinesError::Nondeterminism(format!(
-                    "No command scheduled for event {}",
-                    event
+                    "No command scheduled for event {event}"
                 )));
             };
 
@@ -599,7 +723,7 @@ impl WorkflowMachines {
 
             if !canceled_before_sent {
                 // Feed the machine the event
-                self.submachine_handle_event(command.machine, event, true)?;
+                self.submachine_handle_event(command.machine, event_dat)?;
                 break command;
             }
         };
@@ -609,26 +733,22 @@ impl WorkflowMachines {
                 .insert(event_id, consumed_cmd.machine);
         }
 
-        Ok(())
+        Ok(EventHandlingOutcome::Normal)
     }
 
-    fn handle_non_stateful_event(
-        &mut self,
-        event: HistoryEvent,
-        has_next_event: bool,
-    ) -> Result<()> {
+    fn handle_non_stateful_event(&mut self, event_dat: HistEventData) -> Result<()> {
         trace!(
-            event = %event,
+            event = %event_dat.event,
             "handling non-stateful event"
         );
-        let event_id = event.event_id;
-        match EventType::from_i32(event.event_type) {
+        let event_id = event_dat.event.event_id;
+        match EventType::from_i32(event_dat.event.event_type) {
             Some(EventType::WorkflowExecutionStarted) => {
                 if let Some(history_event::Attributes::WorkflowExecutionStartedEventAttributes(
                     attrs,
-                )) = event.attributes
+                )) = event_dat.event.attributes
                 {
-                    if let Some(st) = event.event_time.clone() {
+                    if let Some(st) = event_dat.event.event_time.clone() {
                         let as_systime: SystemTime = st.try_into()?;
                         self.workflow_start_time = Some(as_systime);
                         // Set the workflow time to be the event time of the first event, so that
@@ -640,26 +760,25 @@ impl WorkflowMachines {
                     self.drive_me.start(
                         self.workflow_id.clone(),
                         str_to_randomness_seed(&attrs.original_execution_run_id),
-                        event.event_time.unwrap_or_default(),
+                        event_dat.event.event_time.unwrap_or_default(),
                         attrs,
                     );
                 } else {
                     return Err(WFMachinesError::Fatal(format!(
-                        "WorkflowExecutionStarted event did not have appropriate attributes: {}",
-                        event
+                        "WorkflowExecutionStarted event did not have appropriate attributes: {event_dat}"
                     )));
                 }
             }
             Some(EventType::WorkflowTaskScheduled) => {
                 let wf_task_sm = WorkflowTaskMachine::new(self.next_started_event_id);
                 let key = self.all_machines.insert(wf_task_sm.into());
-                self.submachine_handle_event(key, event, has_next_event)?;
+                self.submachine_handle_event(key, event_dat)?;
                 self.machines_by_event_id.insert(event_id, key);
             }
             Some(EventType::WorkflowExecutionSignaled) => {
                 if let Some(history_event::Attributes::WorkflowExecutionSignaledEventAttributes(
                     attrs,
-                )) = event.attributes
+                )) = event_dat.event.attributes
                 {
                     self.drive_me
                         .send_job(workflow_activation::SignalWorkflow::from(attrs).into());
@@ -672,7 +791,7 @@ impl WorkflowMachines {
                     history_event::Attributes::WorkflowExecutionCancelRequestedEventAttributes(
                         attrs,
                     ),
-                ) = event.attributes
+                ) = event_dat.event.attributes
                 {
                     self.drive_me
                         .send_job(workflow_activation::CancelWorkflow::from(attrs).into());
@@ -682,8 +801,7 @@ impl WorkflowMachines {
             }
             _ => {
                 return Err(WFMachinesError::Fatal(format!(
-                    "The event is not a non-stateful event, but we tried to handle it as one: {}",
-                    event
+                    "The event is not a non-stateful event, but we tried to handle it as one: {event_dat}"
                 )));
             }
         }
@@ -700,13 +818,8 @@ impl WorkflowMachines {
 
     /// Wrapper for calling [TemporalStateMachine::handle_event] which appropriately takes action
     /// on the returned machine responses
-    fn submachine_handle_event(
-        &mut self,
-        sm: MachineKey,
-        event: HistoryEvent,
-        has_next_event: bool,
-    ) -> Result<()> {
-        let machine_responses = self.machine_mut(sm).handle_event(event, has_next_event)?;
+    fn submachine_handle_event(&mut self, sm: MachineKey, event: HistEventData) -> Result<()> {
+        let machine_responses = self.machine_mut(sm).handle_event(event)?;
         self.process_machine_responses(sm, machine_responses)?;
         Ok(())
     }
@@ -756,7 +869,7 @@ impl WorkflowMachines {
     ) -> Result<()> {
         let sm = self.machine(smk);
         if !machine_responses.is_empty() {
-            debug!(responses = %machine_responses.display(), machine_name = %sm.name(),
+            trace!(responses = %machine_responses.display(), machine_name = %sm.name(),
                    "Machine produced responses");
         }
         self.process_machine_resps_impl(smk, machine_responses)
@@ -809,10 +922,18 @@ impl WorkflowMachines {
                             CommandIdKind::CoreInternal,
                         );
                     }
+                    ProtoCmdAttrs::UpsertWorkflowSearchAttributesCommandAttributes(attrs) => {
+                        self.add_cmd_to_wf_task(
+                            upsert_search_attrs_internal(
+                                attrs,
+                                self.observed_internal_flags.clone(),
+                            ),
+                            CommandIdKind::NeverResolves,
+                        );
+                    }
                     c => {
                         return Err(WFMachinesError::Fatal(format!(
-                        "A machine requested to create a new command of an unsupported type: {:?}",
-                        c
+                        "A machine requested to create a new command of an unsupported type: {c:?}"
                     )))
                     }
                 },
@@ -849,7 +970,7 @@ impl WorkflowMachines {
                                 Duration::from_secs(0),
                                 removed_act.attempt,
                                 None,
-                                None,
+                                removed_act.original_schedule_time,
                             )?;
                             self.process_machine_responses(smk, more_responses)?;
                         } else {
@@ -916,7 +1037,11 @@ impl WorkflowMachines {
                 }
                 WFCommand::UpsertSearchAttributes(attrs) => {
                     self.add_cmd_to_wf_task(
-                        upsert_search_attrs(attrs),
+                        upsert_search_attrs(
+                            attrs,
+                            self.observed_internal_flags.clone(),
+                            self.replaying,
+                        ),
                         CommandIdKind::NeverResolves,
                     );
                 }
@@ -925,7 +1050,10 @@ impl WorkflowMachines {
                 }
                 WFCommand::AddActivity(attrs) => {
                     let seq = attrs.seq;
-                    self.add_cmd_to_wf_task(new_activity(attrs), CommandID::Activity(seq).into());
+                    self.add_cmd_to_wf_task(
+                        ActivityMachine::new_scheduled(attrs, self.observed_internal_flags.clone()),
+                        CommandID::Activity(seq).into(),
+                    );
                 }
                 WFCommand::AddLocalActivity(attrs) => {
                     let seq = attrs.seq;
@@ -937,8 +1065,7 @@ impl WorkflowMachines {
                     )
                     .map_err(|e| {
                         WFMachinesError::Fatal(format!(
-                            "Invalid schedule local activity request (seq {}): {}",
-                            seq, e
+                            "Invalid schedule local activity request (seq {seq}): {e}"
                         ))
                     })?;
                     let (la, mach_resp) = new_local_activity(
@@ -946,6 +1073,7 @@ impl WorkflowMachines {
                         self.replaying,
                         self.local_activity_data.take_preresolution(seq),
                         self.current_wf_time,
+                        self.observed_internal_flags.clone(),
                     )?;
                     let machkey = self.all_machines.insert(la.into());
                     self.id_to_machine
@@ -978,13 +1106,21 @@ impl WorkflowMachines {
                 WFCommand::SetPatchMarker(attrs) => {
                     // Do not create commands for change IDs that we have already created commands
                     // for.
-                    if !matches!(self.encountered_change_markers.get(&attrs.patch_id),
+                    let encountered_entry = self.encountered_change_markers.get(&attrs.patch_id);
+                    if !matches!(encountered_entry,
                                  Some(ChangeInfo {created_command}) if *created_command)
                     {
-                        self.add_cmd_to_wf_task(
-                            has_change(attrs.patch_id.clone(), self.replaying, attrs.deprecated),
-                            CommandIdKind::NeverResolves,
-                        );
+                        let (patch_machine, other_cmds) = has_change(
+                            attrs.patch_id.clone(),
+                            self.replaying,
+                            attrs.deprecated,
+                            encountered_entry.is_some(),
+                            self.encountered_change_markers.keys().map(|s| s.as_str()),
+                            self.observed_internal_flags.clone(),
+                        )?;
+                        let mkey =
+                            self.add_cmd_to_wf_task(patch_machine, CommandIdKind::NeverResolves);
+                        self.process_machine_responses(mkey, other_cmds)?;
 
                         if let Some(ci) = self.encountered_change_markers.get_mut(&attrs.patch_id) {
                             ci.created_command = true;
@@ -1001,7 +1137,10 @@ impl WorkflowMachines {
                 WFCommand::AddChildWorkflow(attrs) => {
                     let seq = attrs.seq;
                     self.add_cmd_to_wf_task(
-                        new_child_workflow(attrs),
+                        ChildWorkflowMachine::new_scheduled(
+                            attrs,
+                            self.observed_internal_flags.clone(),
+                        ),
                         CommandID::ChildWorkflowStart(seq).into(),
                     );
                 }
@@ -1075,7 +1214,7 @@ impl WorkflowMachines {
 
     fn get_machine_key(&self, id: CommandID) -> Result<MachineKey> {
         Ok(*self.id_to_machine.get(&id).ok_or_else(|| {
-            WFMachinesError::Fatal(format!("Missing associated machine for {:?}", id))
+            WFMachinesError::Fatal(format!("Missing associated machine for {id:?}"))
         })?)
     }
 
@@ -1086,15 +1225,21 @@ impl WorkflowMachines {
     }
 
     /// Add a new command/machines for that command to the current workflow task
-    fn add_cmd_to_wf_task(&mut self, machine: NewMachineWithCommand, id: CommandIdKind) {
+    fn add_cmd_to_wf_task(
+        &mut self,
+        machine: NewMachineWithCommand,
+        id: CommandIdKind,
+    ) -> MachineKey {
         let mach = self.add_new_command_machine(machine);
+        let key = mach.machine;
         if let CommandIdKind::LangIssued(id) = id {
-            self.id_to_machine.insert(id, mach.machine);
+            self.id_to_machine.insert(id, key);
         }
         if matches!(id, CommandIdKind::CoreInternal) {
-            self.machine_is_core_created.insert(mach.machine, ());
+            self.machine_is_core_created.insert(key, ());
         }
         self.current_wf_task_commands.push_back(mach);
+        key
     }
 
     fn add_new_command_machine(&mut self, machine: NewMachineWithCommand) -> CommandAndMachine {
@@ -1154,15 +1299,20 @@ fn str_to_randomness_seed(run_id: &str) -> u64 {
     s.finish()
 }
 
-enum ChangeMarkerOutcome {
-    SkipEvent,
+#[must_use]
+enum EventHandlingOutcome {
+    SkipEvent { skip_next_event: bool },
     SkipCommand,
     Normal,
 }
 
 /// Special handling for patch markers, when handling command events as in
 /// [WorkflowMachines::handle_command_event]
-fn change_marker_handling(event: &HistoryEvent, mach: &Machines) -> Result<ChangeMarkerOutcome> {
+fn change_marker_handling(
+    event: &HistoryEvent,
+    mach: &Machines,
+    next_event: Option<&HistoryEvent>,
+) -> Result<EventHandlingOutcome> {
     if !mach.matches_event(event) {
         // Version markers can be skipped in the event they are deprecated
         if let Some((patch_name, deprecated)) = event.get_patch_marker_details() {
@@ -1170,12 +1320,22 @@ fn change_marker_handling(event: &HistoryEvent, mach: &Machines) -> Result<Chang
             // markers are allowed without matching changed calls.
             if deprecated {
                 debug!("Deprecated patch marker tried against wrong machine, skipping.");
-                return Ok(ChangeMarkerOutcome::SkipEvent);
+
+                // Also ignore the subsequent upsert event if present
+                let mut skip_next_event = false;
+                if let Some(Attributes::UpsertWorkflowSearchAttributesEventAttributes(atts)) =
+                    next_event.and_then(|ne| ne.attributes.as_ref())
+                {
+                    if let Some(ref sa) = atts.search_attributes {
+                        skip_next_event = sa.indexed_fields.contains_key(VERSION_SEARCH_ATTR_KEY);
+                    }
+                }
+
+                return Ok(EventHandlingOutcome::SkipEvent { skip_next_event });
             }
             return Err(WFMachinesError::Nondeterminism(format!(
-                "Non-deprecated patch marker encountered for change {}, \
-                            but there is no corresponding change command!",
-                patch_name
+                "Non-deprecated patch marker encountered for change {patch_name}, \
+                            but there is no corresponding change command!"
             )));
         }
         // Patch machines themselves may also not *have* matching markers, where non-deprecated
@@ -1183,10 +1343,10 @@ fn change_marker_handling(event: &HistoryEvent, mach: &Machines) -> Result<Chang
         // worker.
         if matches!(mach, Machines::PatchMachine(_)) {
             debug!("Skipping non-matching event against patch machine");
-            return Ok(ChangeMarkerOutcome::SkipCommand);
+            return Ok(EventHandlingOutcome::SkipCommand);
         }
     }
-    Ok(ChangeMarkerOutcome::Normal)
+    Ok(EventHandlingOutcome::Normal)
 }
 
 #[derive(derive_more::From)]