temporalio 0.0.2 → 0.1.0

data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs

@@ -1,11 +1,8 @@
 mod local_acts;
 
-pub(crate) use temporal_sdk_core_api::errors::WFMachinesError;
-
 use super::{
-    activity_state_machine::new_activity, cancel_external_state_machine::new_external_cancel,
+    cancel_external_state_machine::new_external_cancel,
     cancel_workflow_state_machine::cancel_workflow,
-    child_workflow_state_machine::new_child_workflow,
     complete_workflow_state_machine::complete_workflow,
     continue_as_new_workflow_state_machine::continue_as_new,
     fail_workflow_state_machine::fail_workflow, local_activity_state_machine::new_local_activity,
@@ -16,13 +13,21 @@ use super::{
     TemporalStateMachine,
 };
 use crate::{
+    internal_flags::InternalFlags,
     protosext::{HistoryEventExt, ValidScheduleLA},
     telemetry::{metrics::MetricsContext, VecDisplayer},
     worker::{
         workflow::{
-            machines::modify_workflow_properties_state_machine::modify_workflow_properties,
-            CommandID, DrivenWorkflow, HistoryUpdate, LocalResolution, OutgoingJob, WFCommand,
-            WorkflowFetcher, WorkflowStartedInfo,
+            history_update::NextWFT,
+            machines::{
+                activity_state_machine::ActivityMachine,
+                child_workflow_state_machine::ChildWorkflowMachine,
+                modify_workflow_properties_state_machine::modify_workflow_properties,
+                HistEventData,
+            },
+            CommandID, DrivenWorkflow, HistoryUpdate, InternalFlagsRef, LocalResolution,
+            OutgoingJob, RunBasics, WFCommand, WFMachinesError, WorkflowFetcher,
+            WorkflowStartedInfo,
         },
         ExecutingLAId, LocalActRequest, LocalActivityExecutionResult, LocalActivityResolution,
     },
@@ -31,9 +36,11 @@ use siphasher::sip::SipHasher13;
 use slotmap::{SlotMap, SparseSecondaryMap};
 use std::{
     borrow::{Borrow, BorrowMut},
+    cell::RefCell,
     collections::{HashMap, VecDeque},
     convert::TryInto,
     hash::{Hash, Hasher},
+    rc::Rc,
     time::{Duration, Instant, SystemTime},
 };
 use temporal_sdk_core_protos::{
@@ -51,6 +58,7 @@ use temporal_sdk_core_protos::{
         command::v1::{command::Attributes as ProtoCmdAttrs, Command as ProtoCommand},
         enums::v1::EventType,
         history::v1::{history_event, HistoryEvent},
+        sdk::v1::WorkflowTaskCompletedMetadata,
     },
 };
 
@@ -95,6 +103,9 @@ pub(crate) struct WorkflowMachines {
     /// The current workflow time if it has been established. This may differ from the WFT start
     /// time since local activities may advance the clock
     current_wf_time: Option<SystemTime>,
+    /// The internal flags which have been seen so far during this run's execution and thus are
+    /// usable during replay.
+    observed_internal_flags: InternalFlagsRef,
 
     all_machines: SlotMap<MachineKey, Machines>,
     /// If a machine key is in this map, that machine was created internally by core, not as a
@@ -133,7 +144,7 @@ pub(crate) struct WorkflowMachines {
 }
 
 #[derive(Debug, derive_more::Display)]
-#[display(fmt = "Cmd&Machine({})", "command")]
+#[display(fmt = "Cmd&Machine({command})")]
 struct CommandAndMachine {
     command: MachineAssociatedCommand,
     machine: MachineKey,
@@ -142,7 +153,7 @@ struct CommandAndMachine {
 #[derive(Debug, derive_more::Display)]
 enum MachineAssociatedCommand {
     Real(Box<ProtoCommand>),
-    #[display(fmt = "FakeLocalActivityMarker({})", "_0")]
+    #[display(fmt = "FakeLocalActivityMarker({_0})")]
     FakeLocalActivityMarker(u32),
 }
 
@@ -156,7 +167,7 @@ struct ChangeInfo {
 #[must_use]
 #[allow(clippy::large_enum_variant)]
 pub(super) enum MachineResponse {
-    #[display(fmt = "PushWFJob({})", "_0")]
+    #[display(fmt = "PushWFJob({_0})")]
     PushWFJob(OutgoingJob),
 
     /// Pushes a new command into the list that will be sent to server once we respond with the
@@ -165,31 +176,31 @@ pub(super) enum MachineResponse {
     /// The machine requests the creation of another *different* machine. This acts as if lang
     /// had replied to the activation with a command, but we use a special set of IDs to avoid
     /// collisions.
-    #[display(fmt = "NewCoreOriginatedCommand({:?})", "_0")]
+    #[display(fmt = "NewCoreOriginatedCommand({_0:?})")]
     NewCoreOriginatedCommand(ProtoCmdAttrs),
-    #[display(fmt = "IssueFakeLocalActivityMarker({})", "_0")]
+    #[display(fmt = "IssueFakeLocalActivityMarker({_0})")]
     IssueFakeLocalActivityMarker(u32),
     #[display(fmt = "TriggerWFTaskStarted")]
     TriggerWFTaskStarted {
         task_started_event_id: i64,
         time: SystemTime,
     },
-    #[display(fmt = "UpdateRunIdOnWorkflowReset({})", run_id)]
+    #[display(fmt = "UpdateRunIdOnWorkflowReset({run_id})")]
     UpdateRunIdOnWorkflowReset { run_id: String },
 
     /// Queue a local activity to be processed by the worker
     #[display(fmt = "QueueLocalActivity")]
     QueueLocalActivity(ValidScheduleLA),
     /// Request cancellation of an executing local activity
-    #[display(fmt = "RequestCancelLocalActivity({})", "_0")]
+    #[display(fmt = "RequestCancelLocalActivity({_0})")]
     RequestCancelLocalActivity(u32),
     /// Indicates we are abandoning the indicated LA, so we can remove it from "outstanding" LAs
     /// and we will not try to WFT heartbeat because of it.
-    #[display(fmt = "AbandonLocalActivity({:?})", "_0")]
+    #[display(fmt = "AbandonLocalActivity({_0:?})")]
     AbandonLocalActivity(u32),
 
     /// Set the workflow time to the provided time
-    #[display(fmt = "UpdateWFTime({:?})", "_0")]
+    #[display(fmt = "UpdateWFTime({_0:?})")]
     UpdateWFTime(Option<SystemTime>),
 }
 
@@ -203,25 +214,22 @@ where
 }
 
 impl WorkflowMachines {
-    pub(crate) fn new(
-        namespace: String,
-        workflow_id: String,
-        workflow_type: String,
-        run_id: String,
-        history: HistoryUpdate,
-        driven_wf: DrivenWorkflow,
-        metrics: MetricsContext,
-    ) -> Self {
-        let replaying = history.previous_started_event_id > 0;
+    pub(crate) fn new(basics: RunBasics, driven_wf: DrivenWorkflow) -> Self {
+        let replaying = basics.history.previous_wft_started_id > 0;
+        let mut observed_internal_flags = InternalFlags::new(basics.capabilities);
+        // Peek ahead to determine used patches in the first WFT.
+        if let Some(attrs) = basics.history.peek_next_wft_completed(0) {
+            observed_internal_flags.add_from_complete(attrs);
+        };
         Self {
-            last_history_from_server: history,
-            namespace,
-            workflow_id,
-            workflow_type,
-            run_id,
+            last_history_from_server: basics.history,
+            namespace: basics.namespace,
+            workflow_id: basics.workflow_id,
+            workflow_type: basics.workflow_type,
+            run_id: basics.run_id,
             drive_me: driven_wf,
             replaying,
-            metrics,
+            metrics: basics.metrics,
             // In an ideal world one could say ..Default::default() here and it'd still work.
             current_started_event_id: 0,
             next_started_event_id: 0,
@@ -230,6 +238,7 @@ impl WorkflowMachines {
             workflow_end_time: None,
             wft_start_time: None,
             current_wf_time: None,
+            observed_internal_flags: Rc::new(RefCell::new(observed_internal_flags)),
             all_machines: Default::default(),
             machine_is_core_created: Default::default(),
             machines_by_event_id: Default::default(),
@@ -255,10 +264,10 @@ impl WorkflowMachines {
             .and_then(|(st, et)| et.duration_since(st).ok())
     }
 
-    pub(crate) async fn new_history_from_server(&mut self, update: HistoryUpdate) -> Result<()> {
+    pub(crate) fn new_history_from_server(&mut self, update: HistoryUpdate) -> Result<()> {
         self.last_history_from_server = update;
-        self.replaying = self.last_history_from_server.previous_started_event_id > 0;
-        self.apply_next_wft_from_history().await?;
+        self.replaying = self.last_history_from_server.previous_wft_started_id > 0;
+        self.apply_next_wft_from_history()?;
         Ok(())
     }
 
@@ -290,9 +299,8 @@ impl WorkflowMachines {
                     self.process_machine_responses(mk, resps)?;
                 } else {
                     return Err(WFMachinesError::Nondeterminism(format!(
-                        "Command matching activity with seq num {} existed but was not a \
-                        local activity!",
-                        seq
+                        "Command matching activity with seq num {seq} existed but was not a \
+                        local activity!"
                     )));
                 }
                 self.local_activity_data.done_executing(seq);
@@ -350,6 +358,12 @@ impl WorkflowMachines {
             run_id: self.run_id.clone(),
             history_length: self.last_processed_event as u32,
             jobs,
+            available_internal_flags: (*self.observed_internal_flags)
+                .borrow()
+                .all_lang()
+                .iter()
+                .copied()
+                .collect(),
         }
     }
 
@@ -364,10 +378,22 @@ impl WorkflowMachines {
             .any(|v| v.is_la_resolution)
     }
 
+    pub(crate) fn get_metadata_for_wft_complete(&self) -> WorkflowTaskCompletedMetadata {
+        (*self.observed_internal_flags)
+            .borrow_mut()
+            .gather_for_wft_complete()
+    }
+
+    pub(crate) fn add_lang_used_flags(&self, flags: Vec<u32>) {
+        (*self.observed_internal_flags)
+            .borrow_mut()
+            .add_lang_used(flags);
+    }
+
     /// Iterate the state machines, which consists of grabbing any pending outgoing commands from
     /// the workflow code, handling them, and preparing them to be sent off to the server.
-    pub(crate) async fn iterate_machines(&mut self) -> Result<()> {
-        let results = self.drive_me.fetch_workflow_iteration_output().await;
+    pub(crate) fn iterate_machines(&mut self) -> Result<()> {
+        let results = self.drive_me.fetch_workflow_iteration_output();
         self.handle_driven_results(results)?;
         self.prepare_commands()?;
         if self.workflow_is_finished() {
@@ -378,10 +404,16 @@ impl WorkflowMachines {
         Ok(())
     }
 
+    /// Returns true if machines are ready to apply the next WFT sequence, false if events will need
+    /// to be fetched in order to create a complete update with the entire next WFT sequence.
+    pub(crate) fn ready_to_apply_next_wft(&self) -> bool {
+        self.last_history_from_server
+            .can_take_next_wft_sequence(self.current_started_event_id)
+    }
+
     /// Apply the next (unapplied) entire workflow task from history to these machines. Will replay
-    /// any events that need to be replayed until caught up to the newest WFT. May also fetch
-    /// history from server if needed.
-    pub(crate) async fn apply_next_wft_from_history(&mut self) -> Result<usize> {
+    /// any events that need to be replayed until caught up to the newest WFT.
+    pub(crate) fn apply_next_wft_from_history(&mut self) -> Result<usize> {
         // If we have already seen the terminal event for the entire workflow in a previous WFT,
         // then we don't need to do anything here, and in fact we need to avoid re-applying the
         // final WFT.
@@ -389,16 +421,34 @@ impl WorkflowMachines {
             return Ok(0);
         }
 
+        // Update observed patches with any that were used in the task
+        if let Some(next_complete) = self
+            .last_history_from_server
+            .peek_next_wft_completed(self.last_processed_event)
+        {
+            (*self.observed_internal_flags)
+                .borrow_mut()
+                .add_from_complete(next_complete);
+        }
+
         let last_handled_wft_started_id = self.current_started_event_id;
-        let events = {
-            let mut evts = self
-                .last_history_from_server
-                .take_next_wft_sequence(last_handled_wft_started_id)
-                .await
-                .map_err(WFMachinesError::HistoryFetchingError)?;
-            // Do not re-process events we have already processed
-            evts.retain(|e| e.event_id > self.last_processed_event);
-            evts
+        let (events, has_final_event) = match self
+            .last_history_from_server
+            .take_next_wft_sequence(last_handled_wft_started_id)
+        {
+            NextWFT::ReplayOver => (vec![], true),
+            NextWFT::WFT(mut evts, has_final_event) => {
+                // Do not re-process events we have already processed
+                evts.retain(|e| e.event_id > self.last_processed_event);
+                (evts, has_final_event)
+            }
+            NextWFT::NeedFetch => {
+                return Err(WFMachinesError::Fatal(
+                    "Need to fetch history events to continue applying workflow task, but this \
+                     should be prevented ahead of time! This is a Core SDK bug."
+                        .to_string(),
+                ));
+            }
         };
         let num_events_to_process = events.len();
 
@@ -414,6 +464,7 @@ impl WorkflowMachines {
             }
         }
 
+        let mut saw_completed = false;
         let mut history = events.into_iter().peekable();
         while let Some(event) = history.next() {
             if event.event_id != self.last_processed_event + 1 {
@@ -424,17 +475,40 @@ impl WorkflowMachines {
             }
             let next_event = history.peek();
             let eid = event.event_id;
-            let etype = event.event_type();
-            self.handle_event(event, next_event.is_some())?;
-            self.last_processed_event = eid;
-            if etype == EventType::WorkflowTaskStarted && next_event.is_none() {
-                break;
+
+            // This definition of replaying here is that we are no longer replaying as soon as we
+            // see new events that have never been seen or produced by the SDK.
+            //
+            // Specifically, replay ends once we have seen the last command-event which was produced
+            // as a result of the last completed WFT. Thus, replay would be false for things like
+            // signals which were received and after the last completion, and thus generated the
+            // current WFT being handled.
+            if self.replaying && has_final_event && saw_completed && !event.is_command_event() {
+                // Replay is finished
+                self.replaying = false;
+            }
+            if event.event_type() == EventType::WorkflowTaskCompleted {
+                saw_completed = true;
             }
+
+            self.handle_event(
+                HistEventData {
+                    event,
+                    replaying: self.replaying,
+                    current_task_is_last_in_history: has_final_event,
+                },
+                next_event.is_some() || !has_final_event,
+            )?;
+            self.last_processed_event = eid;
         }
 
         // Scan through to the next WFT, searching for any patch / la markers, so that we can
         // pre-resolve them.
-        for e in self.last_history_from_server.peek_next_wft_sequence() {
+        let mut wake_las = vec![];
+        for e in self
+            .last_history_from_server
+            .peek_next_wft_sequence(last_handled_wft_started_id)
+        {
             if let Some((patch_id, _)) = e.get_patch_marker_details() {
                 self.encountered_change_markers.insert(
                     patch_id.clone(),
@@ -448,7 +522,30 @@ impl WorkflowMachines {
                         .into(),
                 );
             } else if e.is_local_activity_marker() {
-                self.local_activity_data.process_peekahead_marker(e)?;
+                if let Some(la_dat) = e.clone().into_local_activity_marker_details() {
+                    if let Ok(mk) =
+                        self.get_machine_key(CommandID::LocalActivity(la_dat.marker_dat.seq))
+                    {
+                        wake_las.push((mk, la_dat));
+                    } else {
+                        self.local_activity_data.insert_peeked_marker(la_dat);
+                    }
+                } else {
+                    return Err(WFMachinesError::Fatal(format!(
+                        "Local activity marker was unparsable: {e:?}"
+                    )));
+                }
+            }
+        }
+        for (mk, la_dat) in wake_las {
+            let mach = self.machine_mut(mk);
+            if let Machines::LocalActivityMachine(ref mut lam) = *mach {
+                if lam.will_accept_resolve_marker() {
+                    let resps = lam.try_resolve_with_dat(la_dat.into())?;
+                    self.process_machine_responses(mk, resps)?;
+                } else {
+                    self.local_activity_data.insert_peeked_marker(la_dat);
+                }
             }
         }
 
@@ -467,8 +564,9 @@ impl WorkflowMachines {
     /// event is applied to the machine, which may also return a nondeterminism error if the machine
     /// does not match the expected type. A fatal error may be returned if the machine is in an
     /// invalid state.
-    #[instrument(skip(self, event), fields(event=%event))]
-    fn handle_event(&mut self, event: HistoryEvent, has_next_event: bool) -> Result<()> {
+    #[instrument(skip(self, event_dat), fields(event=%event_dat))]
+    fn handle_event(&mut self, event_dat: HistEventData, has_next_event: bool) -> Result<()> {
+        let event = &event_dat.event;
         if event.is_final_wf_execution_event() {
             self.have_seen_terminal_event = true;
         }
@@ -486,19 +584,10 @@ impl WorkflowMachines {
                 Ok(())
             };
         }
-        if self.replaying
-            && self.current_started_event_id
-                >= self.last_history_from_server.previous_started_event_id
-            && event.event_type() != EventType::WorkflowTaskCompleted
-        {
-            // Replay is finished
-            self.replaying = false;
-        }
         if event.event_type() == EventType::Unspecified || event.attributes.is_none() {
             return if !event.worker_may_ignore {
                 Err(WFMachinesError::Fatal(format!(
-                    "Event type is unspecified! This history is invalid. Event detail: {:?}",
-                    event
+                    "Event type is unspecified! This history is invalid. Event detail: {event:?}"
                 )))
             } else {
                 debug!("Event is ignorable");
@@ -507,7 +596,7 @@ impl WorkflowMachines {
         }
 
         if event.is_command_event() {
-            self.handle_command_event(event)?;
+            self.handle_command_event(event_dat)?;
             return Ok(());
         }
 
@@ -517,7 +606,7 @@ impl WorkflowMachines {
             let maybe_machine = self.machines_by_event_id.remove(&initial_cmd_id);
             match maybe_machine {
                 Some(sm) => {
-                    self.submachine_handle_event(sm, event, has_next_event)?;
+                    self.submachine_handle_event(sm, event_dat)?;
                     // Restore machine if not in it's final state
                     if !self.machine(sm).is_final_state() {
                         self.machines_by_event_id.insert(initial_cmd_id, sm);
@@ -526,13 +615,12 @@ impl WorkflowMachines {
                 None => {
                     return Err(WFMachinesError::Nondeterminism(format!(
                         "During event handling, this event had an initial command ID but we \
-                            could not find a matching command for it: {:?}",
-                        event
+                            could not find a matching command for it: {event:?}"
                     )));
                 }
             }
         } else {
-            self.handle_non_stateful_event(event, has_next_event)?;
+            self.handle_non_stateful_event(event_dat)?;
         }
 
         Ok(())
@@ -547,23 +635,23 @@ impl WorkflowMachines {
     /// The handling consists of verifying that the next command in the commands queue is associated
     /// with a state machine, which is then notified about the event and the command is removed from
     /// the commands queue.
-    fn handle_command_event(&mut self, event: HistoryEvent) -> Result<()> {
+    fn handle_command_event(&mut self, event_dat: HistEventData) -> Result<()> {
+        let event = &event_dat.event;
         if event.is_local_activity_marker() {
             let deets = event.extract_local_activity_marker_data().ok_or_else(|| {
-                WFMachinesError::Fatal(format!("Local activity marker was unparsable: {:?}", event))
+                WFMachinesError::Fatal(format!("Local activity marker was unparsable: {event:?}"))
             })?;
             let cmdid = CommandID::LocalActivity(deets.seq);
             let mkey = self.get_machine_key(cmdid)?;
             if let Machines::LocalActivityMachine(lam) = self.machine(mkey) {
                 if lam.marker_should_get_special_handling()? {
-                    self.submachine_handle_event(mkey, event, false)?;
+                    self.submachine_handle_event(mkey, event_dat)?;
                     return Ok(());
                 }
             } else {
                 return Err(WFMachinesError::Fatal(format!(
                     "Encountered local activity marker but the associated machine was of the \
-                     wrong type! {:?}",
-                    event
+                     wrong type! {event:?}"
                 )));
             }
         }
@@ -573,7 +661,7 @@ impl WorkflowMachines {
         let consumed_cmd = loop {
             if let Some(peek_machine) = self.commands.front() {
                 let mach = self.machine(peek_machine.machine);
-                match change_marker_handling(&event, mach)? {
+                match change_marker_handling(event, mach)? {
                     ChangeMarkerOutcome::SkipEvent => return Ok(()),
                     ChangeMarkerOutcome::SkipCommand => {
                         self.commands.pop_front();
@@ -588,8 +676,7 @@ impl WorkflowMachines {
                 c
             } else {
                 return Err(WFMachinesError::Nondeterminism(format!(
-                    "No command scheduled for event {}",
-                    event
+                    "No command scheduled for event {event}"
                 )));
             };
 
@@ -599,7 +686,7 @@ impl WorkflowMachines {
 
             if !canceled_before_sent {
                 // Feed the machine the event
-                self.submachine_handle_event(command.machine, event, true)?;
+                self.submachine_handle_event(command.machine, event_dat)?;
                 break command;
             }
         };
@@ -612,23 +699,19 @@ impl WorkflowMachines {
         Ok(())
     }
 
-    fn handle_non_stateful_event(
-        &mut self,
-        event: HistoryEvent,
-        has_next_event: bool,
-    ) -> Result<()> {
+    fn handle_non_stateful_event(&mut self, event_dat: HistEventData) -> Result<()> {
         trace!(
-            event = %event,
+            event = %event_dat.event,
             "handling non-stateful event"
         );
-        let event_id = event.event_id;
-        match EventType::from_i32(event.event_type) {
+        let event_id = event_dat.event.event_id;
+        match EventType::from_i32(event_dat.event.event_type) {
             Some(EventType::WorkflowExecutionStarted) => {
                 if let Some(history_event::Attributes::WorkflowExecutionStartedEventAttributes(
                     attrs,
-                )) = event.attributes
+                )) = event_dat.event.attributes
                 {
-                    if let Some(st) = event.event_time.clone() {
+                    if let Some(st) = event_dat.event.event_time.clone() {
                         let as_systime: SystemTime = st.try_into()?;
                         self.workflow_start_time = Some(as_systime);
                         // Set the workflow time to be the event time of the first event, so that
@@ -640,26 +723,25 @@ impl WorkflowMachines {
                     self.drive_me.start(
                         self.workflow_id.clone(),
                         str_to_randomness_seed(&attrs.original_execution_run_id),
-                        event.event_time.unwrap_or_default(),
+                        event_dat.event.event_time.unwrap_or_default(),
                         attrs,
                     );
                 } else {
                     return Err(WFMachinesError::Fatal(format!(
-                        "WorkflowExecutionStarted event did not have appropriate attributes: {}",
-                        event
+                        "WorkflowExecutionStarted event did not have appropriate attributes: {event_dat}"
                     )));
                 }
             }
             Some(EventType::WorkflowTaskScheduled) => {
                 let wf_task_sm = WorkflowTaskMachine::new(self.next_started_event_id);
                 let key = self.all_machines.insert(wf_task_sm.into());
-                self.submachine_handle_event(key, event, has_next_event)?;
+                self.submachine_handle_event(key, event_dat)?;
                 self.machines_by_event_id.insert(event_id, key);
             }
             Some(EventType::WorkflowExecutionSignaled) => {
                 if let Some(history_event::Attributes::WorkflowExecutionSignaledEventAttributes(
                     attrs,
-                )) = event.attributes
+                )) = event_dat.event.attributes
                 {
                     self.drive_me
                         .send_job(workflow_activation::SignalWorkflow::from(attrs).into());
@@ -672,7 +754,7 @@ impl WorkflowMachines {
                     history_event::Attributes::WorkflowExecutionCancelRequestedEventAttributes(
                         attrs,
                     ),
-                ) = event.attributes
+                ) = event_dat.event.attributes
                 {
                     self.drive_me
                         .send_job(workflow_activation::CancelWorkflow::from(attrs).into());
@@ -682,8 +764,7 @@ impl WorkflowMachines {
             }
             _ => {
                 return Err(WFMachinesError::Fatal(format!(
-                    "The event is not a non-stateful event, but we tried to handle it as one: {}",
-                    event
+                    "The event is not a non-stateful event, but we tried to handle it as one: {event_dat}"
                 )));
             }
         }
@@ -700,13 +781,8 @@ impl WorkflowMachines {
 
     /// Wrapper for calling [TemporalStateMachine::handle_event] which appropriately takes action
     /// on the returned machine responses
-    fn submachine_handle_event(
-        &mut self,
-        sm: MachineKey,
-        event: HistoryEvent,
-        has_next_event: bool,
-    ) -> Result<()> {
-        let machine_responses = self.machine_mut(sm).handle_event(event, has_next_event)?;
+    fn submachine_handle_event(&mut self, sm: MachineKey, event: HistEventData) -> Result<()> {
+        let machine_responses = self.machine_mut(sm).handle_event(event)?;
         self.process_machine_responses(sm, machine_responses)?;
         Ok(())
     }
@@ -756,7 +832,7 @@ impl WorkflowMachines {
     ) -> Result<()> {
         let sm = self.machine(smk);
         if !machine_responses.is_empty() {
-            debug!(responses = %machine_responses.display(), machine_name = %sm.name(),
+            trace!(responses = %machine_responses.display(), machine_name = %sm.name(),
                    "Machine produced responses");
         }
         self.process_machine_resps_impl(smk, machine_responses)
@@ -811,8 +887,7 @@ impl WorkflowMachines {
                     }
                     c => {
                         return Err(WFMachinesError::Fatal(format!(
-                        "A machine requested to create a new command of an unsupported type: {:?}",
-                        c
+                        "A machine requested to create a new command of an unsupported type: {c:?}"
                     )))
                     }
                 },
@@ -925,7 +1000,10 @@ impl WorkflowMachines {
                 }
                 WFCommand::AddActivity(attrs) => {
                     let seq = attrs.seq;
-                    self.add_cmd_to_wf_task(new_activity(attrs), CommandID::Activity(seq).into());
+                    self.add_cmd_to_wf_task(
+                        ActivityMachine::new_scheduled(attrs, self.observed_internal_flags.clone()),
+                        CommandID::Activity(seq).into(),
+                    );
                 }
                 WFCommand::AddLocalActivity(attrs) => {
                     let seq = attrs.seq;
@@ -937,8 +1015,7 @@ impl WorkflowMachines {
                     )
                     .map_err(|e| {
                         WFMachinesError::Fatal(format!(
-                            "Invalid schedule local activity request (seq {}): {}",
-                            seq, e
+                            "Invalid schedule local activity request (seq {seq}): {e}"
                         ))
                     })?;
                     let (la, mach_resp) = new_local_activity(
@@ -946,6 +1023,7 @@ impl WorkflowMachines {
                         self.replaying,
                         self.local_activity_data.take_preresolution(seq),
                         self.current_wf_time,
+                        self.observed_internal_flags.clone(),
                     )?;
                     let machkey = self.all_machines.insert(la.into());
                     self.id_to_machine
@@ -1001,7 +1079,10 @@ impl WorkflowMachines {
                 WFCommand::AddChildWorkflow(attrs) => {
                     let seq = attrs.seq;
                     self.add_cmd_to_wf_task(
-                        new_child_workflow(attrs),
+                        ChildWorkflowMachine::new_scheduled(
+                            attrs,
+                            self.observed_internal_flags.clone(),
+                        ),
                         CommandID::ChildWorkflowStart(seq).into(),
                     );
                 }
@@ -1075,7 +1156,7 @@ impl WorkflowMachines {
 
     fn get_machine_key(&self, id: CommandID) -> Result<MachineKey> {
         Ok(*self.id_to_machine.get(&id).ok_or_else(|| {
-            WFMachinesError::Fatal(format!("Missing associated machine for {:?}", id))
+            WFMachinesError::Fatal(format!("Missing associated machine for {id:?}"))
         })?)
     }
 
@@ -1173,9 +1254,8 @@ fn change_marker_handling(event: &HistoryEvent, mach: &Machines) -> Result<Chang
                 return Ok(ChangeMarkerOutcome::SkipEvent);
             }
             return Err(WFMachinesError::Nondeterminism(format!(
-                "Non-deprecated patch marker encountered for change {}, \
-                            but there is no corresponding change command!",
-                patch_name
+                "Non-deprecated patch marker encountered for change {patch_name}, \
+                            but there is no corresponding change command!"
             )));
         }
         // Patch machines themselves may also not *have* matching markers, where non-deprecated