temporalio 0.1.1 → 0.2.0

data/bridge/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs

@@ -1,1525 +0,0 @@
-use super::{
-    workflow_machines::MachineResponse, Cancellable, EventInfo, OnEventWrapper, WFMachinesAdapter,
-    WFMachinesError,
-};
-use crate::{
-    internal_flags::CoreInternalFlags,
-    protosext::{CompleteLocalActivityData, HistoryEventExt, ValidScheduleLA},
-    worker::{
-        workflow::{machines::HistEventData, InternalFlagsRef, OutgoingJob},
-        LocalActivityExecutionResult,
-    },
-};
-use rustfsm::{fsm, MachineError, StateMachine, TransitionResult};
-use std::{
-    convert::TryFrom,
-    time::{Duration, SystemTime},
-};
-use temporal_sdk_core_protos::{
-    constants::LOCAL_ACTIVITY_MARKER_NAME,
-    coresdk::{
-        activity_result::{
-            ActivityResolution, Cancellation, DoBackoff, Failure as ActFail, Success,
-        },
-        common::build_local_activity_marker_details,
-        external_data::LocalActivityMarkerData,
-        workflow_activation::ResolveActivity,
-        workflow_commands::ActivityCancellationType,
-    },
-    temporal::api::{
-        command::v1::{Command, RecordMarkerCommandAttributes},
-        enums::v1::{CommandType, EventType},
-        failure::v1::failure::FailureInfo,
-        history::v1::HistoryEvent,
-    },
-    utilities::TryIntoOrNone,
-};
-
-fsm! {
-    pub(super) name LocalActivityMachine;
-    command LocalActivityCommand;
-    error WFMachinesError;
-    shared_state SharedState;
-
-    // Machine is created in either executing or replaying (referring to whether or not the workflow
-    // is replaying), and then immediately scheduled and transitions to either requesting that lang
-    // execute the activity, or waiting for the marker from history.
-    Executing --(Schedule, shared on_schedule) --> RequestSent;
-    Replaying --(Schedule, on_schedule) --> WaitingMarkerEvent;
-    ReplayingPreResolved --(Schedule, on_schedule) --> WaitingMarkerEventPreResolved;
-
-    // Execution path =============================================================================
-    RequestSent --(HandleResult(ResolveDat), on_handle_result) --> MarkerCommandCreated;
-    // We loop back on RequestSent here because the LA needs to report its result
-    RequestSent --(Cancel, on_cancel_requested) --> RequestSent;
-    // No wait cancels skip waiting for the LA to report the result, but do generate a command
-    // to record the cancel marker
-    RequestSent --(NoWaitCancel(ActivityCancellationType), shared on_no_wait_cancel)
-      --> MarkerCommandCreated;
-
-    MarkerCommandCreated --(CommandRecordMarker, on_command_record_marker) --> ResultNotified;
-
-    ResultNotified --(MarkerRecorded(CompleteLocalActivityData), shared on_marker_recorded)
-      --> MarkerCommandRecorded;
-
-    // Replay path ================================================================================
-    // LAs on the replay path always need to eventually see the marker
-    WaitingMarkerEvent --(MarkerRecorded(CompleteLocalActivityData), shared on_marker_recorded)
-      --> MarkerCommandRecorded;
-    // If we are told to cancel while waiting for the marker, we still need to wait for the marker.
-    WaitingMarkerEvent --(Cancel, on_cancel_requested) --> WaitingMarkerEvent;
-    // Because there could be non-heartbeat WFTs (ex: signals being received) between scheduling
-    // the LA and the marker being recorded, peekahead might not always resolve the LA *before*
-    // scheduling it. This transition accounts for that.
-    WaitingMarkerEvent --(HandleKnownResult(ResolveDat), on_handle_result) --> WaitingMarkerEvent;
-    WaitingMarkerEvent --(NoWaitCancel(ActivityCancellationType),
-                          on_no_wait_cancel) --> WaitingMarkerEvent;
-
-    // It is entirely possible to have started the LA while replaying, only to find that we have
-    // reached a new WFT and there still was no marker. In such cases we need to execute the LA.
-    // This can easily happen if upon first execution, the worker does WFT heartbeating but then
-    // dies for some reason.
-    WaitingMarkerEvent --(StartedNonReplayWFT, shared on_started_non_replay_wft) --> RequestSent;
-
-    // If the activity is pre resolved we still expect to see marker recorded event at some point,
-    // even though we already resolved the activity.
-    WaitingMarkerEventPreResolved --(MarkerRecorded(CompleteLocalActivityData),
-                                     shared on_marker_recorded) --> MarkerCommandRecorded;
-    // Ignore cancellations when waiting for the marker after being pre-resolved
-    WaitingMarkerEventPreResolved --(Cancel) --> WaitingMarkerEventPreResolved;
-    WaitingMarkerEventPreResolved --(NoWaitCancel(ActivityCancellationType))
-                                     --> WaitingMarkerEventPreResolved;
-
-    // Ignore cancellation in final state
-    MarkerCommandRecorded --(Cancel, on_cancel_requested) --> MarkerCommandRecorded;
-    MarkerCommandRecorded --(NoWaitCancel(ActivityCancellationType),
-                             on_no_wait_cancel) --> MarkerCommandRecorded;
-
-    // LAs reporting status after they've handled their result can simply be ignored. We could
-    // optimize this away higher up but that feels very overkill.
-    MarkerCommandCreated --(HandleResult(ResolveDat)) --> MarkerCommandCreated;
-    ResultNotified --(HandleResult(ResolveDat)) --> ResultNotified;
-    MarkerCommandRecorded --(HandleResult(ResolveDat)) --> MarkerCommandRecorded;
-}
-
-#[derive(Debug, Clone)]
-pub(super) struct ResolveDat {
-    pub(super) result: LocalActivityExecutionResult,
-    pub(super) complete_time: Option<SystemTime>,
-    pub(super) attempt: u32,
-    pub(super) backoff: Option<prost_types::Duration>,
-    pub(super) original_schedule_time: Option<SystemTime>,
-}
-
-impl From<CompleteLocalActivityData> for ResolveDat {
-    fn from(d: CompleteLocalActivityData) -> Self {
-        ResolveDat {
-            result: match d.result {
-                Ok(res) => LocalActivityExecutionResult::Completed(Success { result: Some(res) }),
-                Err(fail) => {
-                    if matches!(fail.failure_info, Some(FailureInfo::CanceledFailureInfo(_))) {
-                        LocalActivityExecutionResult::Cancelled(Cancellation {
-                            failure: Some(fail),
-                        })
-                    } else {
-                        LocalActivityExecutionResult::Failed(ActFail {
-                            failure: Some(fail),
-                        })
-                    }
-                }
-            },
-            complete_time: d.marker_dat.complete_time.try_into_or_none(),
-            attempt: d.marker_dat.attempt,
-            backoff: d.marker_dat.backoff,
-            original_schedule_time: d.marker_dat.original_schedule_time.try_into_or_none(),
-        }
-    }
-}
-
-/// Creates a new local activity state machine & immediately schedules the local activity for
-/// execution. No command is produced immediately to be sent to the server, as the local activity
-/// must resolve before we send a record marker command. A [MachineResponse] may be produced,
-/// to queue the LA for execution if it needs to be.
-pub(super) fn new_local_activity(
-    mut attrs: ValidScheduleLA,
-    replaying_when_invoked: bool,
-    maybe_pre_resolved: Option<ResolveDat>,
-    wf_time: Option<SystemTime>,
-    internal_flags: InternalFlagsRef,
-) -> Result<(LocalActivityMachine, Vec<MachineResponse>), WFMachinesError> {
-    let initial_state = if replaying_when_invoked {
-        if let Some(dat) = maybe_pre_resolved {
-            ReplayingPreResolved { dat }.into()
-        } else {
-            Replaying {}.into()
-        }
-    } else {
-        if maybe_pre_resolved.is_some() {
-            return Err(WFMachinesError::Nondeterminism(
-                "Local activity cannot be created as pre-resolved while not replaying".to_string(),
-            ));
-        }
-        Executing {}.into()
-    };
-
-    // If the scheduled LA doesn't already have an "original" schedule time, assign one.
-    attrs
-        .original_schedule_time
-        .get_or_insert(SystemTime::now());
-
-    let mut machine = LocalActivityMachine {
-        state: initial_state,
-        shared_state: SharedState {
-            attrs,
-            replaying_when_invoked,
-            wf_time_when_started: wf_time,
-            internal_flags,
-        },
-    };
-
-    let mut res = OnEventWrapper::on_event_mut(&mut machine, LocalActivityMachineEvents::Schedule)
-        .expect("Scheduling local activities doesn't fail");
-    let mr = if let Some(res) = res.pop() {
-        machine
-            .adapt_response(res, None)
-            .expect("Adapting LA schedule response doesn't fail")
-    } else {
-        vec![]
-    };
-    Ok((machine, mr))
-}
-
-impl LocalActivityMachine {
-    /// Is called to check if, while handling the LA marker event, we should avoid doing normal
-    /// command-event processing - instead simply applying the event to this machine and then
-    /// skipping over the rest. If this machine is in the `ResultNotified` state, that means
-    /// command handling should proceed as normal (ie: The command needs to be matched and removed).
-    /// The other valid states to make this check in are the `WaitingMarkerEvent[PreResolved]`
-    /// states, which will return true.
-    ///
-    /// Attempting the check in any other state likely means a bug in the SDK.
-    pub(super) fn marker_should_get_special_handling(&self) -> Result<bool, WFMachinesError> {
-        match &self.state {
-            LocalActivityMachineState::ResultNotified(_) => Ok(false),
-            LocalActivityMachineState::WaitingMarkerEvent(_) => Ok(true),
-            LocalActivityMachineState::WaitingMarkerEventPreResolved(_) => Ok(true),
-            _ => Err(WFMachinesError::Fatal(format!(
-                "Attempted to check for LA marker handling in invalid state {}",
-                self.state
-            ))),
-        }
-    }
-
-    /// Returns true if the machine will willingly accept data from a marker in its current state.
-    /// IE: Calling [Self::try_resolve_with_dat] makes sense.
-    pub(super) fn will_accept_resolve_marker(&self) -> bool {
-        matches!(self.state, LocalActivityMachineState::WaitingMarkerEvent(_))
-    }
-
-    /// Must be called if the workflow encounters a non-replay workflow task
-    pub(super) fn encountered_non_replay_wft(
-        &mut self,
-    ) -> Result<Vec<MachineResponse>, WFMachinesError> {
-        // This only applies to the waiting-for-marker state. It can safely be ignored in the others
-        if !matches!(
-            self.state(),
-            LocalActivityMachineState::WaitingMarkerEvent(_)
-        ) {
-            return Ok(vec![]);
-        }
-
-        let mut res =
-            OnEventWrapper::on_event_mut(self, LocalActivityMachineEvents::StartedNonReplayWFT)
-                .map_err(|e| match e {
-                    MachineError::InvalidTransition => WFMachinesError::Fatal(format!(
-                        "Invalid transition while notifying local activity (seq {})\
-                         of non-replay-wft-started in {}",
-                        self.shared_state.attrs.seq,
-                        self.state(),
-                    )),
-                    MachineError::Underlying(e) => e,
-                })?;
-        let res = res.pop().expect("Always produces one response");
-        Ok(self
-            .adapt_response(res, None)
-            .expect("Adapting LA wft-non-replay response doesn't fail"))
-    }
-
-    /// Attempt to resolve the local activity with a result from execution (not from history)
-    pub(super) fn try_resolve(
-        &mut self,
-        result: LocalActivityExecutionResult,
-        runtime: Duration,
-        attempt: u32,
-        backoff: Option<prost_types::Duration>,
-        original_schedule_time: Option<SystemTime>,
-    ) -> Result<Vec<MachineResponse>, WFMachinesError> {
-        self._try_resolve(
-            ResolveDat {
-                result,
-                complete_time: self.shared_state.wf_time_when_started.map(|t| t + runtime),
-                attempt,
-                backoff,
-                original_schedule_time,
-            },
-            false,
-        )
-    }
-
-    /// Attempt to resolve the local activity with already known data, ex pre-resolved data
-    pub(super) fn try_resolve_with_dat(
-        &mut self,
-        dat: ResolveDat,
-    ) -> Result<Vec<MachineResponse>, WFMachinesError> {
-        self._try_resolve(dat, true)
-    }
-
-    fn _try_resolve(
-        &mut self,
-        dat: ResolveDat,
-        from_marker: bool,
-    ) -> Result<Vec<MachineResponse>, WFMachinesError> {
-        let evt = if from_marker {
-            LocalActivityMachineEvents::HandleKnownResult(dat)
-        } else {
-            LocalActivityMachineEvents::HandleResult(dat)
-        };
-        let res = OnEventWrapper::on_event_mut(self, evt).map_err(|e| match e {
-            MachineError::InvalidTransition => WFMachinesError::Fatal(format!(
-                "Invalid transition resolving local activity (seq {}, from marker: {}) in {}",
-                self.shared_state.attrs.seq,
-                from_marker,
-                self.state(),
-            )),
-            MachineError::Underlying(e) => e,
-        })?;
-
-        Ok(res
-            .into_iter()
-            .flat_map(|res| {
-                self.adapt_response(res, None)
-                    .expect("Adapting LA resolve response doesn't fail")
-            })
-            .collect())
-    }
-}
-
-#[derive(Clone)]
-pub(super) struct SharedState {
-    attrs: ValidScheduleLA,
-    replaying_when_invoked: bool,
-    wf_time_when_started: Option<SystemTime>,
-    internal_flags: InternalFlagsRef,
-}
-
-impl SharedState {
-    fn produce_no_wait_cancel_resolve_dat(&self) -> ResolveDat {
-        ResolveDat {
-            result: LocalActivityExecutionResult::empty_cancel(),
-            // Just don't provide a complete time, which means try-cancel/abandon cancels won't
-            // advance the clock. Seems like that's fine, since you can only cancel after awaiting
-            // some other command, which would have appropriately advanced the clock anyway.
-            complete_time: None,
-            attempt: self.attrs.attempt,
-            backoff: None,
-            original_schedule_time: self.attrs.original_schedule_time,
-        }
-    }
-}
-
-#[allow(clippy::large_enum_variant)]
-#[derive(Debug, derive_more::Display)]
-pub(super) enum LocalActivityCommand {
-    RequestActivityExecution(ValidScheduleLA),
-    #[display(fmt = "Resolved")]
-    Resolved(ResolveDat),
-    /// The fake marker is used to avoid special casing marker recorded event handling.
-    /// If we didn't have the fake marker, there would be no "outgoing command" to match
-    /// against the event. This way there is, but the command never will be issued to
-    /// server because it is understood to be meaningless.
-    #[display(fmt = "FakeMarker")]
-    FakeMarker,
-    /// Indicate we want to cancel an LA that is currently executing, or look up if we have
-    /// processed a marker with resolution data since the machine was constructed.
-    #[display(fmt = "Cancel")]
-    RequestCancel,
-}
-
-#[derive(Default, Clone)]
-pub(super) struct Executing {}
-
-impl Executing {
-    pub(super) fn on_schedule(
-        self,
-        dat: SharedState,
-    ) -> LocalActivityMachineTransition<RequestSent> {
-        TransitionResult::commands([LocalActivityCommand::RequestActivityExecution(dat.attrs)])
-    }
-}
-
-#[derive(Clone, Copy, PartialEq, Eq)]
-enum ResultType {
-    Completed,
-    Cancelled,
-    Failed,
-}
-#[derive(Clone)]
-pub(super) struct MarkerCommandCreated {
-    result_type: ResultType,
-}
-impl From<MarkerCommandCreated> for ResultNotified {
-    fn from(mc: MarkerCommandCreated) -> Self {
-        Self {
-            result_type: mc.result_type,
-        }
-    }
-}
-
-impl MarkerCommandCreated {
-    pub(super) fn on_command_record_marker(self) -> LocalActivityMachineTransition<ResultNotified> {
-        TransitionResult::from(self)
-    }
-}
-
-#[derive(Default, Clone)]
-pub(super) struct MarkerCommandRecorded {}
-impl MarkerCommandRecorded {
-    fn on_cancel_requested(self) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
-        // We still must issue a cancel request even if this command is resolved, because if it
-        // failed and we are backing off locally, we must tell the LA dispatcher to quit retrying.
-        TransitionResult::ok([LocalActivityCommand::RequestCancel], self)
-    }
-
-    fn on_no_wait_cancel(
-        self,
-        cancel_type: ActivityCancellationType,
-    ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
-        if matches!(cancel_type, ActivityCancellationType::TryCancel) {
-            // We still must issue a cancel request even if this command is resolved, because if it
-            // failed and we are backing off locally, we must tell the LA dispatcher to quit
-            // retrying.
-            TransitionResult::ok(
-                [LocalActivityCommand::RequestCancel],
-                MarkerCommandRecorded::default(),
-            )
-        } else {
-            TransitionResult::default()
-        }
-    }
-}
-
-#[derive(Default, Clone)]
-pub(super) struct Replaying {}
-impl Replaying {
-    pub(super) fn on_schedule(self) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
-        TransitionResult::ok(
-            [],
-            WaitingMarkerEvent {
-                already_resolved: false,
-            },
-        )
-    }
-}
-
-#[derive(Clone)]
-pub(super) struct ReplayingPreResolved {
-    dat: ResolveDat,
-}
-impl ReplayingPreResolved {
-    pub(super) fn on_schedule(
-        self,
-    ) -> LocalActivityMachineTransition<WaitingMarkerEventPreResolved> {
-        TransitionResult::ok(
-            [
-                LocalActivityCommand::FakeMarker,
-                LocalActivityCommand::Resolved(self.dat),
-            ],
-            WaitingMarkerEventPreResolved {},
-        )
-    }
-}
-
-#[derive(Default, Clone)]
-pub(super) struct RequestSent {}
-
-impl RequestSent {
-    fn on_handle_result(
-        self,
-        dat: ResolveDat,
-    ) -> LocalActivityMachineTransition<MarkerCommandCreated> {
-        let result_type = match &dat.result {
-            LocalActivityExecutionResult::Completed(_) => ResultType::Completed,
-            LocalActivityExecutionResult::Failed(_) => ResultType::Failed,
-            LocalActivityExecutionResult::TimedOut(_) => ResultType::Failed,
-            LocalActivityExecutionResult::Cancelled { .. } => ResultType::Cancelled,
-        };
-        let new_state = MarkerCommandCreated { result_type };
-        TransitionResult::ok([LocalActivityCommand::Resolved(dat)], new_state)
-    }
-
-    fn on_cancel_requested(self) -> LocalActivityMachineTransition<RequestSent> {
-        TransitionResult::ok([LocalActivityCommand::RequestCancel], self)
-    }
-
-    fn on_no_wait_cancel(
-        self,
-        shared: SharedState,
-        cancel_type: ActivityCancellationType,
-    ) -> LocalActivityMachineTransition<MarkerCommandCreated> {
-        let mut cmds = vec![];
-        if matches!(cancel_type, ActivityCancellationType::TryCancel) {
-            // For try-cancels also request the cancel
-            cmds.push(LocalActivityCommand::RequestCancel);
-        }
-        // Immediately resolve
-        cmds.push(LocalActivityCommand::Resolved(
-            shared.produce_no_wait_cancel_resolve_dat(),
-        ));
-        TransitionResult::ok(
-            cmds,
-            MarkerCommandCreated {
-                result_type: ResultType::Cancelled,
-            },
-        )
-    }
-}
-
-macro_rules! verify_marker_dat {
-    ($shared:expr, $dat:expr, $ok_expr:expr) => {
-        if let Err(err) = verify_marker_data_matches($shared, $dat) {
-            TransitionResult::Err(err)
-        } else {
-            $ok_expr
-        }
-    };
-}
-
-#[derive(Clone)]
-pub(super) struct ResultNotified {
-    result_type: ResultType,
-}
-
-impl ResultNotified {
-    pub(super) fn on_marker_recorded(
-        self,
-        shared: SharedState,
-        dat: CompleteLocalActivityData,
-    ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
-        if self.result_type == ResultType::Completed && dat.result.is_err() {
-            return TransitionResult::Err(WFMachinesError::Nondeterminism(format!(
-                "Local activity (seq {}) completed successfully locally, but history said \
-                 it failed!",
-                shared.attrs.seq
-            )));
-        } else if self.result_type == ResultType::Failed && dat.result.is_ok() {
-            return TransitionResult::Err(WFMachinesError::Nondeterminism(format!(
-                "Local activity (seq {}) failed locally, but history said it completed!",
-                shared.attrs.seq
-            )));
-        }
-        verify_marker_dat!(&shared, &dat, TransitionResult::default())
-    }
-}
-
-#[derive(Default, Clone)]
-pub(super) struct WaitingMarkerEvent {
-    already_resolved: bool,
-}
-
-impl WaitingMarkerEvent {
-    pub(super) fn on_marker_recorded(
-        self,
-        shared: SharedState,
-        dat: CompleteLocalActivityData,
-    ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
-        verify_marker_dat!(
-            &shared,
-            &dat,
-            TransitionResult::commands(if self.already_resolved {
-                vec![]
-            } else {
-                vec![LocalActivityCommand::Resolved(dat.into())]
-            })
-        )
-    }
-    fn on_handle_result(
-        self,
-        dat: ResolveDat,
-    ) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
-        TransitionResult::ok(
-            [LocalActivityCommand::Resolved(dat)],
-            WaitingMarkerEvent {
-                already_resolved: true,
-            },
-        )
-    }
-    pub(super) fn on_started_non_replay_wft(
-        self,
-        mut dat: SharedState,
-    ) -> LocalActivityMachineTransition<RequestSent> {
-        // We aren't really "replaying" anymore for our purposes, and want to record the marker.
-        dat.replaying_when_invoked = false;
-        TransitionResult::ok_shared(
-            [LocalActivityCommand::RequestActivityExecution(
-                dat.attrs.clone(),
-            )],
-            RequestSent::default(),
-            dat,
-        )
-    }
-
-    fn on_cancel_requested(self) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
-        // We still "request a cancel" even though we know the local activity should not be running
-        // because the data might be in the pre-resolved list.
-        TransitionResult::ok([LocalActivityCommand::RequestCancel], self)
-    }
-
-    fn on_no_wait_cancel(
-        self,
-        _: ActivityCancellationType,
-    ) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
-        // Markers are always recorded when cancelling, so this is the same as a normal cancel on
-        // the replay path
-        self.on_cancel_requested()
-    }
-}
-
-#[derive(Default, Clone)]
-pub(super) struct WaitingMarkerEventPreResolved {}
-impl WaitingMarkerEventPreResolved {
-    pub(super) fn on_marker_recorded(
-        self,
-        shared: SharedState,
-        dat: CompleteLocalActivityData,
-    ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
-        verify_marker_dat!(&shared, &dat, TransitionResult::default())
-    }
-}
-
-impl Cancellable for LocalActivityMachine {
-    fn cancel(&mut self) -> Result<Vec<MachineResponse>, MachineError<Self::Error>> {
-        let event = match self.shared_state.attrs.cancellation_type {
-            ct @ ActivityCancellationType::TryCancel | ct @ ActivityCancellationType::Abandon => {
-                LocalActivityMachineEvents::NoWaitCancel(ct)
-            }
-            _ => LocalActivityMachineEvents::Cancel,
-        };
-        let cmds = OnEventWrapper::on_event_mut(self, event)?;
-        let mach_resps = cmds
-            .into_iter()
-            .map(|mc| self.adapt_response(mc, None))
-            .collect::<Result<Vec<_>, _>>()?
-            .into_iter()
-            .flatten()
-            .collect();
-        Ok(mach_resps)
-    }
-
-    fn was_cancelled_before_sent_to_server(&self) -> bool {
-        // This needs to always be false because for the situation where we cancel in the same WFT,
-        // no command of any kind is created and no LA request is queued. Otherwise, the command we
-        // create to record a cancel marker *needs* to be sent to the server still, which returning
-        // true here would prevent.
-        false
-    }
-}
-
-#[derive(Default, Clone)]
-pub(super) struct Abandoned {}
-
-impl WFMachinesAdapter for LocalActivityMachine {
-    fn adapt_response(
-        &self,
-        my_command: Self::Command,
-        _event_info: Option<EventInfo>,
-    ) -> Result<Vec<MachineResponse>, WFMachinesError> {
-        match my_command {
-            LocalActivityCommand::RequestActivityExecution(act) => {
-                Ok(vec![MachineResponse::QueueLocalActivity(act)])
-            }
-            LocalActivityCommand::Resolved(ResolveDat {
-                result,
-                complete_time,
-                attempt,
-                backoff,
-                original_schedule_time,
-            }) => {
-                let mut maybe_ok_result = None;
-                let mut maybe_failure = None;
-                // Only issue record marker commands if we weren't replaying
-                let record_marker = !self.shared_state.replaying_when_invoked;
-                let mut will_not_run_again = false;
-                match result.clone() {
-                    LocalActivityExecutionResult::Completed(suc) => {
-                        maybe_ok_result = suc.result;
-                    }
-                    LocalActivityExecutionResult::Failed(fail) => {
-                        maybe_failure = fail.failure;
-                    }
-                    LocalActivityExecutionResult::Cancelled(Cancellation { failure })
-                    | LocalActivityExecutionResult::TimedOut(ActFail { failure }) => {
-                        will_not_run_again = true;
-                        maybe_failure = failure;
-                    }
-                };
-                let resolution = if let Some(b) = backoff.as_ref() {
-                    ActivityResolution {
-                        status: Some(
-                            DoBackoff {
-                                attempt: attempt + 1,
-                                backoff_duration: Some(b.clone()),
-                                original_schedule_time: original_schedule_time.map(Into::into),
-                            }
-                            .into(),
-                        ),
-                    }
-                } else {
-                    result.into()
-                };
-                let mut responses = vec![
-                    MachineResponse::PushWFJob(OutgoingJob {
-                        variant: ResolveActivity {
-                            seq: self.shared_state.attrs.seq,
-                            result: Some(resolution),
-                        }
-                        .into(),
-                        is_la_resolution: true,
-                    }),
-                    MachineResponse::UpdateWFTime(complete_time),
-                ];
-
-                // Cancel-resolves of abandoned activities must be explicitly dropped from tracking
-                // to avoid unnecessary WFT heartbeating.
-                if will_not_run_again
-                    && matches!(
-                        self.shared_state.attrs.cancellation_type,
-                        ActivityCancellationType::Abandon
-                    )
-                {
-                    responses.push(MachineResponse::AbandonLocalActivity(
-                        self.shared_state.attrs.seq,
-                    ));
-                }
-
-                if record_marker {
-                    let marker_data = RecordMarkerCommandAttributes {
-                        marker_name: LOCAL_ACTIVITY_MARKER_NAME.to_string(),
-                        details: build_local_activity_marker_details(
-                            LocalActivityMarkerData {
-                                seq: self.shared_state.attrs.seq,
-                                attempt,
-                                activity_id: self.shared_state.attrs.activity_id.clone(),
-                                activity_type: self.shared_state.attrs.activity_type.clone(),
-                                complete_time: complete_time.map(Into::into),
-                                backoff,
-                                original_schedule_time: original_schedule_time.map(Into::into),
-                            },
-                            maybe_ok_result,
-                        ),
-                        header: None,
-                        failure: maybe_failure,
-                    };
-                    responses.push(MachineResponse::IssueNewCommand(Command {
-                        command_type: CommandType::RecordMarker as i32,
-                        attributes: Some(marker_data.into()),
-                    }));
-                }
-                Ok(responses)
-            }
-            LocalActivityCommand::FakeMarker => {
-                // See docs for `FakeMarker` for more
-                Ok(vec![MachineResponse::IssueFakeLocalActivityMarker(
-                    self.shared_state.attrs.seq,
-                )])
-            }
-            LocalActivityCommand::RequestCancel => {
-                Ok(vec![MachineResponse::RequestCancelLocalActivity(
-                    self.shared_state.attrs.seq,
-                )])
-            }
-        }
-    }
-
-    fn matches_event(&self, event: &HistoryEvent) -> bool {
-        event.is_local_activity_marker()
-    }
-}
-
-impl TryFrom<CommandType> for LocalActivityMachineEvents {
-    type Error = ();
-
-    fn try_from(c: CommandType) -> Result<Self, Self::Error> {
-        Ok(match c {
-            CommandType::RecordMarker => Self::CommandRecordMarker,
-            _ => return Err(()),
-        })
-    }
-}
-
-impl TryFrom<HistEventData> for LocalActivityMachineEvents {
-    type Error = WFMachinesError;
-
-    fn try_from(e: HistEventData) -> Result<Self, Self::Error> {
-        let e = e.event;
-        if e.event_type() != EventType::MarkerRecorded {
-            return Err(WFMachinesError::Nondeterminism(format!(
-                "Local activity machine cannot handle this event: {e}"
-            )));
-        }
-
-        match e.into_local_activity_marker_details() {
-            Some(marker_dat) => Ok(LocalActivityMachineEvents::MarkerRecorded(marker_dat)),
-            _ => Err(WFMachinesError::Nondeterminism(
-                "Local activity machine encountered an unparsable marker".to_string(),
-            )),
-        }
-    }
-}
-
-fn verify_marker_data_matches(
-    shared: &SharedState,
-    dat: &CompleteLocalActivityData,
-) -> Result<(), WFMachinesError> {
-    if shared.attrs.seq != dat.marker_dat.seq {
-        return Err(WFMachinesError::Nondeterminism(format!(
-            "Local activity marker data has sequence number {} but matched against LA \
-            command with sequence number {}",
-            dat.marker_dat.seq, shared.attrs.seq
-        )));
-    }
-    // Here we use whether or not we were replaying when we _first invoked_ the LA, because we
-    // are always replaying when we see the marker recorded event, and that would make this check
-    // a bit pointless.
-    if shared.internal_flags.borrow_mut().try_use(
-        CoreInternalFlags::IdAndTypeDeterminismChecks,
-        !shared.replaying_when_invoked,
-    ) {
-        if dat.marker_dat.activity_id != shared.attrs.activity_id {
-            return Err(WFMachinesError::Nondeterminism(format!(
-                "Activity id of recorded marker '{}' does not \
-                 match activity id of local activity command '{}'",
-                dat.marker_dat.activity_id, shared.attrs.activity_id
-            )));
-        }
-        if dat.marker_dat.activity_type != shared.attrs.activity_type {
-            return Err(WFMachinesError::Nondeterminism(format!(
-                "Activity type of recorded marker '{}' does not \
-                 match activity type of local activity command '{}'",
-                dat.marker_dat.activity_type, shared.attrs.activity_type
-            )));
-        }
-    }
-
-    Ok(())
-}
-
-impl From<LocalActivityExecutionResult> for ActivityResolution {
-    fn from(lar: LocalActivityExecutionResult) -> Self {
-        match lar {
-            LocalActivityExecutionResult::Completed(c) => ActivityResolution {
-                status: Some(c.into()),
-            },
-            LocalActivityExecutionResult::Failed(f) | LocalActivityExecutionResult::TimedOut(f) => {
-                ActivityResolution {
-                    status: Some(f.into()),
-                }
-            }
-            LocalActivityExecutionResult::Cancelled(cancel) => ActivityResolution {
-                status: Some(cancel.into()),
-            },
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        replay::TestHistoryBuilder, test_help::canned_histories, worker::workflow::ManagedWFFunc,
-    };
-    use rstest::rstest;
-    use std::time::Duration;
-    use temporal_sdk::{
-        CancellableFuture, LocalActivityOptions, WfContext, WorkflowFunction, WorkflowResult,
-    };
-    use temporal_sdk_core_protos::{
-        coresdk::{
-            activity_result::ActivityExecutionResult,
-            workflow_activation::{workflow_activation_job, WorkflowActivationJob},
-            workflow_commands::ActivityCancellationType::WaitCancellationCompleted,
-        },
-        temporal::api::{
-            command::v1::command, enums::v1::WorkflowTaskFailedCause, failure::v1::Failure,
-        },
-        DEFAULT_ACTIVITY_TYPE,
-    };
-
-    async fn la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            ..Default::default()
-        })
-        .await;
-        Ok(().into())
-    }
-
-    #[rstest]
-    #[case::incremental(false, true)]
-    #[case::replay(true, true)]
-    #[case::incremental_fail(false, false)]
-    #[case::replay_fail(true, false)]
-    #[tokio::test]
-    async fn one_la_success(#[case] replay: bool, #[case] completes_ok: bool) {
-        let func = WorkflowFunction::new(la_wf);
-        let activity_id = "1";
-        let mut t = TestHistoryBuilder::default();
-        t.add_by_type(EventType::WorkflowExecutionStarted);
-        t.add_full_wf_task();
-        if completes_ok {
-            t.add_local_activity_result_marker(1, activity_id, b"hi".into());
-        } else {
-            t.add_local_activity_fail_marker(
-                1,
-                activity_id,
-                Failure::application_failure("I failed".to_string(), false),
-            );
-        }
-        t.add_workflow_execution_completed();
-
-        let histinfo = if replay {
-            t.get_full_history_info().unwrap().into()
-        } else {
-            t.get_history_info(1).unwrap().into()
-        };
-        let mut wfm = ManagedWFFunc::new_from_update(histinfo, func, vec![]);
-
-        // First activation will have no server commands. Activity will be put into the activity
-        // queue locally
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 0);
-
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        if !replay {
-            assert_eq!(ready_to_execute_las.len(), 1);
-        } else {
-            assert_eq!(ready_to_execute_las.len(), 0);
-        }
-
-        if !replay {
-            if completes_ok {
-                wfm.complete_local_activity(1, ActivityExecutionResult::ok(b"hi".into()))
-                    .unwrap();
-            } else {
-                wfm.complete_local_activity(
-                    1,
-                    ActivityExecutionResult::fail(Failure {
-                        message: "I failed".to_string(),
-                        ..Default::default()
-                    }),
-                )
-                .unwrap();
-            }
-        }
-
-        // Now the next activation will unblock the local activity
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        if replay {
-            assert_eq!(commands.len(), 1);
-            assert_eq!(
-                commands[0].command_type,
-                CommandType::CompleteWorkflowExecution as i32
-            );
-        } else {
-            assert_eq!(commands.len(), 2);
-            assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-            if completes_ok {
-                assert_matches!(
-                    commands[0].attributes.as_ref().unwrap(),
-                    command::Attributes::RecordMarkerCommandAttributes(
-                        RecordMarkerCommandAttributes { failure: None, .. }
-                    )
-                );
-            } else {
-                assert_matches!(
-                    commands[0].attributes.as_ref().unwrap(),
-                    command::Attributes::RecordMarkerCommandAttributes(
-                        RecordMarkerCommandAttributes {
-                            failure: Some(_),
-                            ..
-                        }
-                    )
-                );
-            }
-            assert_eq!(
-                commands[1].command_type,
-                CommandType::CompleteWorkflowExecution as i32
-            );
-        }
-
-        if !replay {
-            wfm.new_history(t.get_full_history_info().unwrap().into())
-                .await
-                .unwrap();
-        }
-        assert_eq!(wfm.drain_queued_local_activities().len(), 0);
-        assert_eq!(wfm.get_next_activation().await.unwrap().jobs.len(), 0);
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 0);
-
-        wfm.shutdown().await.unwrap();
-    }
-
-    async fn two_la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            ..Default::default()
-        })
-        .await;
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            ..Default::default()
-        })
-        .await;
-        Ok(().into())
-    }
-
-    #[rstest]
-    #[case::incremental(false)]
-    #[case::replay(true)]
-    #[tokio::test]
-    async fn two_sequential_las(#[case] replay: bool) {
-        let func = WorkflowFunction::new(two_la_wf);
-        let t = canned_histories::two_local_activities_one_wft(false);
-        let histinfo = if replay {
-            t.get_full_history_info().unwrap().into()
-        } else {
-            t.get_history_info(1).unwrap().into()
-        };
-        let mut wfm = ManagedWFFunc::new_from_update(histinfo, func, vec![]);
-
-        // First activation will have no server commands. Activity will be put into the activity
-        // queue locally
-        let act = wfm.get_next_activation().await.unwrap();
-        let first_act_ts = act.timestamp.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 0);
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        let num_queued = usize::from(!replay);
-        assert_eq!(ready_to_execute_las.len(), num_queued);
-
-        if !replay {
-            wfm.complete_local_activity(1, ActivityExecutionResult::ok(b"Resolved".into()))
-                .unwrap();
-        }
-
-        let act = wfm.get_next_activation().await.unwrap();
-        // Verify LAs advance time (they take 1s in this test)
-        assert_eq!(act.timestamp.unwrap().seconds, first_act_ts.seconds + 1);
-        assert_matches!(
-            act.jobs.as_slice(),
-            [WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::ResolveActivity(ra))
-            }] => assert_eq!(ra.seq, 1)
-        );
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        if !replay {
-            assert_eq!(ready_to_execute_las.len(), 1);
-        } else {
-            assert_eq!(ready_to_execute_las.len(), 0);
-        }
-
-        if !replay {
-            wfm.complete_local_activity(2, ActivityExecutionResult::ok(b"Resolved".into()))
-                .unwrap();
-        }
-
-        let act = wfm.get_next_activation().await.unwrap();
-        assert_eq!(act.timestamp.unwrap().seconds, first_act_ts.seconds + 2);
-        assert_matches!(
-            act.jobs.as_slice(),
-            [WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::ResolveActivity(ra))
-            }] => assert_eq!(ra.seq, 2)
-        );
-        let commands = wfm.get_server_commands().commands;
-        if replay {
-            assert_eq!(commands.len(), 1);
-            assert_eq!(
-                commands[0].command_type,
-                CommandType::CompleteWorkflowExecution as i32
-            );
-        } else {
-            assert_eq!(commands.len(), 3);
-            assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-            assert_eq!(commands[1].command_type, CommandType::RecordMarker as i32);
-            assert_eq!(
-                commands[2].command_type,
-                CommandType::CompleteWorkflowExecution as i32
-            );
-        }
-
-        if !replay {
-            wfm.new_history(t.get_full_history_info().unwrap().into())
-                .await
-                .unwrap();
-        }
-        assert_eq!(wfm.get_next_activation().await.unwrap().jobs.len(), 0);
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 0);
-
-        wfm.shutdown().await.unwrap();
-    }
-
-    async fn two_la_wf_parallel(ctx: WfContext) -> WorkflowResult<()> {
-        tokio::join!(
-            ctx.local_activity(LocalActivityOptions {
-                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-                ..Default::default()
-            }),
-            ctx.local_activity(LocalActivityOptions {
-                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-                ..Default::default()
-            })
-        );
-        Ok(().into())
-    }
-
-    #[rstest]
-    #[case::incremental(false)]
-    #[case::replay(true)]
-    #[tokio::test]
-    async fn two_parallel_las(#[case] replay: bool) {
-        let func = WorkflowFunction::new(two_la_wf_parallel);
-        let t = canned_histories::two_local_activities_one_wft(true);
-        let histinfo = if replay {
-            t.get_full_history_info().unwrap().into()
-        } else {
-            t.get_history_info(1).unwrap().into()
-        };
-        let mut wfm = ManagedWFFunc::new_from_update(histinfo, func, vec![]);
-
-        // First activation will have no server commands. Activity(ies) will be put into the queue
-        // for execution
-        let act = wfm.get_next_activation().await.unwrap();
-        let first_act_ts = act.timestamp.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 0);
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        let num_queued = if !replay { 2 } else { 0 };
-        assert_eq!(ready_to_execute_las.len(), num_queued);
-
-        if !replay {
-            wfm.complete_local_activity(1, ActivityExecutionResult::ok(b"Resolved".into()))
-                .unwrap();
-            wfm.complete_local_activity(2, ActivityExecutionResult::ok(b"Resolved".into()))
-                .unwrap();
-        }
-
-        let act = wfm.get_next_activation().await.unwrap();
-        assert_eq!(act.timestamp.unwrap().seconds, first_act_ts.seconds + 1);
-        assert_matches!(
-            act.jobs.as_slice(),
-            [WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::ResolveActivity(ra))
-            },
-            WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::ResolveActivity(ra2))
-            }] => {assert_eq!(ra.seq, 1); assert_eq!(ra2.seq, 2)}
-        );
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        assert_eq!(ready_to_execute_las.len(), 0);
-
-        let commands = wfm.get_server_commands().commands;
-        if replay {
-            assert_eq!(commands.len(), 1);
-            assert_eq!(
-                commands[0].command_type,
-                CommandType::CompleteWorkflowExecution as i32
-            );
-        } else {
-            assert_eq!(commands.len(), 3);
-            assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-            assert_eq!(commands[1].command_type, CommandType::RecordMarker as i32);
-            assert_eq!(
-                commands[2].command_type,
-                CommandType::CompleteWorkflowExecution as i32
-            );
-        }
-
-        if !replay {
-            wfm.new_history(t.get_full_history_info().unwrap().into())
-                .await
-                .unwrap();
-        }
-        let act = wfm.get_next_activation().await.unwrap();
-        // Still only 1s ahead b/c parallel
-        assert_eq!(act.timestamp.unwrap().seconds, first_act_ts.seconds + 1);
-        assert_eq!(act.jobs.len(), 0);
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 0);
-
-        wfm.shutdown().await.unwrap();
-    }
-
-    async fn la_timer_la(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            ..Default::default()
-        })
-        .await;
-        ctx.timer(Duration::from_secs(5)).await;
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            ..Default::default()
-        })
-        .await;
-        Ok(().into())
-    }
-
-    #[rstest]
-    #[case::incremental(false)]
-    #[case::replay(true)]
-    #[tokio::test]
-    async fn las_separated_by_timer(#[case] replay: bool) {
-        let func = WorkflowFunction::new(la_timer_la);
-        let t = canned_histories::two_local_activities_separated_by_timer();
-        let histinfo = if replay {
-            t.get_full_history_info().unwrap().into()
-        } else {
-            t.get_history_info(1).unwrap().into()
-        };
-        let mut wfm = ManagedWFFunc::new_from_update(histinfo, func, vec![]);
-
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 0);
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        let num_queued = usize::from(!replay);
-        assert_eq!(ready_to_execute_las.len(), num_queued);
-
-        if !replay {
-            wfm.complete_local_activity(1, ActivityExecutionResult::ok(b"Resolved".into()))
-                .unwrap();
-        }
-
-        let act = wfm.get_next_activation().await.unwrap();
-        assert_matches!(
-            act.jobs.as_slice(),
-            [WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::ResolveActivity(ra))
-            }] => assert_eq!(ra.seq, 1)
-        );
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        assert_eq!(ready_to_execute_las.len(), 0);
-
-        let commands = wfm.get_server_commands().commands;
-        if replay {
-            assert_eq!(commands.len(), 1);
-            assert_eq!(commands[0].command_type, CommandType::StartTimer as i32);
-        } else {
-            assert_eq!(commands.len(), 2);
-            assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-            assert_eq!(commands[1].command_type, CommandType::StartTimer as i32);
-        }
-
-        let act = if !replay {
-            wfm.new_history(t.get_history_info(2).unwrap().into())
-                .await
-                .unwrap()
-        } else {
-            wfm.get_next_activation().await.unwrap()
-        };
-        assert_matches!(
-            act.jobs.as_slice(),
-            [WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::FireTimer(_))
-            }]
-        );
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        let num_queued = usize::from(!replay);
-        assert_eq!(ready_to_execute_las.len(), num_queued);
-        if !replay {
-            wfm.complete_local_activity(2, ActivityExecutionResult::ok(b"Resolved".into()))
-                .unwrap();
-        }
-
-        let act = wfm.get_next_activation().await.unwrap();
-        assert_matches!(
-            act.jobs.as_slice(),
-            [WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::ResolveActivity(ra))
-            }] => assert_eq!(ra.seq, 2)
-        );
-
-        let commands = wfm.get_server_commands().commands;
-        if replay {
-            assert_eq!(commands.len(), 1);
-            assert_eq!(
-                commands[0].command_type,
-                CommandType::CompleteWorkflowExecution as i32
-            );
-        } else {
-            assert_eq!(commands.len(), 2);
-            assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-            assert_eq!(
-                commands[1].command_type,
-                CommandType::CompleteWorkflowExecution as i32
-            );
-        }
-
-        wfm.shutdown().await.unwrap();
-    }
-
-    #[tokio::test]
-    async fn one_la_heartbeating_wft_failure_still_executes() {
-        let func = WorkflowFunction::new(la_wf);
-        let mut t = TestHistoryBuilder::default();
-        t.add_by_type(EventType::WorkflowExecutionStarted);
-        // Heartbeats
-        t.add_full_wf_task();
-        // fails a wft for some reason
-        t.add_workflow_task_scheduled_and_started();
-        t.add_workflow_task_failed_with_failure(
-            WorkflowTaskFailedCause::NonDeterministicError,
-            Default::default(),
-        );
-        t.add_workflow_task_scheduled_and_started();
-
-        let histinfo = t.get_full_history_info().unwrap().into();
-        let mut wfm = ManagedWFFunc::new_from_update(histinfo, func, vec![]);
-
-        // First activation will request to run the LA since the heartbeat & wft failure are skipped
-        // over
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 0);
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        assert_eq!(ready_to_execute_las.len(), 1);
-        // We can happily complete it now
-        wfm.complete_local_activity(1, ActivityExecutionResult::ok(b"hi".into()))
-            .unwrap();
-
-        wfm.shutdown().await.unwrap();
-    }
-
-    /// This test verifies something that technically shouldn't really be possible but is worth
-    /// checking anyway. What happens if in memory we think an LA passed but then the next history
-    /// chunk comes back with it failing? We should fail with a mismatch.
-    #[tokio::test]
-    async fn exec_passes_but_history_has_fail() {
-        let func = WorkflowFunction::new(la_wf);
-        let mut t = TestHistoryBuilder::default();
-        t.add_by_type(EventType::WorkflowExecutionStarted);
-        t.add_full_wf_task();
-        t.add_local_activity_fail_marker(
-            1,
-            "1",
-            Failure::application_failure("I failed".to_string(), false),
-        );
-        t.add_workflow_execution_completed();
-
-        let histinfo = t.get_history_info(1).unwrap().into();
-        let mut wfm = ManagedWFFunc::new_from_update(histinfo, func, vec![]);
-
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 0);
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        assert_eq!(ready_to_execute_las.len(), 1);
-        // Completes OK
-        wfm.complete_local_activity(1, ActivityExecutionResult::ok(b"hi".into()))
-            .unwrap();
-
-        // next activation unblocks LA
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 2);
-        assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-        assert_eq!(
-            commands[1].command_type,
-            CommandType::CompleteWorkflowExecution as i32
-        );
-
-        let err = wfm
-            .new_history(t.get_full_history_info().unwrap().into())
-            .await
-            .unwrap_err();
-        assert!(err.to_string().contains("Nondeterminism"));
-        wfm.shutdown().await.unwrap();
-    }
-
-    #[rstest]
-    #[tokio::test]
-    async fn immediate_cancel(
-        #[values(
-            ActivityCancellationType::WaitCancellationCompleted,
-            ActivityCancellationType::TryCancel,
-            ActivityCancellationType::Abandon
-        )]
-        cancel_type: ActivityCancellationType,
-    ) {
-        let func = WorkflowFunction::new(move |ctx| async move {
-            let la = ctx.local_activity(LocalActivityOptions {
-                cancel_type,
-                ..Default::default()
-            });
-            la.cancel(&ctx);
-            la.await;
-            Ok(().into())
-        });
-        let mut t = TestHistoryBuilder::default();
-        t.add_by_type(EventType::WorkflowExecutionStarted);
-        t.add_full_wf_task();
-        t.add_workflow_execution_completed();
-
-        let histinfo = t.get_history_info(1).unwrap().into();
-        let mut wfm = ManagedWFFunc::new_from_update(histinfo, func, vec![]);
-
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 1);
-        // We record the cancel marker
-        assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-        // Importantly, the activity shouldn't get executed since it was insta-cancelled
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        assert_eq!(ready_to_execute_las.len(), 0);
-
-        // next activation unblocks LA, which is cancelled now.
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 2);
-        assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-        assert_eq!(
-            commands[1].command_type,
-            CommandType::CompleteWorkflowExecution as i32
-        );
-
-        wfm.shutdown().await.unwrap();
-    }
-
-    #[rstest]
-    #[case::incremental(false)]
-    #[case::replay(true)]
-    #[tokio::test]
-    async fn cancel_after_act_starts(
-        #[case] replay: bool,
-        #[values(
-            ActivityCancellationType::WaitCancellationCompleted,
-            ActivityCancellationType::TryCancel,
-            ActivityCancellationType::Abandon
-        )]
-        cancel_type: ActivityCancellationType,
-    ) {
-        let func = WorkflowFunction::new(move |ctx| async move {
-            let la = ctx.local_activity(LocalActivityOptions {
-                cancel_type,
-                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-                ..Default::default()
-            });
-            ctx.timer(Duration::from_secs(1)).await;
-            la.cancel(&ctx);
-            // This extra timer is here to ensure the presence of another WF task doesn't mess up
-            // resolving the LA with cancel on replay
-            ctx.timer(Duration::from_secs(1)).await;
-            let resolution = la.await;
-            assert!(resolution.cancelled());
-            Ok(().into())
-        });
-
-        let mut t = TestHistoryBuilder::default();
-        t.add_by_type(EventType::WorkflowExecutionStarted);
-        t.add_full_wf_task();
-        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
-        t.add_timer_fired(timer_started_event_id, "1".to_string());
-        t.add_full_wf_task();
-        if cancel_type != ActivityCancellationType::WaitCancellationCompleted {
-            // With non-wait cancels, the cancel is immediate
-            t.add_local_activity_cancel_marker(1, "1");
-        }
-        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
-        if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
-            // With wait cancels, the cancel marker is not recorded until activity reports.
-            t.add_local_activity_cancel_marker(1, "1");
-        }
-        t.add_timer_fired(timer_started_event_id, "2".to_string());
-        t.add_full_wf_task();
-        t.add_workflow_execution_completed();
-
-        let histinfo = if replay {
-            t.get_full_history_info().unwrap().into()
-        } else {
-            t.get_history_info(1).unwrap().into()
-        };
-        let mut wfm = ManagedWFFunc::new_from_update(histinfo, func, vec![]);
-
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 1);
-        assert_eq!(commands[0].command_type, CommandType::StartTimer as i32);
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
-        let num_queued = usize::from(!replay);
-        assert_eq!(ready_to_execute_las.len(), num_queued);
-
-        // Next activation timer fires and activity cancel will be requested
-        if replay {
-            wfm.get_next_activation().await.unwrap()
-        } else {
-            wfm.new_history(t.get_history_info(2).unwrap().into())
-                .await
-                .unwrap()
-        };
-
-        let commands = wfm.get_server_commands().commands;
-        if cancel_type == ActivityCancellationType::WaitCancellationCompleted || replay {
-            assert_eq!(commands.len(), 1);
-            assert_eq!(commands[0].command_type, CommandType::StartTimer as i32);
-        } else {
-            // Try-cancel/abandon will immediately record marker (when not replaying)
-            assert_eq!(commands.len(), 2);
-            assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-            assert_eq!(commands[1].command_type, CommandType::StartTimer as i32);
-        }
-
-        let commands = if replay {
-            wfm.get_next_activation().await.unwrap();
-            wfm.get_server_commands().commands
-        } else {
-            // On non replay, there's an additional activation, because completing with the cancel
-            // wants to wake up the workflow to see if resolving the LA as cancelled did anything.
-            // In this case, it doesn't really, because we just hit the next timer which is also
-            // what would have happened if we woke up with new history -- but it does mean we
-            // generate the commands at this point. This matters b/c we want to make sure the record
-            // marker command is sent as soon as cancel happens.
-            if cancel_type == WaitCancellationCompleted {
-                wfm.complete_local_activity(1, ActivityExecutionResult::cancel_from_details(None))
-                    .unwrap();
-            }
-            wfm.get_next_activation().await.unwrap();
-            let commands = wfm.get_server_commands().commands;
-            assert_eq!(commands.len(), 2);
-            if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
-                assert_eq!(commands[0].command_type, CommandType::StartTimer as i32);
-                assert_eq!(commands[1].command_type, CommandType::RecordMarker as i32);
-            } else {
-                assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-                assert_eq!(commands[1].command_type, CommandType::StartTimer as i32);
-            }
-
-            wfm.new_history(t.get_history_info(3).unwrap().into())
-                .await
-                .unwrap();
-            wfm.get_next_activation().await.unwrap();
-            wfm.get_server_commands().commands
-        };
-
-        assert_eq!(commands.len(), 1);
-        assert_eq!(
-            commands[0].command_type,
-            CommandType::CompleteWorkflowExecution as i32
-        );
-
-        wfm.shutdown().await.unwrap();
-    }
-}