@temporalio/core-bridge 1.13.1 → 1.14.0

package/sdk-core/{core → crates/sdk-core}/src/worker/workflow/machines/local_activity_state_machine.rs

@@ -1,6 +1,6 @@
 use super::{
-    EventInfo, OnEventWrapper, WFMachinesAdapter, WFMachinesError,
-    workflow_machines::MachineResponse,
+    EventInfo, MachineError, OnEventWrapper, StateMachine, TransitionResult, WFMachinesAdapter,
+    WFMachinesError, fsm, workflow_machines::MachineResponse,
 };
 use crate::{
     internal_flags::CoreInternalFlags,
@@ -8,18 +8,18 @@ use crate::{
     worker::{
         LocalActivityExecutionResult,
         workflow::{
-            InternalFlagsRef,
+            InternalFlagsRef, fatal,
             machines::{HistEventData, activity_state_machine::activity_fail_info},
+            nondeterminism,
         },
     },
 };
 use itertools::Itertools;
-use rustfsm::{MachineError, StateMachine, TransitionResult, fsm};
 use std::{
     convert::TryFrom,
     time::{Duration, SystemTime},
 };
-use temporal_sdk_core_protos::{
+use temporalio_common::protos::{
     constants::LOCAL_ACTIVITY_MARKER_NAME,
     coresdk::{
         activity_result::{
@@ -31,7 +31,7 @@ use temporal_sdk_core_protos::{
         workflow_commands::ActivityCancellationType,
     },
     temporal::api::{
-        command::v1::{RecordMarkerCommandAttributes, command},
+        command::v1::{Command as ProtoCommand, RecordMarkerCommandAttributes, command},
         enums::v1::{CommandType, EventType, RetryState},
         failure::v1::{Failure, failure::FailureInfo},
     },
@@ -163,8 +163,8 @@ pub(super) fn new_local_activity(
         }
     } else {
         if maybe_pre_resolved.is_some() {
-            return Err(WFMachinesError::Nondeterminism(
-                "Local activity cannot be created as pre-resolved while not replaying".to_string(),
+            return Err(nondeterminism!(
+                "Local activity cannot be created as pre-resolved while not replaying"
             ));
         }
         Executing {}.into()
@@ -211,10 +211,10 @@ impl LocalActivityMachine {
             LocalActivityMachineState::ResultNotified(_) => Ok(false),
             LocalActivityMachineState::WaitingMarkerEvent(_) => Ok(true),
             LocalActivityMachineState::WaitingMarkerEventPreResolved(_) => Ok(true),
-            _ => Err(WFMachinesError::Fatal(format!(
+            _ => Err(fatal!(
                 "Attempted to check for LA marker handling in invalid state {}",
                 self.state()
-            ))),
+            )),
         }
     }
 
@@ -242,12 +242,12 @@ impl LocalActivityMachine {
         let mut res =
             OnEventWrapper::on_event_mut(self, LocalActivityMachineEvents::StartedNonReplayWFT)
                 .map_err(|e| match e {
-                    MachineError::InvalidTransition => WFMachinesError::Fatal(format!(
+                    MachineError::InvalidTransition => fatal!(
                         "Invalid transition while notifying local activity (seq {})\
                          of non-replay-wft-started in {}",
                         self.shared_state.attrs.seq,
                         self.state(),
-                    )),
+                    ),
                     MachineError::Underlying(e) => e,
                 })?;
         let res = res.pop().expect("Always produces one response");
@@ -296,12 +296,12 @@ impl LocalActivityMachine {
             LocalActivityMachineEvents::HandleResult(dat)
         };
         let res = OnEventWrapper::on_event_mut(self, evt).map_err(|e| match e {
-            MachineError::InvalidTransition => WFMachinesError::Fatal(format!(
+            MachineError::InvalidTransition => fatal!(
                 "Invalid transition resolving local activity (seq {}, from marker: {}) in {}",
                 self.shared_state.attrs.seq,
                 from_marker,
                 self.state(),
-            )),
+            ),
             MachineError::Underlying(e) => e,
         })?;
 
@@ -543,16 +543,16 @@ impl ResultNotified {
         dat: CompleteLocalActivityData,
     ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
         if self.result_type == ResultType::Completed && dat.result.is_err() {
-            return TransitionResult::Err(WFMachinesError::Nondeterminism(format!(
+            return TransitionResult::Err(nondeterminism!(
                 "Local activity (seq {}) completed successfully locally, but history said \
                  it failed!",
                 shared.attrs.seq
-            )));
+            ));
         } else if self.result_type == ResultType::Failed && dat.result.is_ok() {
-            return TransitionResult::Err(WFMachinesError::Nondeterminism(format!(
+            return TransitionResult::Err(nondeterminism!(
                 "Local activity (seq {}) failed locally, but history said it completed!",
                 shared.attrs.seq
-            )));
+            ));
         }
         verify_marker_dat!(shared, &dat, TransitionResult::default())
     }
@@ -772,9 +772,11 @@ impl WFMachinesAdapter for LocalActivityMachine {
                         header: None,
                         failure: maybe_failure,
                     };
-                    responses.push(MachineResponse::IssueNewCommand(
-                        command::Attributes::RecordMarkerCommandAttributes(marker_data).into(),
-                    ));
+                    let command = ProtoCommand {
+                        user_metadata: self.shared_state.attrs.user_metadata.clone(),
+                        ..command::Attributes::RecordMarkerCommandAttributes(marker_data).into()
+                    };
+                    responses.push(MachineResponse::IssueNewCommand(command));
                 }
                 Ok(responses)
             }
@@ -810,15 +812,15 @@ impl TryFrom<HistEventData> for LocalActivityMachineEvents {
     fn try_from(e: HistEventData) -> Result<Self, Self::Error> {
         let e = e.event;
         if e.event_type() != EventType::MarkerRecorded {
-            return Err(WFMachinesError::Nondeterminism(format!(
+            return Err(nondeterminism!(
                 "Local activity machine cannot handle this event: {e}"
-            )));
+            ));
         }
 
         match e.into_local_activity_marker_details() {
             Some(marker_dat) => Ok(LocalActivityMachineEvents::MarkerRecorded(marker_dat)),
-            _ => Err(WFMachinesError::Nondeterminism(
-                "Local activity machine encountered an unparsable marker".to_string(),
+            _ => Err(nondeterminism!(
+                "Local activity machine encountered an unparsable marker"
             )),
         }
     }
@@ -829,11 +831,12 @@ fn verify_marker_data_matches(
     dat: &CompleteLocalActivityData,
 ) -> Result<(), WFMachinesError> {
     if shared.attrs.seq != dat.marker_dat.seq {
-        return Err(WFMachinesError::Nondeterminism(format!(
+        return Err(nondeterminism!(
             "Local activity marker data has sequence number {} but matched against LA \
             command with sequence number {}",
-            dat.marker_dat.seq, shared.attrs.seq
-        )));
+            dat.marker_dat.seq,
+            shared.attrs.seq
+        ));
     }
     // Here we use whether or not we were replaying when we _first invoked_ the LA, because we
     // are always replaying when we see the marker recorded event, and that would make this check
@@ -843,570 +846,22 @@ fn verify_marker_data_matches(
         !shared.replaying_when_invoked,
     ) {
         if dat.marker_dat.activity_id != shared.attrs.activity_id {
-            return Err(WFMachinesError::Nondeterminism(format!(
+            return Err(nondeterminism!(
                 "Activity id of recorded marker '{}' does not \
                  match activity id of local activity command '{}'",
-                dat.marker_dat.activity_id, shared.attrs.activity_id
-            )));
+                dat.marker_dat.activity_id,
+                shared.attrs.activity_id
+            ));
         }
         if dat.marker_dat.activity_type != shared.attrs.activity_type {
-            return Err(WFMachinesError::Nondeterminism(format!(
+            return Err(nondeterminism!(
                 "Activity type of recorded marker '{}' does not \
                  match activity type of local activity command '{}'",
-                dat.marker_dat.activity_type, shared.attrs.activity_type
-            )));
+                dat.marker_dat.activity_type,
+                shared.attrs.activity_type
+            ));
         }
     }
 
     Ok(())
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        replay::TestHistoryBuilder,
-        test_help::{MockPollCfg, ResponseType, build_fake_sdk, canned_histories},
-    };
-    use anyhow::anyhow;
-    use rstest::rstest;
-    use std::{
-        sync::atomic::{AtomicI64, Ordering},
-        time::Duration,
-    };
-    use temporal_sdk::{
-        ActContext, ActivityError, CancellableFuture, LocalActivityOptions, WfContext,
-        WorkflowResult,
-    };
-    use temporal_sdk_core_protos::{
-        DEFAULT_ACTIVITY_TYPE, DEFAULT_WORKFLOW_TYPE,
-        coresdk::{
-            AsJsonPayloadExt,
-            workflow_activation::{WorkflowActivationJob, workflow_activation_job},
-        },
-        temporal::api::{
-            common::v1::RetryPolicy, enums::v1::WorkflowTaskFailedCause, failure::v1::Failure,
-        },
-    };
-    use temporal_sdk_core_test_utils::interceptors::ActivationAssertionsInterceptor;
-    use tokio_util::sync::CancellationToken;
-
-    async fn la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            input: ().as_json_payload().unwrap(),
-            retry_policy: RetryPolicy {
-                maximum_attempts: 1,
-                ..Default::default()
-            },
-            ..Default::default()
-        })
-        .await;
-        Ok(().into())
-    }
-
-    #[rstest]
-    #[case::incremental(false, true)]
-    #[case::replay(true, true)]
-    #[case::incremental_fail(false, false)]
-    #[case::replay_fail(true, false)]
-    #[tokio::test]
-    async fn one_la_success(#[case] replay: bool, #[case] completes_ok: bool) {
-        let activity_id = "1";
-        let mut t = TestHistoryBuilder::default();
-        t.add_by_type(EventType::WorkflowExecutionStarted);
-        t.add_full_wf_task();
-        if completes_ok {
-            t.add_local_activity_result_marker(1, activity_id, b"hi".into());
-        } else {
-            t.add_local_activity_fail_marker(
-                1,
-                activity_id,
-                Failure::application_failure("I failed".to_string(), false),
-            );
-        }
-        t.add_workflow_task_scheduled_and_started();
-
-        let mut mock_cfg = if replay {
-            MockPollCfg::from_resps(t, [ResponseType::AllHistory])
-        } else {
-            MockPollCfg::from_hist_builder(t)
-        };
-        mock_cfg.completion_asserts_from_expectations(|mut asserts| {
-            asserts.then(move |wft| {
-                let commands = &wft.commands;
-                if !replay {
-                    assert_eq!(commands.len(), 2);
-                    assert_eq!(commands[0].command_type(), CommandType::RecordMarker);
-                    if completes_ok {
-                        assert_matches!(
-                            commands[0].attributes.as_ref().unwrap(),
-                            command::Attributes::RecordMarkerCommandAttributes(
-                                RecordMarkerCommandAttributes { failure: None, .. }
-                            )
-                        );
-                    } else {
-                        assert_matches!(
-                            commands[0].attributes.as_ref().unwrap(),
-                            command::Attributes::RecordMarkerCommandAttributes(
-                                RecordMarkerCommandAttributes {
-                                    failure: Some(_),
-                                    ..
-                                }
-                            )
-                        );
-                    }
-                    assert_eq!(
-                        commands[1].command_type(),
-                        CommandType::CompleteWorkflowExecution
-                    );
-                } else {
-                    assert_eq!(commands.len(), 1);
-                    assert_matches!(
-                        commands[0].command_type(),
-                        CommandType::CompleteWorkflowExecution
-                    );
-                }
-            });
-        });
-
-        let mut worker = build_fake_sdk(mock_cfg);
-        worker.register_wf(DEFAULT_WORKFLOW_TYPE, la_wf);
-        worker.register_activity(
-            DEFAULT_ACTIVITY_TYPE,
-            move |_ctx: ActContext, _: ()| async move {
-                if replay {
-                    panic!("Should not be invoked on replay");
-                }
-                if completes_ok {
-                    Ok("hi")
-                } else {
-                    Err(anyhow!("Oh no I failed!").into())
-                }
-            },
-        );
-        worker.run().await.unwrap();
-    }
-
-    async fn two_la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            input: ().as_json_payload().unwrap(),
-            ..Default::default()
-        })
-        .await;
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            input: ().as_json_payload().unwrap(),
-            ..Default::default()
-        })
-        .await;
-        Ok(().into())
-    }
-
-    async fn two_la_wf_parallel(ctx: WfContext) -> WorkflowResult<()> {
-        tokio::join!(
-            ctx.local_activity(LocalActivityOptions {
-                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-                input: ().as_json_payload().unwrap(),
-                ..Default::default()
-            }),
-            ctx.local_activity(LocalActivityOptions {
-                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-                input: ().as_json_payload().unwrap(),
-                ..Default::default()
-            })
-        );
-        Ok(().into())
-    }
-
-    #[rstest]
-    #[tokio::test]
-    async fn two_sequential_las(
-        #[values(true, false)] replay: bool,
-        #[values(true, false)] parallel: bool,
-    ) {
-        let t = canned_histories::two_local_activities_one_wft(parallel);
-        let mut mock_cfg = if replay {
-            MockPollCfg::from_resps(t, [ResponseType::AllHistory])
-        } else {
-            MockPollCfg::from_hist_builder(t)
-        };
-
-        let mut aai = ActivationAssertionsInterceptor::default();
-        let first_act_ts_seconds: &'static _ = Box::leak(Box::new(AtomicI64::new(-1)));
-        aai.then(|a| {
-            first_act_ts_seconds.store(a.timestamp.as_ref().unwrap().seconds, Ordering::Relaxed)
-        });
-        // Verify LAs advance time (they take 1s as defined in the canned history)
-        aai.then(move |a| {
-            if !parallel {
-                assert_matches!(
-                    a.jobs.as_slice(),
-                    [WorkflowActivationJob {
-                        variant: Some(workflow_activation_job::Variant::ResolveActivity(ra))
-                    }] => assert_eq!(ra.seq, 1)
-                );
-            } else {
-                assert_matches!(
-                    a.jobs.as_slice(),
-                    [WorkflowActivationJob {
-                        variant: Some(workflow_activation_job::Variant::ResolveActivity(ra))
-                     }, WorkflowActivationJob {
-                        variant: Some(workflow_activation_job::Variant::ResolveActivity(ra2))
-                    }] => {assert_eq!(ra.seq, 1); assert_eq!(ra2.seq, 2)}
-                );
-            }
-            if replay {
-                assert!(
-                    a.timestamp.as_ref().unwrap().seconds
-                        > first_act_ts_seconds.load(Ordering::Relaxed)
-                )
-            }
-        });
-        if !parallel {
-            aai.then(move |a| {
-                assert_matches!(
-                    a.jobs.as_slice(),
-                    [WorkflowActivationJob {
-                        variant: Some(workflow_activation_job::Variant::ResolveActivity(ra))
-                    }] => assert_eq!(ra.seq, 2)
-                );
-                if replay {
-                    assert!(
-                        a.timestamp.as_ref().unwrap().seconds
-                            >= first_act_ts_seconds.load(Ordering::Relaxed) + 2
-                    )
-                }
-            });
-        }
-
-        mock_cfg.completion_asserts_from_expectations(|mut asserts| {
-            asserts.then(move |wft| {
-                let commands = &wft.commands;
-                if !replay {
-                    assert_eq!(commands.len(), 3);
-                    assert_eq!(commands[0].command_type(), CommandType::RecordMarker);
-                    assert_eq!(commands[1].command_type(), CommandType::RecordMarker);
-                    assert_matches!(
-                        commands[2].command_type(),
-                        CommandType::CompleteWorkflowExecution
-                    );
-                } else {
-                    assert_eq!(commands.len(), 1);
-                    assert_matches!(
-                        commands[0].command_type(),
-                        CommandType::CompleteWorkflowExecution
-                    );
-                }
-            });
-        });
-
-        let mut worker = build_fake_sdk(mock_cfg);
-        worker.set_worker_interceptor(aai);
-        if parallel {
-            worker.register_wf(DEFAULT_WORKFLOW_TYPE, two_la_wf_parallel);
-        } else {
-            worker.register_wf(DEFAULT_WORKFLOW_TYPE, two_la_wf);
-        }
-        worker.register_activity(
-            DEFAULT_ACTIVITY_TYPE,
-            move |_ctx: ActContext, _: ()| async move { Ok("Resolved") },
-        );
-        worker.run().await.unwrap();
-    }
-
-    async fn la_timer_la(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            input: ().as_json_payload().unwrap(),
-            ..Default::default()
-        })
-        .await;
-        ctx.timer(Duration::from_secs(5)).await;
-        ctx.local_activity(LocalActivityOptions {
-            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-            input: ().as_json_payload().unwrap(),
-            ..Default::default()
-        })
-        .await;
-        Ok(().into())
-    }
-
-    #[rstest]
-    #[case::incremental(false)]
-    #[case::replay(true)]
-    #[tokio::test]
-    async fn las_separated_by_timer(#[case] replay: bool) {
-        let mut t = TestHistoryBuilder::default();
-        t.add_by_type(EventType::WorkflowExecutionStarted);
-        t.add_full_wf_task();
-        t.add_local_activity_result_marker(1, "1", b"hi".into());
-        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
-        t.add_timer_fired(timer_started_event_id, "1".to_string());
-        t.add_full_wf_task();
-        t.add_local_activity_result_marker(2, "2", b"hi2".into());
-        t.add_workflow_task_scheduled_and_started();
-        let mut mock_cfg = if replay {
-            MockPollCfg::from_resps(t, [ResponseType::AllHistory])
-        } else {
-            MockPollCfg::from_hist_builder(t)
-        };
-
-        let mut aai = ActivationAssertionsInterceptor::default();
-        aai.skip_one()
-            .then(|a| {
-                assert_matches!(
-                    a.jobs.as_slice(),
-                    [WorkflowActivationJob {
-                        variant: Some(workflow_activation_job::Variant::ResolveActivity(ra))
-                    }] => assert_eq!(ra.seq, 1)
-                );
-            })
-            .then(|a| {
-                assert_matches!(
-                    a.jobs.as_slice(),
-                    [WorkflowActivationJob {
-                        variant: Some(workflow_activation_job::Variant::FireTimer(_))
-                    }]
-                );
-            });
-
-        mock_cfg.completion_asserts_from_expectations(|mut asserts| {
-            if replay {
-                asserts.then(|wft| {
-                    assert_eq!(wft.commands.len(), 1);
-                    assert_eq!(
-                        wft.commands[0].command_type,
-                        CommandType::CompleteWorkflowExecution as i32
-                    );
-                });
-            } else {
-                asserts
-                    .then(|wft| {
-                        let commands = &wft.commands;
-                        assert_eq!(commands.len(), 2);
-                        assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-                        assert_eq!(commands[1].command_type, CommandType::StartTimer as i32);
-                    })
-                    .then(|wft| {
-                        let commands = &wft.commands;
-                        assert_eq!(commands.len(), 2);
-                        assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-                        assert_eq!(
-                            commands[1].command_type,
-                            CommandType::CompleteWorkflowExecution as i32
-                        );
-                    });
-            }
-        });
-
-        let mut worker = build_fake_sdk(mock_cfg);
-        worker.set_worker_interceptor(aai);
-        worker.register_wf(DEFAULT_WORKFLOW_TYPE, la_timer_la);
-        worker.register_activity(
-            DEFAULT_ACTIVITY_TYPE,
-            move |_ctx: ActContext, _: ()| async move { Ok("Resolved") },
-        );
-        worker.run().await.unwrap();
-    }
-
-    #[tokio::test]
-    async fn one_la_heartbeating_wft_failure_still_executes() {
-        let mut t = TestHistoryBuilder::default();
-        t.add_by_type(EventType::WorkflowExecutionStarted);
-        // Heartbeats
-        t.add_full_wf_task();
-        // fails a wft for some reason
-        t.add_workflow_task_scheduled_and_started();
-        t.add_workflow_task_failed_with_failure(
-            WorkflowTaskFailedCause::NonDeterministicError,
-            Default::default(),
-        );
-        t.add_workflow_task_scheduled_and_started();
-
-        let mut mock_cfg = MockPollCfg::from_hist_builder(t);
-        mock_cfg.completion_asserts_from_expectations(|mut asserts| {
-            asserts.then(move |wft| {
-                assert_eq!(wft.commands.len(), 2);
-                assert_eq!(wft.commands[0].command_type(), CommandType::RecordMarker);
-                assert_matches!(
-                    wft.commands[1].command_type(),
-                    CommandType::CompleteWorkflowExecution
-                );
-            });
-        });
-
-        let mut worker = build_fake_sdk(mock_cfg);
-        worker.register_wf(DEFAULT_WORKFLOW_TYPE, la_wf);
-        worker.register_activity(
-            DEFAULT_ACTIVITY_TYPE,
-            move |_ctx: ActContext, _: ()| async move { Ok("Resolved") },
-        );
-        worker.run().await.unwrap();
-    }
-
-    #[rstest]
-    #[tokio::test]
-    async fn immediate_cancel(
-        #[values(
-            ActivityCancellationType::WaitCancellationCompleted,
-            ActivityCancellationType::TryCancel,
-            ActivityCancellationType::Abandon
-        )]
-        cancel_type: ActivityCancellationType,
-    ) {
-        let mut t = TestHistoryBuilder::default();
-        t.add_by_type(EventType::WorkflowExecutionStarted);
-        t.add_full_wf_task();
-        t.add_workflow_execution_completed();
-
-        let mut mock_cfg = MockPollCfg::from_hist_builder(t);
-        mock_cfg.completion_asserts_from_expectations(|mut asserts| {
-            asserts.then(|wft| {
-                assert_eq!(wft.commands.len(), 2);
-                // We record the cancel marker
-                assert_eq!(wft.commands[0].command_type(), CommandType::RecordMarker);
-                assert_matches!(
-                    wft.commands[1].command_type(),
-                    CommandType::CompleteWorkflowExecution
-                );
-            });
-        });
-
-        let mut worker = build_fake_sdk(mock_cfg);
-        worker.register_wf(DEFAULT_WORKFLOW_TYPE, move |ctx: WfContext| async move {
-            let la = ctx.local_activity(LocalActivityOptions {
-                cancel_type,
-                ..Default::default()
-            });
-            la.cancel(&ctx);
-            la.await;
-            Ok(().into())
-        });
-        // Explicitly don't register an activity, since we shouldn't need to run one.
-        worker.run().await.unwrap();
-    }
-
-    #[rstest]
-    #[case::incremental(false)]
-    #[case::replay(true)]
-    #[tokio::test]
-    async fn cancel_after_act_starts(
-        #[case] replay: bool,
-        #[values(
-            ActivityCancellationType::WaitCancellationCompleted,
-            ActivityCancellationType::TryCancel,
-            ActivityCancellationType::Abandon
-        )]
-        cancel_type: ActivityCancellationType,
-    ) {
-        let mut t = TestHistoryBuilder::default();
-        t.add_wfe_started_with_wft_timeout(Duration::from_millis(100));
-        t.add_full_wf_task();
-        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
-        t.add_timer_fired(timer_started_event_id, "1".to_string());
-        t.add_full_wf_task();
-        // This extra workflow task serves to prevent looking ahead and pre-resolving during
-        // wait-cancel.
-        // TODO: including this on non wait-cancel seems to cause double-send of
-        //   marker recorded cmd
-        if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
-            t.add_full_wf_task();
-        }
-        if cancel_type != ActivityCancellationType::WaitCancellationCompleted {
-            // With non-wait cancels, the cancel is immediate
-            t.add_local_activity_cancel_marker(1, "1");
-        }
-        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
-        if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
-            // With wait cancels, the cancel marker is not recorded until activity reports.
-            t.add_local_activity_cancel_marker(1, "1");
-        }
-        t.add_timer_fired(timer_started_event_id, "2".to_string());
-        t.add_full_wf_task();
-        t.add_workflow_execution_completed();
-
-        let mut mock_cfg = if replay {
-            MockPollCfg::from_resps(t, [ResponseType::AllHistory])
-        } else {
-            MockPollCfg::from_hist_builder(t)
-        };
-        let allow_cancel_barr = CancellationToken::new();
-        let allow_cancel_barr_clone = allow_cancel_barr.clone();
-
-        if !replay {
-            mock_cfg.completion_asserts_from_expectations(|mut asserts| {
-                asserts
-                    .then(move |wft| {
-                        assert_eq!(wft.commands.len(), 1);
-                        assert_eq!(wft.commands[0].command_type, CommandType::StartTimer as i32);
-                    })
-                    .then(move |wft| {
-                        let commands = &wft.commands;
-                        if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
-                            assert_eq!(commands.len(), 1);
-                            assert_eq!(commands[0].command_type, CommandType::StartTimer as i32);
-                        } else {
-                            // Try-cancel/abandon immediately recordsmarker (when not replaying)
-                            assert_eq!(commands.len(), 2);
-                            assert_eq!(commands[0].command_type, CommandType::RecordMarker as i32);
-                            assert_eq!(commands[1].command_type, CommandType::StartTimer as i32);
-                        }
-                        // Allow the wait-cancel to actually cancel
-                        allow_cancel_barr.cancel();
-                    })
-                    .then(move |wft| {
-                        let commands = &wft.commands;
-                        if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
-                            assert_eq!(commands[0].command_type, CommandType::StartTimer as i32);
-                            assert_eq!(commands[1].command_type, CommandType::RecordMarker as i32);
-                        } else {
-                            assert_eq!(
-                                commands[0].command_type,
-                                CommandType::CompleteWorkflowExecution as i32
-                            );
-                        }
-                    });
-            });
-        }
-
-        let mut worker = build_fake_sdk(mock_cfg);
-        worker.register_wf(DEFAULT_WORKFLOW_TYPE, move |ctx: WfContext| async move {
-            let la = ctx.local_activity(LocalActivityOptions {
-                cancel_type,
-                input: ().as_json_payload().unwrap(),
-                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-                ..Default::default()
-            });
-            ctx.timer(Duration::from_secs(1)).await;
-            la.cancel(&ctx);
-            // This extra timer is here to ensure the presence of another WF task doesn't mess up
-            // resolving the LA with cancel on replay
-            ctx.timer(Duration::from_secs(1)).await;
-            let resolution = la.await;
-            assert!(resolution.cancelled());
-            let rfail = resolution.unwrap_failure();
-            assert_matches!(
-                rfail.failure_info,
-                Some(FailureInfo::ActivityFailureInfo(_))
-            );
-            assert_matches!(
-                rfail.cause.unwrap().failure_info,
-                Some(FailureInfo::CanceledFailureInfo(_))
-            );
-            Ok(().into())
-        });
-        worker.register_activity(DEFAULT_ACTIVITY_TYPE, move |ctx: ActContext, _: ()| {
-            let allow_cancel_barr_clone = allow_cancel_barr_clone.clone();
-            async move {
-                if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
-                    ctx.cancelled().await;
-                }
-                allow_cancel_barr_clone.cancelled().await;
-                Result::<(), _>::Err(ActivityError::cancelled())
-            }
-        });
-        worker.run().await.unwrap();
-    }
-}