@temporalio/core-bridge 1.5.2 → 1.7.0

package/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs

@@ -3,8 +3,15 @@ use super::{
     WFMachinesError,
 };
 use crate::{
+    internal_flags::CoreInternalFlags,
     protosext::{CompleteLocalActivityData, HistoryEventExt, ValidScheduleLA},
-    worker::{workflow::OutgoingJob, LocalActivityExecutionResult},
+    worker::{
+        workflow::{
+            machines::{activity_state_machine::activity_fail_info, HistEventData},
+            InternalFlagsRef, OutgoingJob,
+        },
+        LocalActivityExecutionResult,
+    },
 };
 use rustfsm::{fsm, MachineError, StateMachine, TransitionResult};
 use std::{
@@ -24,8 +31,8 @@ use temporal_sdk_core_protos::{
     },
     temporal::api::{
         command::v1::{Command, RecordMarkerCommandAttributes},
-        enums::v1::{CommandType, EventType},
-        failure::v1::failure::FailureInfo,
+        enums::v1::{CommandType, EventType, RetryState},
+        failure::v1::{failure::FailureInfo, Failure},
         history::v1::HistoryEvent,
     },
     utilities::TryIntoOrNone,
@@ -59,15 +66,17 @@ fsm! {
       --> MarkerCommandRecorded;
 
     // Replay path ================================================================================
-    // LAs on the replay path should never have handle result explicitly called on them, but do need
-    // to eventually see the marker
+    // LAs on the replay path always need to eventually see the marker
     WaitingMarkerEvent --(MarkerRecorded(CompleteLocalActivityData), shared on_marker_recorded)
       --> MarkerCommandRecorded;
     // If we are told to cancel while waiting for the marker, we still need to wait for the marker.
-    WaitingMarkerEvent --(Cancel, on_cancel_requested) --> WaitingMarkerEventCancelled;
+    WaitingMarkerEvent --(Cancel, on_cancel_requested) --> WaitingMarkerEvent;
+    // Because there could be non-heartbeat WFTs (ex: signals being received) between scheduling
+    // the LA and the marker being recorded, peekahead might not always resolve the LA *before*
+    // scheduling it. This transition accounts for that.
+    WaitingMarkerEvent --(HandleKnownResult(ResolveDat), on_handle_result) --> WaitingMarkerEvent;
     WaitingMarkerEvent --(NoWaitCancel(ActivityCancellationType),
-                          on_no_wait_cancel) --> WaitingMarkerEventCancelled;
-    WaitingMarkerEventCancelled --(HandleResult(ResolveDat), on_handle_result) --> WaitingMarkerEvent;
+                          on_no_wait_cancel) --> WaitingMarkerEvent;
 
     // It is entirely possible to have started the LA while replaying, only to find that we have
     // reached a new WFT and there still was no marker. In such cases we need to execute the LA.
@@ -111,7 +120,12 @@ impl From<CompleteLocalActivityData> for ResolveDat {
             result: match d.result {
                 Ok(res) => LocalActivityExecutionResult::Completed(Success { result: Some(res) }),
                 Err(fail) => {
-                    if matches!(fail.failure_info, Some(FailureInfo::CanceledFailureInfo(_))) {
+                    if matches!(fail.failure_info, Some(FailureInfo::CanceledFailureInfo(_)))
+                        || matches!(
+                            fail.cause.as_deref().and_then(|f| f.failure_info.as_ref()),
+                            Some(FailureInfo::CanceledFailureInfo(_))
+                        )
+                    {
                         LocalActivityExecutionResult::Cancelled(Cancellation {
                             failure: Some(fail),
                         })
@@ -135,10 +149,11 @@ impl From<CompleteLocalActivityData> for ResolveDat {
 /// must resolve before we send a record marker command. A [MachineResponse] may be produced,
 /// to queue the LA for execution if it needs to be.
 pub(super) fn new_local_activity(
-    attrs: ValidScheduleLA,
+    mut attrs: ValidScheduleLA,
     replaying_when_invoked: bool,
     maybe_pre_resolved: Option<ResolveDat>,
     wf_time: Option<SystemTime>,
+    internal_flags: InternalFlagsRef,
 ) -> Result<(LocalActivityMachine, Vec<MachineResponse>), WFMachinesError> {
     let initial_state = if replaying_when_invoked {
         if let Some(dat) = maybe_pre_resolved {
@@ -155,14 +170,20 @@ pub(super) fn new_local_activity(
         Executing {}.into()
     };
 
-    let mut machine = LocalActivityMachine {
-        state: initial_state,
-        shared_state: SharedState {
+    // If the scheduled LA doesn't already have an "original" schedule time, assign one.
+    attrs
+        .original_schedule_time
+        .get_or_insert(SystemTime::now());
+
+    let mut machine = LocalActivityMachine::from_parts(
+        initial_state,
+        SharedState {
             attrs,
             replaying_when_invoked,
             wf_time_when_started: wf_time,
+            internal_flags,
         },
-    };
+    );
 
     let mut res = OnEventWrapper::on_event_mut(&mut machine, LocalActivityMachineEvents::Schedule)
         .expect("Scheduling local activities doesn't fail");
@@ -186,17 +207,26 @@ impl LocalActivityMachine {
     ///
     /// Attempting the check in any other state likely means a bug in the SDK.
     pub(super) fn marker_should_get_special_handling(&self) -> Result<bool, WFMachinesError> {
-        match &self.state {
+        match self.state() {
             LocalActivityMachineState::ResultNotified(_) => Ok(false),
             LocalActivityMachineState::WaitingMarkerEvent(_) => Ok(true),
             LocalActivityMachineState::WaitingMarkerEventPreResolved(_) => Ok(true),
             _ => Err(WFMachinesError::Fatal(format!(
                 "Attempted to check for LA marker handling in invalid state {}",
-                self.state
+                self.state()
             ))),
         }
     }
 
+    /// Returns true if the machine will willingly accept data from a marker in its current state.
+    /// IE: Calling [Self::try_resolve_with_dat] makes sense.
+    pub(super) fn will_accept_resolve_marker(&self) -> bool {
+        matches!(
+            self.state(),
+            LocalActivityMachineState::WaitingMarkerEvent(_)
+        )
+    }
+
     /// Must be called if the workflow encounters a non-replay workflow task
     pub(super) fn encountered_non_replay_wft(
         &mut self,
@@ -235,28 +265,45 @@ impl LocalActivityMachine {
         backoff: Option<prost_types::Duration>,
         original_schedule_time: Option<SystemTime>,
     ) -> Result<Vec<MachineResponse>, WFMachinesError> {
-        self.try_resolve_with_dat(ResolveDat {
-            result,
-            complete_time: self.shared_state.wf_time_when_started.map(|t| t + runtime),
-            attempt,
-            backoff,
-            original_schedule_time,
-        })
+        self._try_resolve(
+            ResolveDat {
+                result,
+                complete_time: self.shared_state.wf_time_when_started.map(|t| t + runtime),
+                attempt,
+                backoff,
+                original_schedule_time,
+            },
+            false,
+        )
     }
+
     /// Attempt to resolve the local activity with already known data, ex pre-resolved data
     pub(super) fn try_resolve_with_dat(
         &mut self,
         dat: ResolveDat,
     ) -> Result<Vec<MachineResponse>, WFMachinesError> {
-        let res = OnEventWrapper::on_event_mut(self, LocalActivityMachineEvents::HandleResult(dat))
-            .map_err(|e| match e {
-                MachineError::InvalidTransition => WFMachinesError::Fatal(format!(
-                    "Invalid transition resolving local activity (seq {}) in {}",
-                    self.shared_state.attrs.seq,
-                    self.state(),
-                )),
-                MachineError::Underlying(e) => e,
-            })?;
+        self._try_resolve(dat, true)
+    }
+
+    fn _try_resolve(
+        &mut self,
+        dat: ResolveDat,
+        from_marker: bool,
+    ) -> Result<Vec<MachineResponse>, WFMachinesError> {
+        let evt = if from_marker {
+            LocalActivityMachineEvents::HandleKnownResult(dat)
+        } else {
+            LocalActivityMachineEvents::HandleResult(dat)
+        };
+        let res = OnEventWrapper::on_event_mut(self, evt).map_err(|e| match e {
+            MachineError::InvalidTransition => WFMachinesError::Fatal(format!(
+                "Invalid transition resolving local activity (seq {}, from marker: {}) in {}",
+                self.shared_state.attrs.seq,
+                from_marker,
+                self.state(),
+            )),
+            MachineError::Underlying(e) => e,
+        })?;
 
         Ok(res
             .into_iter()
@@ -273,6 +320,7 @@ pub(super) struct SharedState {
     attrs: ValidScheduleLA,
     replaying_when_invoked: bool,
     wf_time_when_started: Option<SystemTime>,
+    internal_flags: InternalFlagsRef,
 }
 
 impl SharedState {
@@ -314,9 +362,11 @@ pub(super) struct Executing {}
 impl Executing {
     pub(super) fn on_schedule(
         self,
-        dat: SharedState,
+        dat: &mut SharedState,
     ) -> LocalActivityMachineTransition<RequestSent> {
-        TransitionResult::commands([LocalActivityCommand::RequestActivityExecution(dat.attrs)])
+        TransitionResult::commands([LocalActivityCommand::RequestActivityExecution(
+            dat.attrs.clone(),
+        )])
     }
 }
 
@@ -426,7 +476,7 @@ impl RequestSent {
 
     fn on_no_wait_cancel(
         self,
-        shared: SharedState,
+        shared: &mut SharedState,
         cancel_type: ActivityCancellationType,
     ) -> LocalActivityMachineTransition<MarkerCommandCreated> {
         let mut cmds = vec![];
@@ -465,7 +515,7 @@ pub(super) struct ResultNotified {
 impl ResultNotified {
     pub(super) fn on_marker_recorded(
         self,
-        shared: SharedState,
+        shared: &mut SharedState,
         dat: CompleteLocalActivityData,
     ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
         if self.result_type == ResultType::Completed && dat.result.is_err() {
@@ -480,7 +530,7 @@ impl ResultNotified {
                 shared.attrs.seq
             )));
         }
-        verify_marker_dat!(&shared, &dat, TransitionResult::default())
+        verify_marker_dat!(shared, &dat, TransitionResult::default())
     }
 }
 
@@ -492,11 +542,11 @@ pub(super) struct WaitingMarkerEvent {
 impl WaitingMarkerEvent {
     pub(super) fn on_marker_recorded(
         self,
-        shared: SharedState,
+        shared: &mut SharedState,
         dat: CompleteLocalActivityData,
     ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
         verify_marker_dat!(
-            &shared,
+            shared,
             &dat,
             TransitionResult::commands(if self.already_resolved {
                 vec![]
@@ -505,65 +555,53 @@ impl WaitingMarkerEvent {
             })
         )
     }
+    fn on_handle_result(
+        self,
+        dat: ResolveDat,
+    ) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
+        TransitionResult::ok(
+            [LocalActivityCommand::Resolved(dat)],
+            WaitingMarkerEvent {
+                already_resolved: true,
+            },
+        )
+    }
     pub(super) fn on_started_non_replay_wft(
         self,
-        mut dat: SharedState,
+        dat: &mut SharedState,
     ) -> LocalActivityMachineTransition<RequestSent> {
         // We aren't really "replaying" anymore for our purposes, and want to record the marker.
         dat.replaying_when_invoked = false;
-        TransitionResult::ok_shared(
-            [LocalActivityCommand::RequestActivityExecution(
-                dat.attrs.clone(),
-            )],
-            RequestSent::default(),
-            dat,
-        )
+        TransitionResult::commands([LocalActivityCommand::RequestActivityExecution(
+            dat.attrs.clone(),
+        )])
     }
 
-    fn on_cancel_requested(self) -> LocalActivityMachineTransition<WaitingMarkerEventCancelled> {
+    fn on_cancel_requested(self) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
         // We still "request a cancel" even though we know the local activity should not be running
         // because the data might be in the pre-resolved list.
-        TransitionResult::ok(
-            [LocalActivityCommand::RequestCancel],
-            WaitingMarkerEventCancelled {},
-        )
+        TransitionResult::ok([LocalActivityCommand::RequestCancel], self)
     }
 
     fn on_no_wait_cancel(
         self,
         _: ActivityCancellationType,
-    ) -> LocalActivityMachineTransition<WaitingMarkerEventCancelled> {
+    ) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
         // Markers are always recorded when cancelling, so this is the same as a normal cancel on
         // the replay path
         self.on_cancel_requested()
     }
 }
 
-#[derive(Default, Clone)]
-pub(super) struct WaitingMarkerEventCancelled {}
-impl WaitingMarkerEventCancelled {
-    fn on_handle_result(
-        self,
-        dat: ResolveDat,
-    ) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
-        TransitionResult::ok(
-            [LocalActivityCommand::Resolved(dat)],
-            WaitingMarkerEvent {
-                already_resolved: true,
-            },
-        )
-    }
-}
-
 #[derive(Default, Clone)]
 pub(super) struct WaitingMarkerEventPreResolved {}
 impl WaitingMarkerEventPreResolved {
     pub(super) fn on_marker_recorded(
         self,
-        shared: SharedState,
+        shared: &mut SharedState,
         dat: CompleteLocalActivityData,
     ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
-        verify_marker_dat!(&shared, &dat, TransitionResult::default())
+        verify_marker_dat!(shared, &dat, TransitionResult::default())
     }
 }
 
@@ -645,8 +683,61 @@ impl WFMachinesAdapter for LocalActivityMachine {
                         ),
                     }
                 } else {
-                    result.into()
+                    // Cancels and timeouts are to be wrapped with an activity failure
+                    macro_rules! wrap_fail {
+                        ($me:ident, $fail:ident, $msg:expr, $info:pat) => {
+                            let mut fail = $fail.failure.take();
+                            let fail_info = fail.as_ref().and_then(|f| f.failure_info.as_ref());
+                            if matches!(fail_info, Some($info)) {
+                                fail = Some(Failure {
+                                    message: $msg,
+                                    cause: fail.map(Box::new),
+                                    failure_info: Some(activity_fail_info(
+                                        $me.shared_state.attrs.activity_type.clone(),
+                                        $me.shared_state.attrs.activity_id.clone(),
+                                        None,
+                                        RetryState::CancelRequested,
+                                        0,
+                                        0,
+                                    )),
+                                    ..Default::default()
+                                });
+                            }
+                            $fail.failure = fail;
+                        };
+                    }
+                    match result {
+                        LocalActivityExecutionResult::Completed(c) => ActivityResolution {
+                            status: Some(c.into()),
+                        },
+                        LocalActivityExecutionResult::Failed(f) => ActivityResolution {
+                            status: Some(f.into()),
+                        },
+                        LocalActivityExecutionResult::TimedOut(mut failure) => {
+                            wrap_fail!(
+                                self,
+                                failure,
+                                "Local Activity timed out".to_string(),
+                                FailureInfo::TimeoutFailureInfo(_)
+                            );
+                            ActivityResolution {
+                                status: Some(failure.into()),
+                            }
+                        }
+                        LocalActivityExecutionResult::Cancelled(mut cancel) => {
+                            wrap_fail!(
+                                self,
+                                cancel,
+                                "Local Activity cancelled".to_string(),
+                                FailureInfo::CanceledFailureInfo(_)
+                            );
+                            ActivityResolution {
+                                status: Some(cancel.into()),
+                            }
+                        }
+                    }
                 };
+
                 let mut responses = vec![
                     MachineResponse::PushWFJob(OutgoingJob {
                         variant: ResolveActivity {
@@ -727,14 +818,14 @@ impl TryFrom<CommandType> for LocalActivityMachineEvents {
     }
 }
 
-impl TryFrom<HistoryEvent> for LocalActivityMachineEvents {
+impl TryFrom<HistEventData> for LocalActivityMachineEvents {
     type Error = WFMachinesError;
 
-    fn try_from(e: HistoryEvent) -> Result<Self, Self::Error> {
+    fn try_from(e: HistEventData) -> Result<Self, Self::Error> {
+        let e = e.event;
         if e.event_type() != EventType::MarkerRecorded {
             return Err(WFMachinesError::Nondeterminism(format!(
-                "Local activity machine cannot handle this event: {}",
-                e
+                "Local activity machine cannot handle this event: {e}"
             )));
         }
 
@@ -758,26 +849,30 @@ fn verify_marker_data_matches(
             dat.marker_dat.seq, shared.attrs.seq
         )));
     }
-
-    Ok(())
-}
-
-impl From<LocalActivityExecutionResult> for ActivityResolution {
-    fn from(lar: LocalActivityExecutionResult) -> Self {
-        match lar {
-            LocalActivityExecutionResult::Completed(c) => ActivityResolution {
-                status: Some(c.into()),
-            },
-            LocalActivityExecutionResult::Failed(f) | LocalActivityExecutionResult::TimedOut(f) => {
-                ActivityResolution {
-                    status: Some(f.into()),
-                }
-            }
-            LocalActivityExecutionResult::Cancelled(cancel) => ActivityResolution {
-                status: Some(cancel.into()),
-            },
+    // Here we use whether or not we were replaying when we _first invoked_ the LA, because we
+    // are always replaying when we see the marker recorded event, and that would make this check
+    // a bit pointless.
+    if shared.internal_flags.borrow_mut().try_use(
+        CoreInternalFlags::IdAndTypeDeterminismChecks,
+        !shared.replaying_when_invoked,
+    ) {
+        if dat.marker_dat.activity_id != shared.attrs.activity_id {
+            return Err(WFMachinesError::Nondeterminism(format!(
+                "Activity id of recorded marker '{}' does not \
+                 match activity id of local activity command '{}'",
+                dat.marker_dat.activity_id, shared.attrs.activity_id
+            )));
+        }
+        if dat.marker_dat.activity_type != shared.attrs.activity_type {
+            return Err(WFMachinesError::Nondeterminism(format!(
+                "Activity type of recorded marker '{}' does not \
+                 match activity type of local activity command '{}'",
+                dat.marker_dat.activity_type, shared.attrs.activity_type
+            )));
         }
     }
+
+    Ok(())
 }
 
 #[cfg(test)]
@@ -795,15 +890,19 @@ mod tests {
         coresdk::{
             activity_result::ActivityExecutionResult,
             workflow_activation::{workflow_activation_job, WorkflowActivationJob},
-            workflow_commands::ActivityCancellationType::WaitCancellationCompleted,
         },
         temporal::api::{
             command::v1::command, enums::v1::WorkflowTaskFailedCause, failure::v1::Failure,
         },
+        DEFAULT_ACTIVITY_TYPE,
     };
 
     async fn la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         Ok(().into())
     }
 
@@ -916,8 +1015,16 @@ mod tests {
     }
 
     async fn two_la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions::default()).await;
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         Ok(().into())
     }
 
@@ -1010,8 +1117,14 @@ mod tests {
 
     async fn two_la_wf_parallel(ctx: WfContext) -> WorkflowResult<()> {
         tokio::join!(
-            ctx.local_activity(LocalActivityOptions::default()),
-            ctx.local_activity(LocalActivityOptions::default())
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                ..Default::default()
+            }),
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                ..Default::default()
+            })
         );
         Ok(().into())
     }
@@ -1094,9 +1207,17 @@ mod tests {
     }
 
     async fn la_timer_la(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         ctx.timer(Duration::from_secs(5)).await;
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         Ok(().into())
     }
 
@@ -1337,6 +1458,7 @@ mod tests {
         let func = WorkflowFunction::new(move |ctx| async move {
             let la = ctx.local_activity(LocalActivityOptions {
                 cancel_type,
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
                 ..Default::default()
             });
             ctx.timer(Duration::from_secs(1)).await;
@@ -1346,20 +1468,29 @@ mod tests {
             ctx.timer(Duration::from_secs(1)).await;
             let resolution = la.await;
             assert!(resolution.cancelled());
+            let rfail = resolution.unwrap_failure();
+            assert_matches!(
+                rfail.failure_info,
+                Some(FailureInfo::ActivityFailureInfo(_))
+            );
+            assert_matches!(
+                rfail.cause.unwrap().failure_info,
+                Some(FailureInfo::CanceledFailureInfo(_))
+            );
             Ok(().into())
         });
 
         let mut t = TestHistoryBuilder::default();
         t.add_by_type(EventType::WorkflowExecutionStarted);
         t.add_full_wf_task();
-        let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
         t.add_timer_fired(timer_started_event_id, "1".to_string());
         t.add_full_wf_task();
         if cancel_type != ActivityCancellationType::WaitCancellationCompleted {
             // With non-wait cancels, the cancel is immediate
             t.add_local_activity_cancel_marker(1, "1");
         }
-        let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
         if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
             // With wait cancels, the cancel marker is not recorded until activity reports.
             t.add_local_activity_cancel_marker(1, "1");
@@ -1403,8 +1534,9 @@ mod tests {
             assert_eq!(commands[1].command_type, CommandType::StartTimer as i32);
         }
 
-        if replay {
-            wfm.get_next_activation().await.unwrap()
+        let commands = if replay {
+            wfm.get_next_activation().await.unwrap();
+            wfm.get_server_commands().commands
         } else {
             // On non replay, there's an additional activation, because completing with the cancel
             // wants to wake up the workflow to see if resolving the LA as cancelled did anything.
@@ -1412,7 +1544,7 @@ mod tests {
             // what would have happened if we woke up with new history -- but it does mean we
             // generate the commands at this point. This matters b/c we want to make sure the record
             // marker command is sent as soon as cancel happens.
-            if cancel_type == WaitCancellationCompleted {
+            if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
                 wfm.complete_local_activity(1, ActivityExecutionResult::cancel_from_details(None))
                     .unwrap();
             }
@@ -1429,11 +1561,11 @@ mod tests {
 
             wfm.new_history(t.get_history_info(3).unwrap().into())
                 .await
-                .unwrap()
+                .unwrap();
+            wfm.get_next_activation().await.unwrap();
+            wfm.get_server_commands().commands
         };
 
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
         assert_eq!(commands.len(), 1);
         assert_eq!(
             commands[0].command_type,