@temporalio/core-bridge 1.6.0 → 1.7.1

package/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs

@@ -3,8 +3,15 @@ use super::{
     WFMachinesError,
 };
 use crate::{
+    internal_flags::CoreInternalFlags,
     protosext::{CompleteLocalActivityData, HistoryEventExt, ValidScheduleLA},
-    worker::{workflow::OutgoingJob, LocalActivityExecutionResult},
+    worker::{
+        workflow::{
+            machines::{activity_state_machine::activity_fail_info, HistEventData},
+            InternalFlagsRef, OutgoingJob,
+        },
+        LocalActivityExecutionResult,
+    },
 };
 use rustfsm::{fsm, MachineError, StateMachine, TransitionResult};
 use std::{
@@ -24,8 +31,8 @@ use temporal_sdk_core_protos::{
     },
     temporal::api::{
         command::v1::{Command, RecordMarkerCommandAttributes},
-        enums::v1::{CommandType, EventType},
-        failure::v1::failure::FailureInfo,
+        enums::v1::{CommandType, EventType, RetryState},
+        failure::v1::{failure::FailureInfo, Failure},
         history::v1::HistoryEvent,
     },
     utilities::TryIntoOrNone,
@@ -113,7 +120,12 @@ impl From<CompleteLocalActivityData> for ResolveDat {
             result: match d.result {
                 Ok(res) => LocalActivityExecutionResult::Completed(Success { result: Some(res) }),
                 Err(fail) => {
-                    if matches!(fail.failure_info, Some(FailureInfo::CanceledFailureInfo(_))) {
+                    if matches!(fail.failure_info, Some(FailureInfo::CanceledFailureInfo(_)))
+                        || matches!(
+                            fail.cause.as_deref().and_then(|f| f.failure_info.as_ref()),
+                            Some(FailureInfo::CanceledFailureInfo(_))
+                        )
+                    {
                         LocalActivityExecutionResult::Cancelled(Cancellation {
                             failure: Some(fail),
                         })
@@ -141,6 +153,7 @@ pub(super) fn new_local_activity(
     replaying_when_invoked: bool,
     maybe_pre_resolved: Option<ResolveDat>,
     wf_time: Option<SystemTime>,
+    internal_flags: InternalFlagsRef,
 ) -> Result<(LocalActivityMachine, Vec<MachineResponse>), WFMachinesError> {
     let initial_state = if replaying_when_invoked {
         if let Some(dat) = maybe_pre_resolved {
@@ -162,14 +175,15 @@ pub(super) fn new_local_activity(
         .original_schedule_time
         .get_or_insert(SystemTime::now());
 
-    let mut machine = LocalActivityMachine {
-        state: initial_state,
-        shared_state: SharedState {
+    let mut machine = LocalActivityMachine::from_parts(
+        initial_state,
+        SharedState {
             attrs,
             replaying_when_invoked,
             wf_time_when_started: wf_time,
+            internal_flags,
         },
-    };
+    );
 
     let mut res = OnEventWrapper::on_event_mut(&mut machine, LocalActivityMachineEvents::Schedule)
         .expect("Scheduling local activities doesn't fail");
@@ -193,13 +207,13 @@ impl LocalActivityMachine {
     ///
     /// Attempting the check in any other state likely means a bug in the SDK.
     pub(super) fn marker_should_get_special_handling(&self) -> Result<bool, WFMachinesError> {
-        match &self.state {
+        match self.state() {
             LocalActivityMachineState::ResultNotified(_) => Ok(false),
             LocalActivityMachineState::WaitingMarkerEvent(_) => Ok(true),
             LocalActivityMachineState::WaitingMarkerEventPreResolved(_) => Ok(true),
             _ => Err(WFMachinesError::Fatal(format!(
                 "Attempted to check for LA marker handling in invalid state {}",
-                self.state
+                self.state()
             ))),
         }
     }
@@ -207,7 +221,10 @@ impl LocalActivityMachine {
     /// Returns true if the machine will willingly accept data from a marker in its current state.
     /// IE: Calling [Self::try_resolve_with_dat] makes sense.
     pub(super) fn will_accept_resolve_marker(&self) -> bool {
-        matches!(self.state, LocalActivityMachineState::WaitingMarkerEvent(_))
+        matches!(
+            self.state(),
+            LocalActivityMachineState::WaitingMarkerEvent(_)
+        )
     }
 
     /// Must be called if the workflow encounters a non-replay workflow task
@@ -303,6 +320,7 @@ pub(super) struct SharedState {
     attrs: ValidScheduleLA,
     replaying_when_invoked: bool,
     wf_time_when_started: Option<SystemTime>,
+    internal_flags: InternalFlagsRef,
 }
 
 impl SharedState {
@@ -344,9 +362,11 @@ pub(super) struct Executing {}
 impl Executing {
     pub(super) fn on_schedule(
         self,
-        dat: SharedState,
+        dat: &mut SharedState,
     ) -> LocalActivityMachineTransition<RequestSent> {
-        TransitionResult::commands([LocalActivityCommand::RequestActivityExecution(dat.attrs)])
+        TransitionResult::commands([LocalActivityCommand::RequestActivityExecution(
+            dat.attrs.clone(),
+        )])
     }
 }
 
@@ -456,7 +476,7 @@ impl RequestSent {
 
     fn on_no_wait_cancel(
         self,
-        shared: SharedState,
+        shared: &mut SharedState,
         cancel_type: ActivityCancellationType,
     ) -> LocalActivityMachineTransition<MarkerCommandCreated> {
         let mut cmds = vec![];
@@ -495,7 +515,7 @@ pub(super) struct ResultNotified {
 impl ResultNotified {
     pub(super) fn on_marker_recorded(
         self,
-        shared: SharedState,
+        shared: &mut SharedState,
         dat: CompleteLocalActivityData,
     ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
         if self.result_type == ResultType::Completed && dat.result.is_err() {
@@ -510,7 +530,7 @@ impl ResultNotified {
                 shared.attrs.seq
             )));
         }
-        verify_marker_dat!(&shared, &dat, TransitionResult::default())
+        verify_marker_dat!(shared, &dat, TransitionResult::default())
     }
 }
 
@@ -522,11 +542,11 @@ pub(super) struct WaitingMarkerEvent {
 impl WaitingMarkerEvent {
     pub(super) fn on_marker_recorded(
         self,
-        shared: SharedState,
+        shared: &mut SharedState,
         dat: CompleteLocalActivityData,
     ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
         verify_marker_dat!(
-            &shared,
+            shared,
             &dat,
             TransitionResult::commands(if self.already_resolved {
                 vec![]
@@ -548,17 +568,13 @@ impl WaitingMarkerEvent {
     }
     pub(super) fn on_started_non_replay_wft(
         self,
-        mut dat: SharedState,
+        dat: &mut SharedState,
     ) -> LocalActivityMachineTransition<RequestSent> {
         // We aren't really "replaying" anymore for our purposes, and want to record the marker.
         dat.replaying_when_invoked = false;
-        TransitionResult::ok_shared(
-            [LocalActivityCommand::RequestActivityExecution(
-                dat.attrs.clone(),
-            )],
-            RequestSent::default(),
-            dat,
-        )
+        TransitionResult::commands([LocalActivityCommand::RequestActivityExecution(
+            dat.attrs.clone(),
+        )])
     }
 
     fn on_cancel_requested(self) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
@@ -582,10 +598,10 @@ pub(super) struct WaitingMarkerEventPreResolved {}
 impl WaitingMarkerEventPreResolved {
     pub(super) fn on_marker_recorded(
         self,
-        shared: SharedState,
+        shared: &mut SharedState,
         dat: CompleteLocalActivityData,
     ) -> LocalActivityMachineTransition<MarkerCommandRecorded> {
-        verify_marker_dat!(&shared, &dat, TransitionResult::default())
+        verify_marker_dat!(shared, &dat, TransitionResult::default())
     }
 }
 
@@ -667,8 +683,61 @@ impl WFMachinesAdapter for LocalActivityMachine {
                         ),
                     }
                 } else {
-                    result.into()
+                    // Cancels and timeouts are to be wrapped with an activity failure
+                    macro_rules! wrap_fail {
+                        ($me:ident, $fail:ident, $msg:expr, $info:pat) => {
+                            let mut fail = $fail.failure.take();
+                            let fail_info = fail.as_ref().and_then(|f| f.failure_info.as_ref());
+                            if matches!(fail_info, Some($info)) {
+                                fail = Some(Failure {
+                                    message: $msg,
+                                    cause: fail.map(Box::new),
+                                    failure_info: Some(activity_fail_info(
+                                        $me.shared_state.attrs.activity_type.clone(),
+                                        $me.shared_state.attrs.activity_id.clone(),
+                                        None,
+                                        RetryState::CancelRequested,
+                                        0,
+                                        0,
+                                    )),
+                                    ..Default::default()
+                                });
+                            }
+                            $fail.failure = fail;
+                        };
+                    }
+                    match result {
+                        LocalActivityExecutionResult::Completed(c) => ActivityResolution {
+                            status: Some(c.into()),
+                        },
+                        LocalActivityExecutionResult::Failed(f) => ActivityResolution {
+                            status: Some(f.into()),
+                        },
+                        LocalActivityExecutionResult::TimedOut(mut failure) => {
+                            wrap_fail!(
+                                self,
+                                failure,
+                                "Local Activity timed out".to_string(),
+                                FailureInfo::TimeoutFailureInfo(_)
+                            );
+                            ActivityResolution {
+                                status: Some(failure.into()),
+                            }
+                        }
+                        LocalActivityExecutionResult::Cancelled(mut cancel) => {
+                            wrap_fail!(
+                                self,
+                                cancel,
+                                "Local Activity cancelled".to_string(),
+                                FailureInfo::CanceledFailureInfo(_)
+                            );
+                            ActivityResolution {
+                                status: Some(cancel.into()),
+                            }
+                        }
+                    }
                 };
+
                 let mut responses = vec![
                     MachineResponse::PushWFJob(OutgoingJob {
                         variant: ResolveActivity {
@@ -749,10 +818,11 @@ impl TryFrom<CommandType> for LocalActivityMachineEvents {
     }
 }
 
-impl TryFrom<HistoryEvent> for LocalActivityMachineEvents {
+impl TryFrom<HistEventData> for LocalActivityMachineEvents {
     type Error = WFMachinesError;
 
-    fn try_from(e: HistoryEvent) -> Result<Self, Self::Error> {
+    fn try_from(e: HistEventData) -> Result<Self, Self::Error> {
+        let e = e.event;
         if e.event_type() != EventType::MarkerRecorded {
             return Err(WFMachinesError::Nondeterminism(format!(
                 "Local activity machine cannot handle this event: {e}"
@@ -779,26 +849,30 @@ fn verify_marker_data_matches(
             dat.marker_dat.seq, shared.attrs.seq
         )));
     }
-
-    Ok(())
-}
-
-impl From<LocalActivityExecutionResult> for ActivityResolution {
-    fn from(lar: LocalActivityExecutionResult) -> Self {
-        match lar {
-            LocalActivityExecutionResult::Completed(c) => ActivityResolution {
-                status: Some(c.into()),
-            },
-            LocalActivityExecutionResult::Failed(f) | LocalActivityExecutionResult::TimedOut(f) => {
-                ActivityResolution {
-                    status: Some(f.into()),
-                }
-            }
-            LocalActivityExecutionResult::Cancelled(cancel) => ActivityResolution {
-                status: Some(cancel.into()),
-            },
+    // Here we use whether or not we were replaying when we _first invoked_ the LA, because we
+    // are always replaying when we see the marker recorded event, and that would make this check
+    // a bit pointless.
+    if shared.internal_flags.borrow_mut().try_use(
+        CoreInternalFlags::IdAndTypeDeterminismChecks,
+        !shared.replaying_when_invoked,
+    ) {
+        if dat.marker_dat.activity_id != shared.attrs.activity_id {
+            return Err(WFMachinesError::Nondeterminism(format!(
+                "Activity id of recorded marker '{}' does not \
+                 match activity id of local activity command '{}'",
+                dat.marker_dat.activity_id, shared.attrs.activity_id
+            )));
+        }
+        if dat.marker_dat.activity_type != shared.attrs.activity_type {
+            return Err(WFMachinesError::Nondeterminism(format!(
+                "Activity type of recorded marker '{}' does not \
+                 match activity type of local activity command '{}'",
+                dat.marker_dat.activity_type, shared.attrs.activity_type
+            )));
         }
     }
+
+    Ok(())
 }
 
 #[cfg(test)]
@@ -816,15 +890,19 @@ mod tests {
         coresdk::{
             activity_result::ActivityExecutionResult,
             workflow_activation::{workflow_activation_job, WorkflowActivationJob},
-            workflow_commands::ActivityCancellationType::WaitCancellationCompleted,
         },
         temporal::api::{
             command::v1::command, enums::v1::WorkflowTaskFailedCause, failure::v1::Failure,
         },
+        DEFAULT_ACTIVITY_TYPE,
     };
 
     async fn la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         Ok(().into())
     }
 
@@ -937,8 +1015,16 @@ mod tests {
     }
 
     async fn two_la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions::default()).await;
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         Ok(().into())
     }
 
@@ -1031,8 +1117,14 @@ mod tests {
 
     async fn two_la_wf_parallel(ctx: WfContext) -> WorkflowResult<()> {
         tokio::join!(
-            ctx.local_activity(LocalActivityOptions::default()),
-            ctx.local_activity(LocalActivityOptions::default())
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                ..Default::default()
+            }),
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                ..Default::default()
+            })
         );
         Ok(().into())
     }
@@ -1115,9 +1207,17 @@ mod tests {
     }
 
     async fn la_timer_la(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         ctx.timer(Duration::from_secs(5)).await;
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         Ok(().into())
     }
 
@@ -1358,6 +1458,7 @@ mod tests {
         let func = WorkflowFunction::new(move |ctx| async move {
             let la = ctx.local_activity(LocalActivityOptions {
                 cancel_type,
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
                 ..Default::default()
             });
             ctx.timer(Duration::from_secs(1)).await;
@@ -1367,20 +1468,29 @@ mod tests {
             ctx.timer(Duration::from_secs(1)).await;
             let resolution = la.await;
             assert!(resolution.cancelled());
+            let rfail = resolution.unwrap_failure();
+            assert_matches!(
+                rfail.failure_info,
+                Some(FailureInfo::ActivityFailureInfo(_))
+            );
+            assert_matches!(
+                rfail.cause.unwrap().failure_info,
+                Some(FailureInfo::CanceledFailureInfo(_))
+            );
             Ok(().into())
         });
 
         let mut t = TestHistoryBuilder::default();
         t.add_by_type(EventType::WorkflowExecutionStarted);
         t.add_full_wf_task();
-        let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
         t.add_timer_fired(timer_started_event_id, "1".to_string());
         t.add_full_wf_task();
         if cancel_type != ActivityCancellationType::WaitCancellationCompleted {
             // With non-wait cancels, the cancel is immediate
             t.add_local_activity_cancel_marker(1, "1");
         }
-        let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
         if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
             // With wait cancels, the cancel marker is not recorded until activity reports.
             t.add_local_activity_cancel_marker(1, "1");
@@ -1434,7 +1544,7 @@ mod tests {
             // what would have happened if we woke up with new history -- but it does mean we
             // generate the commands at this point. This matters b/c we want to make sure the record
             // marker command is sent as soon as cancel happens.
-            if cancel_type == WaitCancellationCompleted {
+            if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
                 wfm.complete_local_activity(1, ActivityExecutionResult::cancel_from_details(None))
                     .unwrap();
             }

package/sdk-core/core/src/worker/workflow/machines/mod.rs

@@ -73,7 +73,7 @@ enum Machines {
 ///
 /// Formerly known as `EntityStateMachine` in Java.
 #[enum_dispatch::enum_dispatch(Machines)]
-trait TemporalStateMachine: Send {
+trait TemporalStateMachine {
     fn handle_command(
         &mut self,
         command_type: CommandType,
@@ -88,8 +88,7 @@ trait TemporalStateMachine: Send {
     /// to update the overall state of the workflow. EX: To issue outgoing WF activations.
     fn handle_event(
         &mut self,
-        event: HistoryEvent,
-        has_next_event: bool,
+        event: HistEventData,
     ) -> Result<Vec<MachineResponse>, WFMachinesError>;
 
     /// Attempt to cancel the command associated with this state machine, if it is cancellable
@@ -108,9 +107,9 @@ trait TemporalStateMachine: Send {
 
 impl<SM> TemporalStateMachine for SM
 where
-    SM: StateMachine + WFMachinesAdapter + Cancellable + OnEventWrapper + Clone + Send + 'static,
-    <SM as StateMachine>::Event: TryFrom<HistoryEvent> + TryFrom<CommandType> + Display,
-    WFMachinesError: From<<<SM as StateMachine>::Event as TryFrom<HistoryEvent>>::Error>,
+    SM: StateMachine + WFMachinesAdapter + Cancellable + OnEventWrapper + Clone + 'static,
+    <SM as StateMachine>::Event: TryFrom<HistEventData> + TryFrom<CommandType> + Display,
+    WFMachinesError: From<<<SM as StateMachine>::Event as TryFrom<HistEventData>>::Error>,
     <SM as StateMachine>::Command: Debug + Display,
     <SM as StateMachine>::State: Display,
     <SM as StateMachine>::Error: Into<WFMachinesError> + 'static + Send + Sync,
@@ -152,21 +151,19 @@ where
 
     fn handle_event(
         &mut self,
-        event: HistoryEvent,
-        has_next_event: bool,
+        event_dat: HistEventData,
     ) -> Result<Vec<MachineResponse>, WFMachinesError> {
         trace!(
-            event = %event,
+            event = %event_dat.event,
             machine_name = %self.name(),
             state = %self.state(),
             "handling event"
         );
         let event_info = EventInfo {
-            event_id: event.event_id,
-            event_type: event.event_type(),
-            has_next_event,
+            event_id: event_dat.event.event_id,
+            event_type: event_dat.event.event_type(),
         };
-        let converted_event: <Self as StateMachine>::Event = event.try_into()?;
+        let converted_event: <Self as StateMachine>::Event = event_dat.try_into()?;
 
         match OnEventWrapper::on_event_mut(self, converted_event) {
             Ok(c) => process_machine_commands(self, c, Some(event_info)),
@@ -245,11 +242,22 @@ trait WFMachinesAdapter: StateMachine {
     fn matches_event(&self, event: &HistoryEvent) -> bool;
 }
 
+/// Wraps a history event with extra relevant data that a machine might care about while the event
+/// is being applied to it.
+#[derive(Debug, derive_more::Display)]
+#[display(fmt = "{event}")]
+struct HistEventData {
+    event: HistoryEvent,
+    /// Is the current workflow task under replay or not during application of this event?
+    replaying: bool,
+    /// Is the current workflow task the last task in history?
+    current_task_is_last_in_history: bool,
+}
+
 #[derive(Debug, Copy, Clone)]
 struct EventInfo {
     event_id: i64,
     event_type: EventType,
-    has_next_event: bool,
 }
 
 trait Cancellable: StateMachine {
@@ -270,7 +278,7 @@ trait Cancellable: StateMachine {
     }
 }
 
-/// We need to wrap calls to [StateMachine::on_event_mut] to track coverage, or anything else
+/// We need to wrap calls to [StateMachine::on_event] to track coverage, or anything else
 /// we'd like to do on every call.
 pub(crate) trait OnEventWrapper: StateMachine
 where
@@ -287,7 +295,7 @@ where
         #[cfg(test)]
         let converted_event_str = event.to_string();
 
-        let res = StateMachine::on_event_mut(self, event);
+        let res = StateMachine::on_event(self, event);
         if res.is_ok() {
             #[cfg(test)]
             add_coverage(

package/sdk-core/core/src/worker/workflow/machines/modify_workflow_properties_state_machine.rs

@@ -1,9 +1,9 @@
 use super::{workflow_machines::MachineResponse, NewMachineWithCommand};
 use crate::worker::workflow::{
-    machines::{Cancellable, EventInfo, WFMachinesAdapter},
+    machines::{Cancellable, EventInfo, HistEventData, WFMachinesAdapter},
     WFMachinesError,
 };
-use rustfsm::{fsm, TransitionResult};
+use rustfsm::{fsm, StateMachine, TransitionResult};
 use temporal_sdk_core_protos::{
     coresdk::workflow_commands::ModifyWorkflowProperties,
     temporal::api::{
@@ -28,10 +28,7 @@ fsm! {
 pub(super) fn modify_workflow_properties(
     lang_cmd: ModifyWorkflowProperties,
 ) -> NewMachineWithCommand {
-    let sm = ModifyWorkflowPropertiesMachine {
-        state: Created {}.into(),
-        shared_state: (),
-    };
+    let sm = ModifyWorkflowPropertiesMachine::from_parts(Created {}.into(), ());
     let cmd = Command {
         command_type: CommandType::ModifyWorkflowProperties as i32,
         attributes: Some(lang_cmd.into()),
@@ -74,10 +71,11 @@ impl WFMachinesAdapter for ModifyWorkflowPropertiesMachine {
 
 impl Cancellable for ModifyWorkflowPropertiesMachine {}
 
-impl TryFrom<HistoryEvent> for ModifyWorkflowPropertiesMachineEvents {
+impl TryFrom<HistEventData> for ModifyWorkflowPropertiesMachineEvents {
     type Error = WFMachinesError;
 
-    fn try_from(e: HistoryEvent) -> Result<Self, Self::Error> {
+    fn try_from(e: HistEventData) -> Result<Self, Self::Error> {
+        let e = e.event;
         match e.event_type() {
             EventType::WorkflowPropertiesModified => {
                 Ok(ModifyWorkflowPropertiesMachineEvents::CommandRecorded)