temporalio 0.0.2 → 0.1.0

data/bridge/sdk-core/core/src/worker/workflow/machines/fail_workflow_state_machine.rs

@@ -2,6 +2,7 @@ use super::{
     workflow_machines::MachineResponse, Cancellable, EventInfo, NewMachineWithCommand,
     OnEventWrapper, WFMachinesAdapter, WFMachinesError,
 };
+use crate::worker::workflow::machines::HistEventData;
 use rustfsm::{fsm, TransitionResult};
 use std::convert::TryFrom;
 use temporal_sdk_core_protos::{
@@ -85,16 +86,16 @@ impl From<FailWorkflowCommandCreated> for FailWorkflowCommandRecorded {
     }
 }
 
-impl TryFrom<HistoryEvent> for FailWorkflowMachineEvents {
+impl TryFrom<HistEventData> for FailWorkflowMachineEvents {
     type Error = WFMachinesError;
 
-    fn try_from(e: HistoryEvent) -> Result<Self, Self::Error> {
+    fn try_from(e: HistEventData) -> Result<Self, Self::Error> {
+        let e = e.event;
         Ok(match e.event_type() {
             EventType::WorkflowExecutionFailed => Self::WorkflowExecutionFailed,
             _ => {
                 return Err(WFMachinesError::Nondeterminism(format!(
-                    "Fail workflow machine does not handle this event: {}",
-                    e
+                    "Fail workflow machine does not handle this event: {e}"
                 )))
             }
         })

data/bridge/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs

@@ -3,8 +3,12 @@ use super::{
     WFMachinesError,
 };
 use crate::{
+    internal_flags::CoreInternalFlags,
     protosext::{CompleteLocalActivityData, HistoryEventExt, ValidScheduleLA},
-    worker::{workflow::OutgoingJob, LocalActivityExecutionResult},
+    worker::{
+        workflow::{machines::HistEventData, InternalFlagsRef, OutgoingJob},
+        LocalActivityExecutionResult,
+    },
 };
 use rustfsm::{fsm, MachineError, StateMachine, TransitionResult};
 use std::{
@@ -59,15 +63,17 @@ fsm! {
       --> MarkerCommandRecorded;
 
     // Replay path ================================================================================
-    // LAs on the replay path should never have handle result explicitly called on them, but do need
-    // to eventually see the marker
+    // LAs on the replay path always need to eventually see the marker
     WaitingMarkerEvent --(MarkerRecorded(CompleteLocalActivityData), shared on_marker_recorded)
       --> MarkerCommandRecorded;
     // If we are told to cancel while waiting for the marker, we still need to wait for the marker.
-    WaitingMarkerEvent --(Cancel, on_cancel_requested) --> WaitingMarkerEventCancelled;
+    WaitingMarkerEvent --(Cancel, on_cancel_requested) --> WaitingMarkerEvent;
+    // Because there could be non-heartbeat WFTs (ex: signals being received) between scheduling
+    // the LA and the marker being recorded, peekahead might not always resolve the LA *before*
+    // scheduling it. This transition accounts for that.
+    WaitingMarkerEvent --(HandleKnownResult(ResolveDat), on_handle_result) --> WaitingMarkerEvent;
     WaitingMarkerEvent --(NoWaitCancel(ActivityCancellationType),
-                          on_no_wait_cancel) --> WaitingMarkerEventCancelled;
-    WaitingMarkerEventCancelled --(HandleResult(ResolveDat), on_handle_result) --> WaitingMarkerEvent;
+                          on_no_wait_cancel) --> WaitingMarkerEvent;
 
     // It is entirely possible to have started the LA while replaying, only to find that we have
     // reached a new WFT and there still was no marker. In such cases we need to execute the LA.
@@ -139,6 +145,7 @@ pub(super) fn new_local_activity(
     replaying_when_invoked: bool,
     maybe_pre_resolved: Option<ResolveDat>,
     wf_time: Option<SystemTime>,
+    internal_flags: InternalFlagsRef,
 ) -> Result<(LocalActivityMachine, Vec<MachineResponse>), WFMachinesError> {
     let initial_state = if replaying_when_invoked {
         if let Some(dat) = maybe_pre_resolved {
@@ -166,6 +173,7 @@ pub(super) fn new_local_activity(
             attrs,
             replaying_when_invoked,
             wf_time_when_started: wf_time,
+            internal_flags,
         },
     };
 
@@ -202,6 +210,12 @@ impl LocalActivityMachine {
         }
     }
 
+    /// Returns true if the machine will willingly accept data from a marker in its current state.
+    /// IE: Calling [Self::try_resolve_with_dat] makes sense.
+    pub(super) fn will_accept_resolve_marker(&self) -> bool {
+        matches!(self.state, LocalActivityMachineState::WaitingMarkerEvent(_))
+    }
+
     /// Must be called if the workflow encounters a non-replay workflow task
     pub(super) fn encountered_non_replay_wft(
         &mut self,
@@ -240,28 +254,45 @@ impl LocalActivityMachine {
         backoff: Option<prost_types::Duration>,
         original_schedule_time: Option<SystemTime>,
     ) -> Result<Vec<MachineResponse>, WFMachinesError> {
-        self.try_resolve_with_dat(ResolveDat {
-            result,
-            complete_time: self.shared_state.wf_time_when_started.map(|t| t + runtime),
-            attempt,
-            backoff,
-            original_schedule_time,
-        })
+        self._try_resolve(
+            ResolveDat {
+                result,
+                complete_time: self.shared_state.wf_time_when_started.map(|t| t + runtime),
+                attempt,
+                backoff,
+                original_schedule_time,
+            },
+            false,
+        )
     }
+
     /// Attempt to resolve the local activity with already known data, ex pre-resolved data
     pub(super) fn try_resolve_with_dat(
         &mut self,
         dat: ResolveDat,
     ) -> Result<Vec<MachineResponse>, WFMachinesError> {
-        let res = OnEventWrapper::on_event_mut(self, LocalActivityMachineEvents::HandleResult(dat))
-            .map_err(|e| match e {
-                MachineError::InvalidTransition => WFMachinesError::Fatal(format!(
-                    "Invalid transition resolving local activity (seq {}) in {}",
-                    self.shared_state.attrs.seq,
-                    self.state(),
-                )),
-                MachineError::Underlying(e) => e,
-            })?;
+        self._try_resolve(dat, true)
+    }
+
+    fn _try_resolve(
+        &mut self,
+        dat: ResolveDat,
+        from_marker: bool,
+    ) -> Result<Vec<MachineResponse>, WFMachinesError> {
+        let evt = if from_marker {
+            LocalActivityMachineEvents::HandleKnownResult(dat)
+        } else {
+            LocalActivityMachineEvents::HandleResult(dat)
+        };
+        let res = OnEventWrapper::on_event_mut(self, evt).map_err(|e| match e {
+            MachineError::InvalidTransition => WFMachinesError::Fatal(format!(
+                "Invalid transition resolving local activity (seq {}, from marker: {}) in {}",
+                self.shared_state.attrs.seq,
+                from_marker,
+                self.state(),
+            )),
+            MachineError::Underlying(e) => e,
+        })?;
 
         Ok(res
             .into_iter()
@@ -278,6 +309,7 @@ pub(super) struct SharedState {
     attrs: ValidScheduleLA,
     replaying_when_invoked: bool,
     wf_time_when_started: Option<SystemTime>,
+    internal_flags: InternalFlagsRef,
 }
 
 impl SharedState {
@@ -510,6 +542,17 @@ impl WaitingMarkerEvent {
             })
         )
     }
+    fn on_handle_result(
+        self,
+        dat: ResolveDat,
+    ) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
+        TransitionResult::ok(
+            [LocalActivityCommand::Resolved(dat)],
+            WaitingMarkerEvent {
+                already_resolved: true,
+            },
+        )
+    }
     pub(super) fn on_started_non_replay_wft(
         self,
         mut dat: SharedState,
@@ -525,41 +568,22 @@ impl WaitingMarkerEvent {
         )
     }
 
-    fn on_cancel_requested(self) -> LocalActivityMachineTransition<WaitingMarkerEventCancelled> {
+    fn on_cancel_requested(self) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
         // We still "request a cancel" even though we know the local activity should not be running
         // because the data might be in the pre-resolved list.
-        TransitionResult::ok(
-            [LocalActivityCommand::RequestCancel],
-            WaitingMarkerEventCancelled {},
-        )
+        TransitionResult::ok([LocalActivityCommand::RequestCancel], self)
     }
 
     fn on_no_wait_cancel(
         self,
         _: ActivityCancellationType,
-    ) -> LocalActivityMachineTransition<WaitingMarkerEventCancelled> {
+    ) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
         // Markers are always recorded when cancelling, so this is the same as a normal cancel on
         // the replay path
         self.on_cancel_requested()
     }
 }
 
-#[derive(Default, Clone)]
-pub(super) struct WaitingMarkerEventCancelled {}
-impl WaitingMarkerEventCancelled {
-    fn on_handle_result(
-        self,
-        dat: ResolveDat,
-    ) -> LocalActivityMachineTransition<WaitingMarkerEvent> {
-        TransitionResult::ok(
-            [LocalActivityCommand::Resolved(dat)],
-            WaitingMarkerEvent {
-                already_resolved: true,
-            },
-        )
-    }
-}
-
 #[derive(Default, Clone)]
 pub(super) struct WaitingMarkerEventPreResolved {}
 impl WaitingMarkerEventPreResolved {
@@ -732,14 +756,14 @@ impl TryFrom<CommandType> for LocalActivityMachineEvents {
     }
 }
 
-impl TryFrom<HistoryEvent> for LocalActivityMachineEvents {
+impl TryFrom<HistEventData> for LocalActivityMachineEvents {
     type Error = WFMachinesError;
 
-    fn try_from(e: HistoryEvent) -> Result<Self, Self::Error> {
+    fn try_from(e: HistEventData) -> Result<Self, Self::Error> {
+        let e = e.event;
         if e.event_type() != EventType::MarkerRecorded {
             return Err(WFMachinesError::Nondeterminism(format!(
-                "Local activity machine cannot handle this event: {}",
-                e
+                "Local activity machine cannot handle this event: {e}"
             )));
         }
 
@@ -763,6 +787,28 @@ fn verify_marker_data_matches(
             dat.marker_dat.seq, shared.attrs.seq
         )));
     }
+    // Here we use whether or not we were replaying when we _first invoked_ the LA, because we
+    // are always replaying when we see the marker recorded event, and that would make this check
+    // a bit pointless.
+    if shared.internal_flags.borrow_mut().try_use(
+        CoreInternalFlags::IdAndTypeDeterminismChecks,
+        !shared.replaying_when_invoked,
+    ) {
+        if dat.marker_dat.activity_id != shared.attrs.activity_id {
+            return Err(WFMachinesError::Nondeterminism(format!(
+                "Activity id of recorded marker '{}' does not \
+                 match activity id of local activity command '{}'",
+                dat.marker_dat.activity_id, shared.attrs.activity_id
+            )));
+        }
+        if dat.marker_dat.activity_type != shared.attrs.activity_type {
+            return Err(WFMachinesError::Nondeterminism(format!(
+                "Activity type of recorded marker '{}' does not \
+                 match activity type of local activity command '{}'",
+                dat.marker_dat.activity_type, shared.attrs.activity_type
+            )));
+        }
+    }
 
     Ok(())
 }
@@ -805,10 +851,15 @@ mod tests {
         temporal::api::{
             command::v1::command, enums::v1::WorkflowTaskFailedCause, failure::v1::Failure,
         },
+        DEFAULT_ACTIVITY_TYPE,
     };
 
     async fn la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         Ok(().into())
     }
 
@@ -921,8 +972,16 @@ mod tests {
     }
 
     async fn two_la_wf(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions::default()).await;
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         Ok(().into())
     }
 
@@ -1015,8 +1074,14 @@ mod tests {
 
     async fn two_la_wf_parallel(ctx: WfContext) -> WorkflowResult<()> {
         tokio::join!(
-            ctx.local_activity(LocalActivityOptions::default()),
-            ctx.local_activity(LocalActivityOptions::default())
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                ..Default::default()
+            }),
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                ..Default::default()
+            })
         );
         Ok(().into())
     }
@@ -1099,9 +1164,17 @@ mod tests {
     }
 
     async fn la_timer_la(ctx: WfContext) -> WorkflowResult<()> {
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         ctx.timer(Duration::from_secs(5)).await;
-        ctx.local_activity(LocalActivityOptions::default()).await;
+        ctx.local_activity(LocalActivityOptions {
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+            ..Default::default()
+        })
+        .await;
         Ok(().into())
     }
 
@@ -1342,6 +1415,7 @@ mod tests {
         let func = WorkflowFunction::new(move |ctx| async move {
             let la = ctx.local_activity(LocalActivityOptions {
                 cancel_type,
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
                 ..Default::default()
             });
             ctx.timer(Duration::from_secs(1)).await;
@@ -1357,14 +1431,14 @@ mod tests {
         let mut t = TestHistoryBuilder::default();
         t.add_by_type(EventType::WorkflowExecutionStarted);
         t.add_full_wf_task();
-        let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
         t.add_timer_fired(timer_started_event_id, "1".to_string());
         t.add_full_wf_task();
         if cancel_type != ActivityCancellationType::WaitCancellationCompleted {
             // With non-wait cancels, the cancel is immediate
             t.add_local_activity_cancel_marker(1, "1");
         }
-        let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+        let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
         if cancel_type == ActivityCancellationType::WaitCancellationCompleted {
             // With wait cancels, the cancel marker is not recorded until activity reports.
             t.add_local_activity_cancel_marker(1, "1");
@@ -1408,8 +1482,9 @@ mod tests {
             assert_eq!(commands[1].command_type, CommandType::StartTimer as i32);
         }
 
-        if replay {
-            wfm.get_next_activation().await.unwrap()
+        let commands = if replay {
+            wfm.get_next_activation().await.unwrap();
+            wfm.get_server_commands().commands
         } else {
             // On non replay, there's an additional activation, because completing with the cancel
             // wants to wake up the workflow to see if resolving the LA as cancelled did anything.
@@ -1434,11 +1509,11 @@ mod tests {
 
             wfm.new_history(t.get_history_info(3).unwrap().into())
                 .await
-                .unwrap()
+                .unwrap();
+            wfm.get_next_activation().await.unwrap();
+            wfm.get_server_commands().commands
         };
 
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
         assert_eq!(commands.len(), 1);
         assert_eq!(
             commands[0].command_type,

data/bridge/sdk-core/core/src/worker/workflow/machines/mod.rs

@@ -18,9 +18,9 @@ mod workflow_task_state_machine;
 #[cfg(test)]
 mod transition_coverage;
 
-pub(crate) use workflow_machines::{WFMachinesError, WorkflowMachines};
+pub(crate) use workflow_machines::WorkflowMachines;
 
-use crate::telemetry::VecDisplayer;
+use crate::{telemetry::VecDisplayer, worker::workflow::WFMachinesError};
 use activity_state_machine::ActivityMachine;
 use cancel_external_state_machine::CancelExternalMachine;
 use cancel_workflow_state_machine::CancelWorkflowMachine;
@@ -73,7 +73,7 @@ enum Machines {
 ///
 /// Formerly known as `EntityStateMachine` in Java.
 #[enum_dispatch::enum_dispatch(Machines)]
-trait TemporalStateMachine: Send {
+trait TemporalStateMachine {
     fn handle_command(
         &mut self,
         command_type: CommandType,
@@ -88,8 +88,7 @@ trait TemporalStateMachine: Send {
     /// to update the overall state of the workflow. EX: To issue outgoing WF activations.
     fn handle_event(
         &mut self,
-        event: HistoryEvent,
-        has_next_event: bool,
+        event: HistEventData,
     ) -> Result<Vec<MachineResponse>, WFMachinesError>;
 
     /// Attempt to cancel the command associated with this state machine, if it is cancellable
@@ -108,9 +107,9 @@ trait TemporalStateMachine: Send {
 
 impl<SM> TemporalStateMachine for SM
 where
-    SM: StateMachine + WFMachinesAdapter + Cancellable + OnEventWrapper + Clone + Send + 'static,
-    <SM as StateMachine>::Event: TryFrom<HistoryEvent> + TryFrom<CommandType> + Display,
-    WFMachinesError: From<<<SM as StateMachine>::Event as TryFrom<HistoryEvent>>::Error>,
+    SM: StateMachine + WFMachinesAdapter + Cancellable + OnEventWrapper + Clone + 'static,
+    <SM as StateMachine>::Event: TryFrom<HistEventData> + TryFrom<CommandType> + Display,
+    WFMachinesError: From<<<SM as StateMachine>::Event as TryFrom<HistEventData>>::Error>,
     <SM as StateMachine>::Command: Debug + Display,
     <SM as StateMachine>::State: Display,
     <SM as StateMachine>::Error: Into<WFMachinesError> + 'static + Send + Sync,
@@ -152,21 +151,19 @@ where
 
     fn handle_event(
         &mut self,
-        event: HistoryEvent,
-        has_next_event: bool,
+        event_dat: HistEventData,
     ) -> Result<Vec<MachineResponse>, WFMachinesError> {
         trace!(
-            event = %event,
+            event = %event_dat.event,
             machine_name = %self.name(),
             state = %self.state(),
             "handling event"
         );
         let event_info = EventInfo {
-            event_id: event.event_id,
-            event_type: event.event_type(),
-            has_next_event,
+            event_id: event_dat.event.event_id,
+            event_type: event_dat.event.event_type(),
         };
-        let converted_event: <Self as StateMachine>::Event = event.try_into()?;
+        let converted_event: <Self as StateMachine>::Event = event_dat.try_into()?;
 
         match OnEventWrapper::on_event_mut(self, converted_event) {
             Ok(c) => process_machine_commands(self, c, Some(event_info)),
@@ -217,7 +214,7 @@ where
     <SM as StateMachine>::State: Display,
 {
     if !commands.is_empty() {
-        debug!(commands=%commands.display(), state=%machine.state(),
+        trace!(commands=%commands.display(), state=%machine.state(),
                machine_name=%TemporalStateMachine::name(machine), "Machine produced commands");
     }
     let mut machine_responses = vec![];
@@ -245,11 +242,22 @@ trait WFMachinesAdapter: StateMachine {
     fn matches_event(&self, event: &HistoryEvent) -> bool;
 }
 
+/// Wraps a history event with extra relevant data that a machine might care about while the event
+/// is being applied to it.
+#[derive(Debug, derive_more::Display)]
+#[display(fmt = "{event}")]
+struct HistEventData {
+    event: HistoryEvent,
+    /// Is the current workflow task under replay or not during application of this event?
+    replaying: bool,
+    /// Is the current workflow task the last task in history?
+    current_task_is_last_in_history: bool,
+}
+
 #[derive(Debug, Copy, Clone)]
 struct EventInfo {
     event_id: i64,
     event_type: EventType,
-    has_next_event: bool,
 }
 
 trait Cancellable: StateMachine {

data/bridge/sdk-core/core/src/worker/workflow/machines/modify_workflow_properties_state_machine.rs

@@ -1,8 +1,8 @@
-use super::{
-    workflow_machines::{MachineResponse, WFMachinesError},
-    NewMachineWithCommand,
+use super::{workflow_machines::MachineResponse, NewMachineWithCommand};
+use crate::worker::workflow::{
+    machines::{Cancellable, EventInfo, HistEventData, WFMachinesAdapter},
+    WFMachinesError,
 };
-use crate::worker::workflow::machines::{Cancellable, EventInfo, WFMachinesAdapter};
 use rustfsm::{fsm, TransitionResult};
 use temporal_sdk_core_protos::{
     coresdk::workflow_commands::ModifyWorkflowProperties,
@@ -74,10 +74,11 @@ impl WFMachinesAdapter for ModifyWorkflowPropertiesMachine {
 
 impl Cancellable for ModifyWorkflowPropertiesMachine {}
 
-impl TryFrom<HistoryEvent> for ModifyWorkflowPropertiesMachineEvents {
+impl TryFrom<HistEventData> for ModifyWorkflowPropertiesMachineEvents {
     type Error = WFMachinesError;
 
-    fn try_from(e: HistoryEvent) -> Result<Self, Self::Error> {
+    fn try_from(e: HistEventData) -> Result<Self, Self::Error> {
+        let e = e.event;
         match e.event_type() {
             EventType::WorkflowPropertiesModified => {
                 Ok(ModifyWorkflowPropertiesMachineEvents::CommandRecorded)