@temporalio/core-bridge 1.4.4 → 1.5.1

package/sdk-core/core/src/worker/workflow/history_update.rs

@@ -204,20 +204,20 @@ impl HistoryUpdate {
         &mut self,
         from_wft_started_id: i64,
     ) -> Result<Vec<HistoryEvent>, tonic::Status> {
-        let (next_wft_events, maybe_bonus_event) = self
+        let (next_wft_events, maybe_bonus_events) = self
             .take_next_wft_sequence_impl(from_wft_started_id)
             .await?;
-        if let Some(be) = maybe_bonus_event {
-            self.buffered.push_back(be);
+        if !maybe_bonus_events.is_empty() {
+            self.buffered.extend(maybe_bonus_events);
         }
 
         if let Some(last_event_id) = next_wft_events.last().map(|he| he.event_id) {
             // Always attempt to fetch the *next* WFT sequence as well, to buffer it for lookahead
-            let (buffer_these_events, maybe_bonus_event) =
+            let (buffer_these_events, maybe_bonus_events) =
                 self.take_next_wft_sequence_impl(last_event_id).await?;
             self.buffered.extend(buffer_these_events);
-            if let Some(be) = maybe_bonus_event {
-                self.buffered.push_back(be);
+            if !maybe_bonus_events.is_empty() {
+                self.buffered.extend(maybe_bonus_events);
             }
         }
 
@@ -232,44 +232,61 @@ impl HistoryUpdate {
         self.buffered.iter()
     }
 
+    /// Retrieve the next WFT sequence, first from buffered events and then from the real stream.
+    /// Returns (events up to the next logical wft sequence, extra events that were taken but
+    /// should be re-appended to the end of the buffer).
     async fn take_next_wft_sequence_impl(
         &mut self,
         from_event_id: i64,
-    ) -> Result<(Vec<HistoryEvent>, Option<HistoryEvent>), tonic::Status> {
+    ) -> Result<(Vec<HistoryEvent>, Vec<HistoryEvent>), tonic::Status> {
         let mut events_to_next_wft_started: Vec<HistoryEvent> = vec![];
 
         // This flag tracks if, while determining events to be returned, we have seen the next
         // logically significant WFT started event which follows the one that was passed in as a
-        // parameter. If a WFT fails or times out, it is not significant. So we will stop returning
-        // events (exclusive) as soon as we see an event following a WFT started that is *not*
-        // failed or timed out.
-        let mut saw_next_wft = false;
+        // parameter. If a WFT fails, times out, or is devoid of commands (ie: a heartbeat) it is
+        // not significant. So we will stop returning events (exclusive) as soon as we see an event
+        // following a WFT started that is *not* failed, timed out, or completed with a command.
+        let mut next_wft_state = NextWftState::NotSeen;
         let mut should_pop = |e: &HistoryEvent| {
             if e.event_id <= from_event_id {
                 return true;
-            } else if e.event_type == EventType::WorkflowTaskStarted as i32 {
-                saw_next_wft = true;
+            } else if e.event_type() == EventType::WorkflowTaskStarted {
+                next_wft_state = NextWftState::Seen;
                 return true;
             }
 
-            if saw_next_wft {
-                // Must ignore failures and timeouts
-                if e.event_type == EventType::WorkflowTaskFailed as i32
-                    || e.event_type == EventType::WorkflowTaskTimedOut as i32
-                {
-                    saw_next_wft = false;
-                    return true;
+            match next_wft_state {
+                NextWftState::Seen => {
+                    // Must ignore failures and timeouts
+                    if e.event_type() == EventType::WorkflowTaskFailed
+                        || e.event_type() == EventType::WorkflowTaskTimedOut
+                    {
+                        next_wft_state = NextWftState::NotSeen;
+                        return true;
+                    } else if e.event_type() == EventType::WorkflowTaskCompleted {
+                        next_wft_state = NextWftState::SeenCompleted;
+                        return true;
+                    }
+                    false
+                }
+                NextWftState::SeenCompleted => {
+                    // If we've seen the WFT be completed, and this event is another scheduled, then
+                    // this was an empty heartbeat we should ignore.
+                    if e.event_type() == EventType::WorkflowTaskScheduled {
+                        next_wft_state = NextWftState::NotSeen;
+                        return true;
+                    }
+                    // Otherwise, we're done here
+                    false
                 }
-                return false;
+                NextWftState::NotSeen => true,
             }
-
-            true
         };
 
         // Fetch events from the buffer first, then from the network
         let mut event_q = stream::iter(self.buffered.drain(..).map(Ok)).chain(&mut self.events);
 
-        let mut extra_e = None;
+        let mut extra_e = vec![];
         let mut last_seen_id = None;
         while let Some(e) = event_q.next().await {
             let e = e?;
@@ -288,7 +305,17 @@ impl HistoryUpdate {
             // command on completion), where we may need to skip events we already handled.
             if e.event_id > from_event_id {
                 if !should_pop(&e) {
-                    extra_e = Some(e);
+                    if next_wft_state == NextWftState::SeenCompleted {
+                        // We have seen the wft completed event, but decided to exit. We don't
+                        // want to return that event as part of this sequence, so include it for
+                        // re-buffering along with the event we're currently on.
+                        extra_e.push(
+                            events_to_next_wft_started
+                                .pop()
+                                .expect("There is an element here by definition"),
+                        );
+                    }
+                    extra_e.push(e);
                     break;
                 }
                 events_to_next_wft_started.push(e);
@@ -299,6 +326,13 @@ impl HistoryUpdate {
     }
 }
 
+#[derive(Eq, PartialEq, Debug)]
+enum NextWftState {
+    NotSeen,
+    Seen,
+    SeenCompleted,
+}
+
 impl From<HistoryInfo> for HistoryUpdate {
     fn from(v: HistoryInfo) -> Self {
         Self::new_from_events(v.events().to_vec(), v.previous_started_event_id())
@@ -366,6 +400,46 @@ pub mod tests {
         assert_eq!(seq_2.last().unwrap().event_id, 8);
     }
 
+    #[tokio::test]
+    async fn history_ends_abruptly() {
+        let mut timer_hist = canned_histories::single_timer("t");
+        timer_hist.add_workflow_execution_terminated();
+        let mut update = timer_hist.as_history_update();
+        let seq_2 = update.take_next_wft_sequence(3).await.unwrap();
+        assert_eq!(seq_2.len(), 5);
+        assert_eq!(seq_2.last().unwrap().event_id, 8);
+    }
+
+    #[tokio::test]
+    async fn heartbeats_skipped() {
+        let mut t = TestHistoryBuilder::default();
+        t.add_by_type(EventType::WorkflowExecutionStarted);
+        t.add_full_wf_task();
+        t.add_full_wf_task();
+        t.add_get_event_id(EventType::TimerStarted, None);
+        t.add_full_wf_task();
+        t.add_full_wf_task();
+        t.add_full_wf_task();
+        t.add_full_wf_task();
+        t.add_get_event_id(EventType::TimerStarted, None);
+        t.add_full_wf_task();
+        t.add_we_signaled("whee", vec![]);
+        t.add_full_wf_task();
+        t.add_workflow_execution_completed();
+
+        let mut update = t.as_history_update();
+        let seq = update.take_next_wft_sequence(0).await.unwrap();
+        assert_eq!(seq.len(), 6);
+        let seq = update.take_next_wft_sequence(6).await.unwrap();
+        assert_eq!(seq.len(), 13);
+        let seq = update.take_next_wft_sequence(19).await.unwrap();
+        assert_eq!(seq.len(), 4);
+        let seq = update.take_next_wft_sequence(23).await.unwrap();
+        assert_eq!(seq.len(), 4);
+        let seq = update.take_next_wft_sequence(27).await.unwrap();
+        assert_eq!(seq.len(), 2);
+    }
+
     #[tokio::test]
     async fn paginator_fetches_new_pages() {
         // Note that this test triggers the "event ids that went backwards" error, acceptably.

package/sdk-core/core/src/worker/workflow/machines/activity_state_machine.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::large_enum_variant)]
 
 use super::{
-    workflow_machines::MachineResponse, Cancellable, EventInfo, MachineKind, NewMachineWithCommand,
+    workflow_machines::MachineResponse, Cancellable, EventInfo, NewMachineWithCommand,
     OnEventWrapper, WFMachinesAdapter, WFMachinesError,
 };
 use rustfsm::{fsm, MachineError, StateMachine, TransitionResult};
@@ -288,10 +288,6 @@ impl WFMachinesAdapter for ActivityMachine {
                 | EventType::ActivityTaskCanceled
         )
     }
-
-    fn kind(&self) -> MachineKind {
-        MachineKind::Activity
-    }
 }
 
 impl TryFrom<CommandType> for ActivityMachineEvents {

package/sdk-core/core/src/worker/workflow/machines/cancel_external_state_machine.rs

@@ -1,5 +1,5 @@
 use super::{
-    workflow_machines::MachineResponse, Cancellable, EventInfo, MachineKind, NewMachineWithCommand,
+    workflow_machines::MachineResponse, Cancellable, EventInfo, NewMachineWithCommand,
     OnEventWrapper, WFMachinesAdapter, WFMachinesError,
 };
 use rustfsm::{fsm, TransitionResult};
@@ -222,10 +222,6 @@ impl WFMachinesAdapter for CancelExternalMachine {
                 | EventType::RequestCancelExternalWorkflowExecutionInitiated
         )
     }
-
-    fn kind(&self) -> MachineKind {
-        MachineKind::CancelExternalWorkflow
-    }
 }
 
 impl Cancellable for CancelExternalMachine {}

package/sdk-core/core/src/worker/workflow/machines/cancel_workflow_state_machine.rs

@@ -1,5 +1,5 @@
 use super::{
-    workflow_machines::MachineResponse, Cancellable, EventInfo, HistoryEvent, MachineKind,
+    workflow_machines::MachineResponse, Cancellable, EventInfo, HistoryEvent,
     NewMachineWithCommand, OnEventWrapper, WFMachinesAdapter, WFMachinesError,
 };
 use rustfsm::{fsm, TransitionResult};
@@ -111,10 +111,6 @@ impl WFMachinesAdapter for CancelWorkflowMachine {
     fn matches_event(&self, event: &HistoryEvent) -> bool {
         event.event_type() == EventType::WorkflowExecutionCanceled
     }
-
-    fn kind(&self) -> MachineKind {
-        MachineKind::CancelWorkflow
-    }
 }
 
 impl Cancellable for CancelWorkflowMachine {}

package/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs

@@ -1,5 +1,5 @@
 use super::{
-    workflow_machines::MachineResponse, Cancellable, EventInfo, MachineKind, NewMachineWithCommand,
+    workflow_machines::MachineResponse, Cancellable, EventInfo, NewMachineWithCommand,
     OnEventWrapper, WFMachinesAdapter, WFMachinesError,
 };
 use rustfsm::{fsm, MachineError, TransitionResult};
@@ -585,10 +585,6 @@ impl WFMachinesAdapter for ChildWorkflowMachine {
                 | EventType::ChildWorkflowExecutionCanceled
         )
     }
-
-    fn kind(&self) -> MachineKind {
-        MachineKind::ChildWorkflow
-    }
 }
 
 impl TryFrom<CommandType> for ChildWorkflowMachineEvents {

package/sdk-core/core/src/worker/workflow/machines/complete_workflow_state_machine.rs

@@ -1,5 +1,5 @@
 use super::{
-    workflow_machines::MachineResponse, Cancellable, EventInfo, MachineKind, NewMachineWithCommand,
+    workflow_machines::MachineResponse, Cancellable, EventInfo, NewMachineWithCommand,
     OnEventWrapper, WFMachinesAdapter, WFMachinesError,
 };
 use rustfsm::{fsm, TransitionResult};
@@ -131,10 +131,6 @@ impl WFMachinesAdapter for CompleteWorkflowMachine {
     fn matches_event(&self, event: &HistoryEvent) -> bool {
         event.event_type() == EventType::WorkflowExecutionCompleted
     }
-
-    fn kind(&self) -> MachineKind {
-        MachineKind::CompleteWorkflow
-    }
 }
 
 impl Cancellable for CompleteWorkflowMachine {}

package/sdk-core/core/src/worker/workflow/machines/continue_as_new_workflow_state_machine.rs

@@ -1,6 +1,6 @@
 use super::{
-    Cancellable, EventInfo, HistoryEvent, MachineKind, MachineResponse, NewMachineWithCommand,
-    OnEventWrapper, WFMachinesAdapter, WFMachinesError,
+    Cancellable, EventInfo, HistoryEvent, MachineResponse, NewMachineWithCommand, OnEventWrapper,
+    WFMachinesAdapter, WFMachinesError,
 };
 use rustfsm::{fsm, TransitionResult};
 use std::convert::TryFrom;
@@ -110,10 +110,6 @@ impl WFMachinesAdapter for ContinueAsNewWorkflowMachine {
     fn matches_event(&self, event: &HistoryEvent) -> bool {
         event.event_type() == EventType::WorkflowExecutionContinuedAsNew
     }
-
-    fn kind(&self) -> MachineKind {
-        MachineKind::ContinueAsNewWorkflow
-    }
 }
 
 impl Cancellable for ContinueAsNewWorkflowMachine {}

package/sdk-core/core/src/worker/workflow/machines/fail_workflow_state_machine.rs

@@ -1,5 +1,5 @@
 use super::{
-    workflow_machines::MachineResponse, Cancellable, EventInfo, MachineKind, NewMachineWithCommand,
+    workflow_machines::MachineResponse, Cancellable, EventInfo, NewMachineWithCommand,
     OnEventWrapper, WFMachinesAdapter, WFMachinesError,
 };
 use rustfsm::{fsm, TransitionResult};
@@ -124,10 +124,6 @@ impl WFMachinesAdapter for FailWorkflowMachine {
     fn matches_event(&self, event: &HistoryEvent) -> bool {
         event.event_type() == EventType::WorkflowExecutionFailed
     }
-
-    fn kind(&self) -> MachineKind {
-        MachineKind::FailWorkflow
-    }
 }
 
 impl Cancellable for FailWorkflowMachine {}

package/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs

@@ -1,10 +1,10 @@
 use super::{
-    workflow_machines::MachineResponse, Cancellable, EventInfo, MachineKind, OnEventWrapper,
-    WFMachinesAdapter, WFMachinesError,
+    workflow_machines::MachineResponse, Cancellable, EventInfo, OnEventWrapper, WFMachinesAdapter,
+    WFMachinesError,
 };
 use crate::{
     protosext::{CompleteLocalActivityData, HistoryEventExt, ValidScheduleLA},
-    worker::LocalActivityExecutionResult,
+    worker::{workflow::OutgoingJob, LocalActivityExecutionResult},
 };
 use rustfsm::{fsm, MachineError, StateMachine, TransitionResult};
 use std::{
@@ -79,6 +79,10 @@ fsm! {
     // even though we already resolved the activity.
     WaitingMarkerEventPreResolved --(MarkerRecorded(CompleteLocalActivityData),
                                      shared on_marker_recorded) --> MarkerCommandRecorded;
+    // Ignore cancellations when waiting for the marker after being pre-resolved
+    WaitingMarkerEventPreResolved --(Cancel) --> WaitingMarkerEventPreResolved;
+    WaitingMarkerEventPreResolved --(NoWaitCancel(ActivityCancellationType))
+                                     --> WaitingMarkerEventPreResolved;
 
     // Ignore cancellation in final state
     MarkerCommandRecorded --(Cancel, on_cancel_requested) --> MarkerCommandRecorded;
@@ -286,6 +290,7 @@ impl SharedState {
     }
 }
 
+#[allow(clippy::large_enum_variant)]
 #[derive(Debug, derive_more::Display)]
 pub(super) enum LocalActivityCommand {
     RequestActivityExecution(ValidScheduleLA),
@@ -643,13 +648,14 @@ impl WFMachinesAdapter for LocalActivityMachine {
                     result.into()
                 };
                 let mut responses = vec![
-                    MachineResponse::PushWFJob(
-                        ResolveActivity {
+                    MachineResponse::PushWFJob(OutgoingJob {
+                        variant: ResolveActivity {
                             seq: self.shared_state.attrs.seq,
                             result: Some(resolution),
                         }
                         .into(),
-                    ),
+                        is_la_resolution: true,
+                    }),
                     MachineResponse::UpdateWFTime(complete_time),
                 ];
 
@@ -708,10 +714,6 @@ impl WFMachinesAdapter for LocalActivityMachine {
     fn matches_event(&self, event: &HistoryEvent) -> bool {
         event.is_local_activity_marker()
     }
-
-    fn kind(&self) -> MachineKind {
-        MachineKind::LocalActivity
-    }
 }
 
 impl TryFrom<CommandType> for LocalActivityMachineEvents {
@@ -940,7 +942,7 @@ mod tests {
         let commands = wfm.get_server_commands().commands;
         assert_eq!(commands.len(), 0);
         let ready_to_execute_las = wfm.drain_queued_local_activities();
-        let num_queued = if !replay { 1 } else { 0 };
+        let num_queued = usize::from(!replay);
         assert_eq!(ready_to_execute_las.len(), num_queued);
 
         if !replay {
@@ -1116,7 +1118,7 @@ mod tests {
         let commands = wfm.get_server_commands().commands;
         assert_eq!(commands.len(), 0);
         let ready_to_execute_las = wfm.drain_queued_local_activities();
-        let num_queued = if !replay { 1 } else { 0 };
+        let num_queued = usize::from(!replay);
         assert_eq!(ready_to_execute_las.len(), num_queued);
 
         if !replay {
@@ -1158,7 +1160,7 @@ mod tests {
             }]
         );
         let ready_to_execute_las = wfm.drain_queued_local_activities();
-        let num_queued = if !replay { 1 } else { 0 };
+        let num_queued = usize::from(!replay);
         assert_eq!(ready_to_execute_las.len(), num_queued);
         if !replay {
             wfm.complete_local_activity(2, ActivityExecutionResult::ok(b"Resolved".into()))
@@ -1210,17 +1212,12 @@ mod tests {
         let histinfo = t.get_full_history_info().unwrap().into();
         let mut wfm = ManagedWFFunc::new_from_update(histinfo, func, vec![]);
 
-        // First activation will request to run the LA, but it will *not* be queued for execution
-        // yet as we're still replaying.
+        // First activation will request to run the LA since the heartbeat & wft failure are skipped
+        // over
         wfm.get_next_activation().await.unwrap();
         let commands = wfm.get_server_commands().commands;
         assert_eq!(commands.len(), 0);
         let ready_to_execute_las = wfm.drain_queued_local_activities();
-        assert_eq!(ready_to_execute_las.len(), 0);
-
-        // On the *next* activation, we are no longer replaying and the activity should be queued
-        wfm.get_next_activation().await.unwrap();
-        let ready_to_execute_las = wfm.drain_queued_local_activities();
         assert_eq!(ready_to_execute_las.len(), 1);
         // We can happily complete it now
         wfm.complete_local_activity(1, ActivityExecutionResult::ok(b"hi".into()))
@@ -1383,7 +1380,7 @@ mod tests {
         assert_eq!(commands.len(), 1);
         assert_eq!(commands[0].command_type, CommandType::StartTimer as i32);
         let ready_to_execute_las = wfm.drain_queued_local_activities();
-        let num_queued = if !replay { 1 } else { 0 };
+        let num_queued = usize::from(!replay);
         assert_eq!(ready_to_execute_las.len(), num_queued);
 
         // Next activation timer fires and activity cancel will be requested

package/sdk-core/core/src/worker/workflow/machines/mod.rs

@@ -8,11 +8,8 @@ mod complete_workflow_state_machine;
 mod continue_as_new_workflow_state_machine;
 mod fail_workflow_state_machine;
 mod local_activity_state_machine;
-#[allow(unused)]
-mod mutable_side_effect_state_machine;
+mod modify_workflow_properties_state_machine;
 mod patch_state_machine;
-#[allow(unused)]
-mod side_effect_state_machine;
 mod signal_external_state_machine;
 mod timer_state_machine;
 mod upsert_search_attributes_state_machine;
@@ -32,8 +29,8 @@ use complete_workflow_state_machine::CompleteWorkflowMachine;
 use continue_as_new_workflow_state_machine::ContinueAsNewWorkflowMachine;
 use fail_workflow_state_machine::FailWorkflowMachine;
 use local_activity_state_machine::LocalActivityMachine;
+use modify_workflow_properties_state_machine::ModifyWorkflowPropertiesMachine;
 use patch_state_machine::PatchMachine;
-use prost::alloc::fmt::Formatter;
 use rustfsm::{MachineError, StateMachine};
 use signal_external_state_machine::SignalExternalMachine;
 use std::{
@@ -53,23 +50,6 @@ use workflow_task_state_machine::WorkflowTaskMachine;
 #[cfg(test)]
 use transition_coverage::add_coverage;
 
-#[derive(Copy, Clone, Debug, derive_more::Display, Eq, PartialEq)]
-enum MachineKind {
-    Activity,
-    CancelWorkflow,
-    ChildWorkflow,
-    CompleteWorkflow,
-    ContinueAsNewWorkflow,
-    FailWorkflow,
-    Timer,
-    Patch,
-    WorkflowTask,
-    SignalExternalWorkflow,
-    CancelExternalWorkflow,
-    LocalActivity,
-    UpsertSearchAttributes,
-}
-
 #[enum_dispatch::enum_dispatch]
 #[allow(clippy::enum_variant_names, clippy::large_enum_variant)]
 enum Machines {
@@ -86,6 +66,7 @@ enum Machines {
     TimerMachine,
     WorkflowTaskMachine,
     UpsertSearchAttributesMachine,
+    ModifyWorkflowPropertiesMachine,
 }
 
 /// Extends [rustfsm::StateMachine] with some functionality specific to the temporal SDK.
@@ -93,7 +74,6 @@ enum Machines {
 /// Formerly known as `EntityStateMachine` in Java.
 #[enum_dispatch::enum_dispatch(Machines)]
 trait TemporalStateMachine: Send {
-    fn kind(&self) -> MachineKind;
     fn handle_command(
         &mut self,
         command_type: CommandType,
@@ -121,6 +101,9 @@ trait TemporalStateMachine: Send {
 
     /// Returns true if the state machine is in a final state
     fn is_final_state(&self) -> bool;
+
+    /// Returns a friendly name for the type of this machine
+    fn name(&self) -> &str;
 }
 
 impl<SM> TemporalStateMachine for SM
@@ -132,10 +115,6 @@ where
     <SM as StateMachine>::State: Display,
     <SM as StateMachine>::Error: Into<WFMachinesError> + 'static + Send + Sync,
 {
-    fn kind(&self) -> MachineKind {
-        self.kind()
-    }
-
     fn handle_command(
         &mut self,
         command_type: CommandType,
@@ -162,7 +141,7 @@ where
             Err(WFMachinesError::Nondeterminism(format!(
                 "Unexpected command {:?} generated by a {:?} machine",
                 command_type,
-                self.kind()
+                self.name()
             )))
         }
     }
@@ -220,6 +199,10 @@ where
     fn is_final_state(&self) -> bool {
         self.has_reached_final_state()
     }
+
+    fn name(&self) -> &str {
+        self.name()
+    }
 }
 
 fn process_machine_commands<SM>(
@@ -235,7 +218,7 @@ where
 {
     if !commands.is_empty() {
         debug!(commands=%commands.display(), state=%machine.state(),
-               machine_name=%TemporalStateMachine::kind(machine), "Machine produced commands");
+               machine_name=%TemporalStateMachine::name(machine), "Machine produced commands");
     }
     let mut machine_responses = vec![];
     for cmd in commands {
@@ -260,9 +243,6 @@ trait WFMachinesAdapter: StateMachine {
     /// ahead of time if it's worth trying to call [TemporalStateMachine::handle_event] without
     /// requiring mutable access.
     fn matches_event(&self, event: &HistoryEvent) -> bool;
-
-    /// Allows robust identification of the type of machine
-    fn kind(&self) -> MachineKind;
 }
 
 #[derive(Debug, Copy, Clone)]
@@ -334,9 +314,3 @@ struct NewMachineWithCommand {
     command: ProtoCommand,
     machine: Machines,
 }
-
-impl Debug for dyn TemporalStateMachine {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        std::fmt::Display::fmt(&self.kind(), f)
-    }
-}