@temporalio/core-bridge 1.6.0 → 1.7.0

package/sdk-core/core/src/worker/workflow/machines/workflow_task_state_machine.rs

@@ -3,7 +3,8 @@
 use super::{
     workflow_machines::MachineResponse, Cancellable, EventInfo, WFMachinesAdapter, WFMachinesError,
 };
-use rustfsm::{fsm, TransitionResult};
+use crate::worker::workflow::machines::HistEventData;
+use rustfsm::{fsm, StateMachine, TransitionResult};
 use std::{
     convert::{TryFrom, TryInto},
     time::SystemTime,
@@ -31,12 +32,12 @@ fsm! {
 
 impl WorkflowTaskMachine {
     pub(super) fn new(wf_task_started_event_id: i64) -> Self {
-        Self {
-            state: Created {}.into(),
-            shared_state: SharedState {
+        Self::from_parts(
+            Created {}.into(),
+            SharedState {
                 wf_task_started_event_id,
             },
-        }
+        )
     }
 }
 
@@ -63,8 +64,8 @@ impl WFMachinesAdapter for WorkflowTaskMachine {
                 task_started_event_id,
                 time,
             } => {
-                let (event_id, event_type, has_next_event) = if let Some(ei) = event_info {
-                    (ei.event_id, ei.event_type, ei.has_next_event)
+                let (event_id, event_type) = if let Some(ei) = event_info {
+                    (ei.event_id, ei.event_type)
                 } else {
                     return Err(WFMachinesError::Fatal(
                         "WF Task machine should never issue a task started trigger \
@@ -74,9 +75,7 @@ impl WFMachinesAdapter for WorkflowTaskMachine {
                 };
 
                 let cur_event_past_or_at_start = event_id >= task_started_event_id;
-                if event_type == EventType::WorkflowTaskStarted
-                    && (!cur_event_past_or_at_start || has_next_event)
-                {
+                if event_type == EventType::WorkflowTaskStarted && !cur_event_past_or_at_start {
                     return Ok(vec![]);
                 }
                 Ok(vec![MachineResponse::TriggerWFTaskStarted {
@@ -102,10 +101,11 @@ impl WFMachinesAdapter for WorkflowTaskMachine {
     }
 }
 
-impl TryFrom<HistoryEvent> for WorkflowTaskMachineEvents {
+impl TryFrom<HistEventData> for WorkflowTaskMachineEvents {
     type Error = WFMachinesError;
 
-    fn try_from(e: HistoryEvent) -> Result<Self, Self::Error> {
+    fn try_from(e: HistEventData) -> Result<Self, Self::Error> {
+        let e = e.event;
         Ok(match e.event_type() {
             EventType::WorkflowTaskScheduled => Self::WorkflowTaskScheduled,
             EventType::WorkflowTaskStarted => Self::WorkflowTaskStarted({
@@ -201,7 +201,7 @@ pub(super) struct WFTFailedDat {
 impl Scheduled {
     pub(super) fn on_workflow_task_started(
         self,
-        shared: SharedState,
+        shared: &mut SharedState,
         WFTStartedDat {
             current_time_millis,
             started_event_id,

package/sdk-core/core/src/worker/workflow/managed_run/managed_wf_test.rs

@@ -3,6 +3,7 @@ use crate::{
     replay::TestHistoryBuilder,
     test_help::TEST_Q,
     worker::{
+        client::mocks::DEFAULT_TEST_CAPABILITIES,
         workflow::{
             history_update::tests::TestHBExt, machines::WorkflowMachines, WFCommand,
             WorkflowFetcher,
@@ -87,13 +88,16 @@ impl ManagedWFFunc {
             completions_q: completions_sync_rx,
         };
         let state_machines = WorkflowMachines::new(
-            "test_namespace".to_string(),
-            "wfid".to_string(),
-            "wftype".to_string(),
-            "runid".to_string(),
-            hist,
+            RunBasics {
+                namespace: "test_namespace".to_string(),
+                workflow_id: "wfid".to_string(),
+                workflow_type: "wftype".to_string(),
+                run_id: "runid".to_string(),
+                history: hist,
+                metrics: MetricsContext::no_op(),
+                capabilities: &DEFAULT_TEST_CAPABILITIES,
+            },
             Box::new(driver).into(),
-            MetricsContext::no_op(),
         );
         let mgr = WorkflowManager::new_from_machines(state_machines);
         Self {

package/sdk-core/core/src/worker/workflow/managed_run.rs

@@ -13,7 +13,7 @@ use crate::{
             ActivationCompleteOutcome, ActivationCompleteResult, ActivationOrAuto,
             EvictionRequestResult, FailedActivationWFTReport, HeartbeatTimeoutMsg, HistoryUpdate,
             LocalActivityRequestSink, LocalResolution, NextPageReq, OutgoingServerCommands,
-            OutstandingActivation, OutstandingTask, PermittedWFT, RequestEvictMsg,
+            OutstandingActivation, OutstandingTask, PermittedWFT, RequestEvictMsg, RunBasics,
             ServerCommandsWithWorkflowInfo, WFCommand, WFMachinesError, WFTReportStatus,
             WorkflowBridge, WorkflowTaskInfo, WFT_HEARTBEAT_TIMEOUT_FRACTION,
         },
@@ -25,7 +25,8 @@ use futures_util::future::AbortHandle;
 use std::{
     collections::HashSet,
     ops::Add,
-    sync::{mpsc::Sender, Arc},
+    rc::Rc,
+    sync::mpsc::Sender,
     time::{Duration, Instant},
 };
 use temporal_sdk_core_protos::{
@@ -65,7 +66,7 @@ pub(super) struct ManagedRun {
     /// pushing things out and then directly back in. The downside is this is the only "impure" part
     /// of the in/out nature of workflow state management. If there's ever a sensible way to lift it
     /// up, that'd be nice.
-    local_activity_request_sink: Arc<dyn LocalActivityRequestSink>,
+    local_activity_request_sink: Rc<dyn LocalActivityRequestSink>,
     /// Set if the run is currently waiting on the execution of some local activities.
     waiting_on_la: Option<WaitingOnLAs>,
     /// Is set to true if the machines encounter an error and the only subsequent thing we should
@@ -93,24 +94,12 @@ pub(super) struct ManagedRun {
     completion_waiting_on_page_fetch: Option<RunActivationCompletion>,
 }
 impl ManagedRun {
-    #[allow(clippy::too_many_arguments)] // Ok with this here. Nothing reusable to extract.
     pub(super) fn new(
-        history_update: HistoryUpdate,
-        namespace: String,
-        workflow_id: String,
-        workflow_type: String,
-        run_id: String,
-        local_activity_request_sink: Arc<dyn LocalActivityRequestSink>,
-        metrics: MetricsContext,
+        basics: RunBasics,
+        local_activity_request_sink: Rc<dyn LocalActivityRequestSink>,
     ) -> Self {
-        let wfm = WorkflowManager::new(
-            history_update,
-            namespace,
-            workflow_id,
-            workflow_type,
-            run_id,
-            metrics.clone(),
-        );
+        let metrics = basics.metrics.clone();
+        let wfm = WorkflowManager::new(basics);
         Self {
             wfm,
             local_activity_request_sink,
@@ -283,6 +272,10 @@ impl ManagedRun {
             if let Some(ot) = &retme {
                 self.metrics.wf_task_latency(ot.start_time.elapsed());
             }
+            // Tell the LA manager that we're done with the WFT
+            self.local_activity_request_sink.sink_reqs(vec![
+                LocalActRequest::IndicateWorkflowTaskCompleted(self.wfm.machines.run_id.clone()),
+            ]);
         }
 
         retme
@@ -354,6 +347,7 @@ impl ManagedRun {
     pub(super) fn successful_completion(
         &mut self,
         mut commands: Vec<WFCommand>,
+        used_flags: Vec<u32>,
         resp_chan: Option<oneshot::Sender<ActivationCompleteResult>>,
     ) -> Result<RunUpdateAct, NextPageReq> {
         let activation_was_only_eviction = self.activation_has_only_eviction();
@@ -372,7 +366,20 @@ impl ManagedRun {
                     self.run_id()
                 );
             }
-            self.reply_to_complete(ActivationCompleteOutcome::DoNothing, resp_chan);
+            let outcome = if let Some((tt, reason)) = self.trying_to_evict.as_mut().and_then(|te| {
+                te.auto_reply_fail_tt
+                    .take()
+                    .map(|tt| (tt, te.message.clone()))
+            }) {
+                ActivationCompleteOutcome::ReportWFTFail(FailedActivationWFTReport::Report(
+                    tt,
+                    WorkflowTaskFailedCause::Unspecified,
+                    Failure::application_failure(reason, true).into(),
+                ))
+            } else {
+                ActivationCompleteOutcome::DoNothing
+            };
+            self.reply_to_complete(outcome, resp_chan);
             return Ok(None);
         };
 
@@ -422,6 +429,7 @@ impl ManagedRun {
                 activation_was_only_eviction,
                 has_pending_query,
                 query_responses,
+                used_flags,
                 resp_chan,
             };
 
@@ -516,6 +524,7 @@ impl ManagedRun {
             run_id: self.run_id().to_string(),
             message,
             reason,
+            auto_reply_fail_tt: None,
         });
         let should_report = match &evict_req_outcome {
             EvictionRequestResult::EvictionRequested(Some(attempt), _)
@@ -568,6 +577,8 @@ impl ManagedRun {
             activation_was_only_eviction: completion.activation_was_only_eviction,
         };
 
+        self.wfm.machines.add_lang_used_flags(completion.used_flags);
+
         // If this is just bookkeeping after a reply to an only-eviction activation, we can bypass
         // everything, since there is no reason to continue trying to update machines.
         if completion.activation_was_only_eviction {
@@ -688,9 +699,6 @@ impl ManagedRun {
                 // Auto-reply WFT complete
                 return true;
             }
-        } else {
-            // If a heartbeat timeout happened, we should always have been waiting on LAs
-            dbg_panic!("WFT heartbeat timeout fired but we were not waiting on any LAs");
         }
         false
     }
@@ -776,6 +784,8 @@ impl ManagedRun {
             self.trying_to_evict = Some(info);
             EvictionRequestResult::EvictionRequested(attempts, self.check_more_activations())
         } else {
+            // Always store the most recent eviction reason
+            self.trying_to_evict = Some(info);
             EvictionRequestResult::EvictionAlreadyRequested(attempts)
         }
     }
@@ -844,7 +854,9 @@ impl ManagedRun {
                             None
                         }
                     }
-                    a @ Some(ActivationOrAuto::Autocomplete { .. }) => a,
+                    a @ Some(
+                        ActivationOrAuto::Autocomplete { .. } | ActivationOrAuto::AutoFail { .. },
+                    ) => a,
                     None => {
                         if let Some(reason) = self.trying_to_evict.as_ref() {
                             // If we had nothing to do, but we're trying to evict, just do that now
@@ -906,12 +918,10 @@ impl ManagedRun {
                     )
                 } else {
                     warn!(error=?fail.source, "Error while updating workflow");
-                    self.request_eviction(RequestEvictMsg {
-                        run_id: self.run_id().to_string(),
-                        message: format!("Error while updating workflow: {:?}", fail.source),
-                        reason: fail.source.evict_reason(),
+                    Some(ActivationOrAuto::AutoFail {
+                        run_id: self.run_id().to_owned(),
+                        machines_err: fail.source,
                     })
-                    .into_run_update_resp()
                 };
                 rur
             }
@@ -930,7 +940,9 @@ impl ManagedRun {
                     }
                 }
             }
-            ActivationOrAuto::Autocomplete { .. } => OutstandingActivation::Autocomplete,
+            ActivationOrAuto::Autocomplete { .. } | ActivationOrAuto::AutoFail { .. } => {
+                OutstandingActivation::Autocomplete
+            }
         };
         if let Some(old_act) = self.activation {
             // This is a panic because we have screwed up core logic if this is violated. It must be
@@ -949,12 +961,18 @@ impl ManagedRun {
         data: CompletionDataForWFT,
         due_to_heartbeat_timeout: bool,
     ) -> FulfillableActivationComplete {
-        let outgoing_cmds = self.wfm.get_server_commands();
+        let mut outgoing_cmds = self.wfm.get_server_commands();
         if data.activation_was_only_eviction && !outgoing_cmds.commands.is_empty() {
-            dbg_panic!(
+            if self.am_broken {
+                // If we broke there could be commands in the pipe that we didn't get a chance to
+                // handle properly during replay, just wipe them all out.
+                outgoing_cmds.commands = vec![];
+            } else {
+                dbg_panic!(
                 "There should not be any outgoing commands when preparing a completion response \
                  if the activation was only an eviction. This is an SDK bug."
-            );
+                );
+            }
         }
 
         let query_responses = data.query_responses;
@@ -970,7 +988,8 @@ impl ManagedRun {
         let should_respond = !(self.wfm.machines.has_pending_jobs()
             || outgoing_cmds.replaying
             || is_query_playback
-            || data.activation_was_only_eviction);
+            || data.activation_was_only_eviction
+            || self.wfm.machines.have_seen_terminal_event);
         // If there are pending LA resolutions, and we're responding to a query here,
         // we want to make sure to force a new task, as otherwise once we tell lang about
         // the LA resolution there wouldn't be any task to reply to with the result of iterating
@@ -986,16 +1005,14 @@ impl ManagedRun {
                     force_new_wft,
                     commands: outgoing_cmds.commands,
                     query_responses,
+                    sdk_metadata: self.wfm.machines.get_metadata_for_wft_complete(),
                 },
             })
         } else {
             ActivationCompleteOutcome::DoNothing
         };
         FulfillableActivationComplete {
-            result: ActivationCompleteResult {
-                most_recently_processed_event: self.wfm.machines.last_processed_event as usize,
-                outcome,
-            },
+            result: self.build_activation_complete_result(outcome),
             resp_chan,
         }
     }
@@ -1023,11 +1040,19 @@ impl ManagedRun {
         chan: Option<oneshot::Sender<ActivationCompleteResult>>,
     ) {
         if let Some(chan) = chan {
-            chan.send(ActivationCompleteResult {
-                most_recently_processed_event: self.most_recently_processed_event_number() as usize,
-                outcome,
-            })
-            .expect("Rcv half of activation reply not dropped");
+            chan.send(self.build_activation_complete_result(outcome))
+                .expect("Rcv half of activation reply not dropped");
+        }
+    }
+
+    fn build_activation_complete_result(
+        &self,
+        outcome: ActivationCompleteOutcome,
+    ) -> ActivationCompleteResult {
+        ActivationCompleteResult {
+            outcome,
+            most_recently_processed_event: self.most_recently_processed_event_number() as usize,
+            replaying: self.wfm.machines.replaying,
         }
     }
 
@@ -1139,24 +1164,9 @@ struct WorkflowManager {
 impl WorkflowManager {
     /// Create a new workflow manager given workflow history and execution info as would be found
     /// in [PollWorkflowTaskQueueResponse]
-    fn new(
-        history: HistoryUpdate,
-        namespace: String,
-        workflow_id: String,
-        workflow_type: String,
-        run_id: String,
-        metrics: MetricsContext,
-    ) -> Self {
+    fn new(basics: RunBasics) -> Self {
         let (wfb, cmd_sink) = WorkflowBridge::new();
-        let state_machines = WorkflowMachines::new(
-            namespace,
-            workflow_id,
-            workflow_type,
-            run_id,
-            history,
-            Box::new(wfb).into(),
-            metrics,
-        );
+        let state_machines = WorkflowMachines::new(basics, Box::new(wfb).into());
         Self {
             machines: state_machines,
             command_sink: Some(cmd_sink),
@@ -1289,6 +1299,7 @@ struct RunActivationCompletion {
     activation_was_only_eviction: bool,
     has_pending_query: bool,
     query_responses: Vec<QueryResult>,
+    used_flags: Vec<u32>,
     /// Used to notify the worker when the completion is done processing and the completion can
     /// unblock. Must always be `Some` when initialized.
     resp_chan: Option<oneshot::Sender<ActivationCompleteResult>>,