@temporalio/core-bridge 1.10.2 → 1.11.0

package/sdk-core/core/src/worker/workflow/managed_run.rs

@@ -1,5 +1,6 @@
 use crate::{
     abstractions::dbg_panic,
+    internal_flags::CoreInternalFlags,
     protosext::{protocol_messages::IncomingProtocolMessage, WorkflowActivationExt},
     telemetry::metrics,
     worker::{
@@ -422,20 +423,7 @@ impl ManagedRun {
             );
             Ok(None)
         } else {
-            // First strip out query responses from other commands that actually affect machines
-            // Would be prettier with `drain_filter`
-            let mut query_responses = vec![];
-            commands = std::mem::take(&mut commands)
-                .into_iter()
-                .filter_map(|x| {
-                    if let WFCommand::QueryResponse(qr) = x {
-                        query_responses.push(qr);
-                        None
-                    } else {
-                        Some(x)
-                    }
-                })
-                .collect();
+            let (commands, query_responses) = self.preprocess_command_sequence(commands);
 
             if activation_was_only_eviction && !commands.is_empty() {
                 dbg_panic!("Reply to an eviction included commands");
@@ -479,6 +467,38 @@ impl ManagedRun {
         }
     }
 
+    /// Core has received from lang a sequence containing all commands generated
+    /// by all workflow coroutines. Return a command sequence containing all
+    /// non-terminal (i.e. non-workflow-terminating) commands, followed by the
+    /// first terminal command if there are any. Also strip out and return query
+    /// results (these don't affect machines and are handled separately
+    /// downstream)
+    ///
+    /// The reordering is done in order that all non-terminal commands generated
+    /// by workflow coroutines are given a chance for the server to honor them.
+    /// For example, in order to deliver an update result to a client as the
+    /// workflow completes.
+    ///
+    /// Behavior here has changed backwards-incompatibly, so a flag is set if
+    /// the outcome differs from what the outcome would have been previously.
+    /// See also CoreInternalFlags::MoveTerminalCommands docstring and
+    /// https://github.com/temporalio/features/issues/481.
+    fn preprocess_command_sequence(
+        &mut self,
+        commands: Vec<WFCommand>,
+    ) -> (Vec<WFCommand>, Vec<QueryResult>) {
+        if self.wfm.machines.replaying
+            && !self
+                .wfm
+                .machines
+                .try_use_flag(CoreInternalFlags::MoveTerminalCommands, false)
+        {
+            preprocess_command_sequence_old_behavior(commands)
+        } else {
+            preprocess_command_sequence(commands)
+        }
+    }
+
     /// Called after the higher-up machinery has fetched more pages of event history needed to apply
     /// the next workflow task. The history update and paginator used to perform the fetch are
     /// passed in, with the update being used to apply the task, and the paginator stored to be
@@ -1169,6 +1189,60 @@ impl ManagedRun {
     }
 }
 
+// Construct a new command sequence with query responses removed, and any
+// terminal responses removed, except for the first terminal response, which is
+// placed at the end. Return new command sequence and query commands. Note that
+// multiple coroutines may have generated a terminal command, leading to
+// multiple terminal commands in the input to this function.
+fn preprocess_command_sequence(commands: Vec<WFCommand>) -> (Vec<WFCommand>, Vec<QueryResult>) {
+    let mut query_results = vec![];
+    let mut terminals = vec![];
+
+    let mut commands: Vec<_> = commands
+        .into_iter()
+        .filter_map(|c| {
+            if let WFCommand::QueryResponse(qr) = c {
+                query_results.push(qr);
+                None
+            } else if c.is_terminal() {
+                terminals.push(c);
+                None
+            } else {
+                Some(c)
+            }
+        })
+        .collect();
+    if let Some(first_terminal) = terminals.into_iter().next() {
+        commands.push(first_terminal);
+    }
+    (commands, query_results)
+}
+
+fn preprocess_command_sequence_old_behavior(
+    commands: Vec<WFCommand>,
+) -> (Vec<WFCommand>, Vec<QueryResult>) {
+    let mut query_results = vec![];
+    let mut seen_terminal = false;
+
+    let commands: Vec<_> = commands
+        .into_iter()
+        .filter_map(|c| {
+            if let WFCommand::QueryResponse(qr) = c {
+                query_results.push(qr);
+                None
+            } else if seen_terminal {
+                None
+            } else {
+                if c.is_terminal() {
+                    seen_terminal = true;
+                }
+                Some(c)
+            }
+        })
+        .collect();
+    (commands, query_results)
+}
+
 /// Drains pending queries from the workflow task and appends them to the activation's jobs
 fn put_queries_in_act(act: &mut WorkflowActivation, wft: &mut OutstandingTask) {
     // Nothing to do if there are no pending queries
@@ -1397,3 +1471,129 @@ impl From<WFMachinesError> for RunUpdateErr {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::worker::workflow::WFCommand;
+    use std::mem::{discriminant, Discriminant};
+
+    use command_utils::*;
+
+    #[rstest::rstest]
+    #[case::empty(
+        vec![],
+        vec![])]
+    #[case::non_terminal_is_retained(
+        vec![update_response()],
+        vec![update_response()])]
+    #[case::terminal_is_retained(
+        vec![complete()],
+        vec![complete()])]
+    #[case::post_terminal_is_retained(
+        vec![complete(), update_response()],
+        vec![update_response(), complete()])]
+    #[case::second_terminal_is_discarded(
+        vec![cancel(), complete()],
+        vec![cancel()])]
+    #[case::move_terminals_to_end_and_retain_first(
+        vec![update_response(), complete(), update_response(), cancel(), update_response()],
+        vec![update_response(), update_response(), update_response(), complete()])]
+    #[test]
+    fn preprocess_command_sequence(
+        #[case] commands_in: Vec<WFCommand>,
+        #[case] expected_commands: Vec<WFCommand>,
+    ) {
+        let (commands, _) = super::preprocess_command_sequence(commands_in);
+        assert_eq!(command_types(&commands), command_types(&expected_commands));
+    }
+
+    #[rstest::rstest]
+    #[case::query_responses_extracted(
+        vec![query_response(), update_response(), query_response(), complete(), query_response()],
+        3,
+    )]
+    #[test]
+    fn preprocess_command_sequence_extracts_queries(
+        #[case] commands_in: Vec<WFCommand>,
+        #[case] expected_queries_out: usize,
+    ) {
+        let (_, query_responses_out) = super::preprocess_command_sequence(commands_in);
+        assert_eq!(query_responses_out.len(), expected_queries_out);
+    }
+
+    #[rstest::rstest]
+    #[case::empty(
+        vec![],
+        vec![])]
+    #[case::non_terminal_is_retained(
+        vec![update_response()],
+        vec![update_response()])]
+    #[case::terminal_is_retained(
+        vec![complete()],
+        vec![complete()])]
+    #[case::post_terminal_is_discarded(
+        vec![complete(), update_response()],
+        vec![complete()])]
+    #[case::second_terminal_is_discarded(
+        vec![cancel(), complete()],
+        vec![cancel()])]
+    #[case::truncate_at_first_complete(
+        vec![update_response(), complete(), update_response(), cancel()],
+        vec![update_response(), complete()])]
+    #[test]
+    fn preprocess_command_sequence_old_behavior(
+        #[case] commands_in: Vec<WFCommand>,
+        #[case] expected_out: Vec<WFCommand>,
+    ) {
+        let (commands_out, _) = super::preprocess_command_sequence_old_behavior(commands_in);
+        assert_eq!(command_types(&commands_out), command_types(&expected_out));
+    }
+
+    #[rstest::rstest]
+    #[case::query_responses_extracted(
+        vec![query_response(), update_response(), query_response(), complete(), query_response()],
+        3,
+    )]
+    #[test]
+    fn preprocess_command_sequence_old_behavior_extracts_queries(
+        #[case] commands_in: Vec<WFCommand>,
+        #[case] expected_queries_out: usize,
+    ) {
+        let (_, query_responses_out) = super::preprocess_command_sequence_old_behavior(commands_in);
+        assert_eq!(query_responses_out.len(), expected_queries_out);
+    }
+
+    mod command_utils {
+        use temporal_sdk_core_protos::coresdk::workflow_commands::{
+            CancelWorkflowExecution, CompleteWorkflowExecution, QueryResult, UpdateResponse,
+        };
+
+        use super::*;
+
+        pub(crate) fn complete() -> WFCommand {
+            WFCommand::CompleteWorkflow(CompleteWorkflowExecution { result: None })
+        }
+
+        pub(crate) fn cancel() -> WFCommand {
+            WFCommand::CancelWorkflow(CancelWorkflowExecution {})
+        }
+
+        pub(crate) fn query_response() -> WFCommand {
+            WFCommand::QueryResponse(QueryResult {
+                query_id: "".into(),
+                variant: None,
+            })
+        }
+
+        pub(crate) fn update_response() -> WFCommand {
+            WFCommand::UpdateResponse(UpdateResponse {
+                protocol_instance_id: "".into(),
+                response: None,
+            })
+        }
+
+        pub(crate) fn command_types(commands: &[WFCommand]) -> Vec<Discriminant<WFCommand>> {
+            commands.iter().map(discriminant).collect()
+        }
+    }
+}

package/sdk-core/core/src/worker/workflow/mod.rs

@@ -40,6 +40,7 @@ use crate::{
 use anyhow::anyhow;
 use futures::{stream::BoxStream, Stream, StreamExt};
 use futures_util::{future::abortable, stream};
+use itertools::Itertools;
 use prost_types::TimestampError;
 use std::{
     cell::RefCell,
@@ -223,9 +224,11 @@ impl Workflows {
                                         .expect("Activation processor channel not dropped");
                                 }
                             }
-                            Err(e) => activation_tx
-                                .send(Err(e))
-                                .expect("Activation processor channel not dropped"),
+                            Err(e) => {
+                                let _ = activation_tx.send(Err(e)).inspect_err(|e| {
+                                    error!(activation=?e.0, "Activation processor channel dropped");
+                                });
+                            }
                         }
                     }
                 });
@@ -267,33 +270,41 @@ impl Workflows {
                     break Ok(act);
                 }
                 ActivationOrAuto::Autocomplete { run_id } => {
-                    self.activation_completed(
-                        WorkflowActivationCompletion {
-                            run_id,
-                            status: Some(
-                                workflow_completion::Success::from_variants(vec![]).into(),
-                            ),
-                        },
-                        true,
-                        // We need to say a type, but the type is irrelevant, so imagine some
-                        // boxed function we'll never call.
-                        Option::<Box<dyn Fn(PostActivateHookData) + Send>>::None,
-                    )
-                    .await?;
+                    if let Err(e) = self
+                        .activation_completed(
+                            WorkflowActivationCompletion {
+                                run_id,
+                                status: Some(
+                                    workflow_completion::Success::from_variants(vec![]).into(),
+                                ),
+                            },
+                            true,
+                            // We need to say a type, but the type is irrelevant, so imagine some
+                            // boxed function we'll never call.
+                            Option::<Box<dyn Fn(PostActivateHookData) + Send>>::None,
+                        )
+                        .await
+                    {
+                        error!(error=?e, "Error while auto-completing workflow task");
+                    }
                 }
                 ActivationOrAuto::AutoFail {
                     run_id,
                     machines_err,
                 } => {
-                    self.activation_completed(
-                        WorkflowActivationCompletion {
-                            run_id,
-                            status: Some(machines_err.as_failure().into()),
-                        },
-                        true,
-                        Option::<Box<dyn Fn(PostActivateHookData) + Send>>::None,
-                    )
-                    .await?;
+                    if let Err(e) = self
+                        .activation_completed(
+                            WorkflowActivationCompletion {
+                                run_id,
+                                status: Some(machines_err.as_failure().into()),
+                            },
+                            true,
+                            Option::<Box<dyn Fn(PostActivateHookData) + Send>>::None,
+                        )
+                        .await
+                    {
+                        error!(error=?e, "Error while auto-failing workflow task");
+                    }
                 }
             }
         }
@@ -793,6 +804,18 @@ impl PreparedWFT {
         let no_new_history = self.update.wft_started_id == 0;
         no_new_history && self.legacy_query.is_some()
     }
+
+    /// Useful for showing detailed info on incoming WFTs
+    #[allow(dead_code)]
+    fn print_details(&self) -> String {
+        format!(
+            "WFT events: [{}], messages: {:?}, legacy_query: {:?}, queries: {:?}",
+            self.update.get_events().iter().format(", "),
+            &self.messages,
+            &self.legacy_query,
+            &self.query_requests
+        )
+    }
 }
 
 #[derive(Debug)]
@@ -1020,7 +1043,7 @@ fn validate_completion(
     match completion.status {
         Some(workflow_activation_completion::Status::Successful(success)) => {
             // Convert to wf commands
-            let mut commands = success
+            let commands = success
                 .commands
                 .into_iter()
                 .map(|c| c.try_into())
@@ -1047,16 +1070,6 @@ fn validate_completion(
                 });
             }
 
-            // Any non-query-response commands after a terminal command should be ignored
-            if let Some(term_cmd_pos) = commands.iter().position(|c| c.is_terminal()) {
-                // Query responses are just fine, so keep them.
-                let queries = commands
-                    .split_off(term_cmd_pos + 1)
-                    .into_iter()
-                    .filter(|c| matches!(c, WFCommand::QueryResponse(_)));
-                commands.extend(queries);
-            }
-
             Ok(ValidatedCompletion::Success {
                 run_id: completion.run_id,
                 commands,

package/sdk-core/core/src/worker/workflow/workflow_stream.rs

@@ -87,7 +87,7 @@ impl WFStream {
             buffered_polls_need_cache_slot: Default::default(),
             runs: RunCache::new(
                 basics.worker_config.clone(),
-                basics.server_capabilities.clone(),
+                basics.server_capabilities,
                 local_activity_request_sink,
                 basics.metrics.clone(),
             ),
@@ -165,7 +165,6 @@ impl WFStream {
                         None
                     }
                     WFStreamInput::PollerError(e) => {
-                        warn!("WFT poller errored, shutting down");
                         return Err(PollWfError::TonicError(e));
                     }
                 };

package/sdk-core/core-api/src/errors.rs

@@ -2,23 +2,29 @@
 
 use temporal_sdk_core_protos::coresdk::activity_result::ActivityExecutionResult;
 
+/// Errors thrown by [crate::Worker::validate]
+#[derive(thiserror::Error, Debug)]
+pub enum WorkerValidationError {
+    /// The namespace provided to the worker does not exist on the server.
+    #[error("Namespace {namespace} was not found or otherwise could not be described: {source:?}")]
+    NamespaceDescribeError {
+        source: tonic::Status,
+        namespace: String,
+    },
+}
+
 /// Errors thrown by [crate::Worker::poll_workflow_activation]
 #[derive(thiserror::Error, Debug)]
 pub enum PollWfError {
     /// [crate::Worker::shutdown] was called, and there are no more replay tasks to be handled. Lang
-    /// must call [crate::Worker::complete_workflow_activation] for any remaining tasks, and then may
-    /// exit.
+    /// must call [crate::Worker::complete_workflow_activation] for any remaining tasks, and then
+    /// may exit.
     #[error("Core is shut down and there are no more workflow replay tasks")]
     ShutDown,
     /// Unhandled error when calling the temporal server. Core will attempt to retry any non-fatal
     /// errors, so lang should consider this fatal.
     #[error("Unhandled grpc error when workflow polling: {0:?}")]
     TonicError(#[from] tonic::Status),
-    /// Unhandled error when completing a workflow during a poll -- this can happen when there is no
-    /// work for lang to perform, but the server sent us a workflow task (EX: An activity completed
-    /// even though we already cancelled it)
-    #[error("Unhandled error when auto-completing workflow task: {0:?}")]
-    AutocompleteError(#[from] CompleteWfError),
 }
 
 /// Errors thrown by [crate::Worker::poll_activity_task]

package/sdk-core/core-api/src/lib.rs

@@ -3,7 +3,10 @@ pub mod telemetry;
 pub mod worker;
 
 use crate::{
-    errors::{CompleteActivityError, CompleteWfError, PollActivityError, PollWfError},
+    errors::{
+        CompleteActivityError, CompleteWfError, PollActivityError, PollWfError,
+        WorkerValidationError,
+    },
     worker::WorkerConfig,
 };
 use temporal_sdk_core_protos::coresdk::{
@@ -16,6 +19,11 @@ use temporal_sdk_core_protos::coresdk::{
 /// and is bound to a specific task queue.
 #[async_trait::async_trait]
 pub trait Worker: Send + Sync {
+    /// Validate that the worker can properly connect to server, plus any other validation that
+    /// needs to be done asynchronously. Lang SDKs should call this function once before calling
+    /// any others.
+    async fn validate(&self) -> Result<(), WorkerValidationError>;
+
     /// Ask the worker for some work, returning a [WorkflowActivation]. It is then the language
     /// SDK's responsibility to call the appropriate workflow code with the provided inputs. Blocks
     /// indefinitely until such work is available or [Worker::shutdown] is called.

package/sdk-core/sdk/Cargo.toml

@@ -18,7 +18,7 @@ crossbeam-channel = "0.5"
 derive_more = { workspace = true }
 futures = "0.3"
 parking_lot = { version = "0.12", features = ["send_guard"] }
-prost-types = { version = "0.5", package = "prost-wkt-types" }
+prost-types = { version = "0.6", package = "prost-wkt-types" }
 serde = "1.0"
 tokio = { version = "1.26", features = ["rt", "rt-multi-thread", "parking_lot", "time", "fs"] }
 tokio-util = { version = "0.7" }

package/sdk-core/sdk/src/activity_context.rs

@@ -188,10 +188,9 @@ fn calculate_deadline(
         ) => {
             let scheduled: SystemTime = maybe_convert_timestamp(scheduled)?;
             let started: SystemTime = maybe_convert_timestamp(started)?;
-            let start_to_close_timeout: StdDuration =
-                start_to_close_timeout.clone().try_into().ok()?;
+            let start_to_close_timeout: StdDuration = (*start_to_close_timeout).try_into().ok()?;
             let schedule_to_close_timeout: StdDuration =
-                schedule_to_close_timeout.clone().try_into().ok()?;
+                (*schedule_to_close_timeout).try_into().ok()?;
 
             let start_to_close_deadline: SystemTime =
                 started.checked_add(start_to_close_timeout)?;
@@ -215,7 +214,7 @@ fn calculate_deadline(
 /// Helper function lifted from prost_types::Timestamp implementation to prevent double cloning in
 /// error construction
 fn maybe_convert_timestamp(timestamp: &Timestamp) -> Option<SystemTime> {
-    let mut timestamp = timestamp.clone();
+    let mut timestamp = *timestamp;
     timestamp.normalize();
 
     let system_time = if timestamp.seconds >= 0 {