@temporalio/core-bridge 1.11.8 → 1.12.0

package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_activation/workflow_activation.proto

@@ -37,15 +37,15 @@ import "temporal/sdk/core/nexus/nexus.proto";
 // This is because:
 // * Patches are expected to apply to the entire activation
 // * Signal and update handlers should be invoked before workflow routines are iterated. That is to
-//   say before the users' main workflow function and anything spawned by it is allowed to continue.
+//  say before the users' main workflow function and anything spawned by it is allowed to continue.
 // * Local activities resolutions go after other normal jobs because while *not* replaying, they
-//   will always take longer than anything else that produces an immediate job (which is
-//   effectively instant). When *replaying* we need to scan ahead for LA markers so that we can
-//   resolve them in the same activation that they completed in when not replaying. However, doing
-//   so would, by default, put those resolutions *before* any other immediate jobs that happened
-//   in that same activation (prime example: cancelling not-wait-for-cancel activities). So, we do
-//   this to ensure the LA resolution happens after that cancel (or whatever else it may be) as it
-//   normally would have when executing.
+//  will always take longer than anything else that produces an immediate job (which is
+//  effectively instant). When *replaying* we need to scan ahead for LA markers so that we can
+//  resolve them in the same activation that they completed in when not replaying. However, doing
+//  so would, by default, put those resolutions *before* any other immediate jobs that happened
+//  in that same activation (prime example: cancelling not-wait-for-cancel activities). So, we do
+//  this to ensure the LA resolution happens after that cancel (or whatever else it may be) as it
+//  normally would have when executing.
 // * Queries always go last (and, in fact, always come in their own activation)
 // * Evictions also always come in their own activation
 //
@@ -84,10 +84,14 @@ message WorkflowActivation {
     uint64 history_size_bytes = 7;
     // Set true if the most recent WFT started event had this suggestion
     bool continue_as_new_suggested = 8;
-    // Set to the Build ID of the worker that processed this task, which may be empty. During replay
-    // this id may not equal the id of the replaying worker. If not replaying and this worker has
-    // a defined Build ID, it will equal that ID. It will also be empty for evict-only activations.
-    string build_id_for_current_task = 9;
+    // Set to the deployment version of the worker that processed this task,
+    // which may be empty. During replay this version may not equal the version
+    // of the replaying worker. If not replaying and this worker has a defined
+    // Deployment Version, it will equal that. It will also be empty for
+    // evict-only activations. The deployment name may be empty, but not the
+    // build id, if this worker was using the deprecated Build ID-only
+    // feature(s).
+    common.WorkerDeploymentVersion deployment_version_for_current_task = 9;
 }
 
 message WorkflowActivationJob {
@@ -195,6 +199,8 @@ message InitializeWorkflow {
     //
     // See field in WorkflowExecutionStarted for more detail.
     temporal.api.common.v1.WorkflowExecution root_workflow = 24;
+    // Priority of this workflow execution
+    temporal.api.common.v1.Priority priority = 25;
 }
 
 // Notify a workflow that a timer has fired
@@ -337,10 +343,10 @@ message ResolveNexusOperationStart {
     // Sequence number as provided by lang in the corresponding ScheduleNexusOperation command
     uint32 seq = 1;
     oneof status {
-        // The operation started asynchronously. Contains an ID that can be used to perform
+        // The operation started asynchronously. Contains a token that can be used to perform
         // operations on the started operation by, ex, clients. A `ResolveNexusOperation` job will
         // follow at some point.
-        string operation_id = 2;
+        string operation_token = 2;
         // If true the operation "started" but only because it's also already resolved. A
         // `ResolveNexusOperation` job will be in the same activation.
         bool started_sync = 3;
@@ -389,4 +395,4 @@ message RemoveFromCache {
         WORKFLOW_EXECUTION_ENDING = 10;
     }
     EvictionReason reason = 2;
-}
+}

package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto

@@ -16,6 +16,7 @@ import "temporal/api/enums/v1/workflow.proto";
 import "temporal/api/failure/v1/message.proto";
 import "temporal/api/sdk/v1/user_metadata.proto";
 import "temporal/sdk/core/child_workflow/child_workflow.proto";
+import "temporal/sdk/core/nexus/nexus.proto";
 import "temporal/sdk/core/common/common.proto";
 
 message WorkflowCommand {
@@ -375,6 +376,8 @@ message ScheduleNexusOperation {
     // activities and child workflows, these are transmitted to Nexus operations that may be
     // external and are not traditional payloads.
     map<string, string> nexus_header = 7;
+    // Defines behaviour of the underlying nexus operation when operation cancellation has been requested.
+    nexus.NexusOperationCancellationType cancellation_type = 8;
 }
 
 // Request cancellation of a nexus operation started via `ScheduleNexusOperation`

package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_completion/workflow_completion.proto

@@ -5,6 +5,7 @@ option ruby_package = "Temporalio::Internal::Bridge::Api::WorkflowCompletion";
 
 import "temporal/api/failure/v1/message.proto";
 import "temporal/api/enums/v1/failed_cause.proto";
+import "temporal/api/enums/v1/workflow.proto";
 import "temporal/sdk/core/common/common.proto";
 import "temporal/sdk/core/workflow_commands/workflow_commands.proto";
 
@@ -24,6 +25,8 @@ message Success {
     repeated workflow_commands.WorkflowCommand commands = 1;
     // Any internal flags which the lang SDK used in the processing of this activation
     repeated uint32 used_internal_flags = 6;
+    // The versioning behavior this workflow is currently using
+    temporal.api.enums.v1.VersioningBehavior versioning_behavior = 7;
 }
 
 // Failure to activate or execute a workflow

package/sdk-core/sdk-core-protos/src/lib.rs

@@ -40,7 +40,9 @@ pub mod coresdk {
         ENCODING_PAYLOAD_KEY, JSON_ENCODING_VAL,
         temporal::api::{
             common::v1::{Payload, Payloads, WorkflowExecution},
-            enums::v1::{TimeoutType, WorkflowTaskFailedCause},
+            enums::v1::{
+                ApplicationErrorCategory, TimeoutType, VersioningBehavior, WorkflowTaskFailedCause,
+            },
             failure::v1::{
                 ActivityFailureInfo, ApplicationFailureInfo, Failure, TimeoutFailureInfo,
                 failure::FailureInfo,
@@ -67,23 +69,43 @@ pub mod coresdk {
         tonic::include_proto!("coresdk.activity_task");
 
         impl ActivityTask {
-            pub fn cancel_from_ids(task_token: Vec<u8>, reason: ActivityCancelReason) -> Self {
+            pub fn cancel_from_ids(
+                task_token: Vec<u8>,
+                reason: ActivityCancelReason,
+                details: ActivityCancellationDetails,
+            ) -> Self {
                 Self {
                     task_token,
                     variant: Some(activity_task::Variant::Cancel(Cancel {
                         reason: reason as i32,
+                        details: Some(details),
                     })),
                 }
             }
 
+            // Checks if both the primary reason or details have a timeout cancellation.
             pub fn is_timeout(&self) -> bool {
                 match &self.variant {
-                    Some(activity_task::Variant::Cancel(Cancel { reason })) => {
+                    Some(activity_task::Variant::Cancel(Cancel { reason, details })) => {
                         *reason == ActivityCancelReason::TimedOut as i32
+                            || details.as_ref().is_some_and(|d| d.is_timed_out)
                     }
                     _ => false,
                 }
             }
+
+            pub fn primary_reason_to_cancellation_details(
+                reason: ActivityCancelReason,
+            ) -> ActivityCancellationDetails {
+                ActivityCancellationDetails {
+                    is_not_found: reason == ActivityCancelReason::NotFound,
+                    is_cancelled: reason == ActivityCancelReason::Cancelled,
+                    is_paused: reason == ActivityCancelReason::Paused,
+                    is_timed_out: reason == ActivityCancelReason::TimedOut,
+                    is_worker_shutdown: reason == ActivityCancelReason::WorkerShutdown,
+                    is_reset: reason == ActivityCancelReason::Reset,
+                }
+            }
         }
 
         impl Display for ActivityTaskCompletion {
@@ -489,7 +511,7 @@ pub mod coresdk {
                 available_internal_flags: vec![],
                 history_size_bytes: 0,
                 continue_as_new_suggested: false,
-                build_id_for_current_task: "".to_string(),
+                deployment_version_for_current_task: None,
             }
         }
 
@@ -626,8 +648,8 @@ pub mod coresdk {
                     workflow_activation_job::Variant::ResolveRequestCancelExternalWorkflow(_) => {
                         write!(f, "ResolveRequestCancelExternalWorkflow")
                     }
-                    workflow_activation_job::Variant::DoUpdate(_) => {
-                        write!(f, "DoUpdate")
+                    workflow_activation_job::Variant::DoUpdate(u) => {
+                        write!(f, "DoUpdate({})", u.id)
                     }
                     workflow_activation_job::Variant::ResolveNexusOperationStart(_) => {
                         write!(f, "ResolveNexusOperationStart")
@@ -726,6 +748,7 @@ pub mod coresdk {
                 search_attributes: attrs.search_attributes,
                 start_time: Some(start_time),
                 root_workflow: attrs.root_workflow_execution,
+                priority: attrs.priority,
             }
         }
     }
@@ -1016,6 +1039,7 @@ pub mod coresdk {
             Self {
                 commands: v,
                 used_internal_flags: vec![],
+                versioning_behavior: VersioningBehavior::Unspecified.into(),
             }
         }
     }
@@ -1141,18 +1165,22 @@ pub mod coresdk {
 
         /// Returns true if the activation contains a complete workflow execution command
         pub fn has_complete_workflow_execution(&self) -> bool {
+            self.complete_workflow_execution_value().is_some()
+        }
+
+        /// Returns the completed execution result value, if any
+        pub fn complete_workflow_execution_value(&self) -> Option<&Payload> {
             if let Some(workflow_activation_completion::Status::Successful(s)) = &self.status {
-                return s.commands.iter().any(|wfc| {
-                    matches!(
-                        wfc,
-                        WorkflowCommand {
-                            variant: Some(workflow_command::Variant::CompleteWorkflowExecution(_)),
-                            ..
-                        },
-                    )
-                });
+                s.commands.iter().find_map(|wfc| match wfc {
+                    WorkflowCommand {
+                        variant: Some(workflow_command::Variant::CompleteWorkflowExecution(v)),
+                        ..
+                    } => v.result.as_ref(),
+                    _ => None,
+                })
+            } else {
+                None
             }
-            false
         }
 
         /// Returns true if the activation completion is a success with no commands
@@ -1265,6 +1293,7 @@ pub mod coresdk {
                         start_to_close_timeout: r.start_to_close_timeout,
                         heartbeat_timeout: r.heartbeat_timeout,
                         retry_policy: r.retry_policy,
+                        priority: r.priority,
                         is_local: false,
                     },
                 )),
@@ -1343,6 +1372,12 @@ pub mod coresdk {
                 None
             }
         }
+
+        // Checks if a failure is an ApplicationFailure with Benign category.
+        pub fn is_benign_application_failure(&self) -> bool {
+            self.maybe_application_failure()
+                .is_some_and(|app_info| app_info.category() == ApplicationErrorCategory::Benign)
+        }
     }
 
     impl Display for Failure {
@@ -2188,6 +2223,8 @@ pub mod temporal {
                                 Attributes::NexusOperationTimedOutEventAttributes(a) => Some(a.scheduled_event_id),
                                 Attributes::NexusOperationCanceledEventAttributes(a) => Some(a.scheduled_event_id),
                                 Attributes::NexusOperationCancelRequestedEventAttributes(a) => Some(a.scheduled_event_id),
+                                Attributes::NexusOperationCancelRequestCompletedEventAttributes(a) => Some(a.scheduled_event_id),
+                                Attributes::NexusOperationCancelRequestFailedEventAttributes(a) => Some(a.scheduled_event_id),
                                 _ => None
                             }
                         })
@@ -2296,6 +2333,8 @@ pub mod temporal {
                             Attributes::NexusOperationTimedOutEventAttributes(_) => { EventType::NexusOperationTimedOut }
                             Attributes::NexusOperationCancelRequestedEventAttributes(_) => { EventType::NexusOperationCancelRequested }
                             Attributes::WorkflowExecutionOptionsUpdatedEventAttributes(_) => { EventType::WorkflowExecutionOptionsUpdated }
+                            Attributes::NexusOperationCancelRequestCompletedEventAttributes(_) => { EventType::NexusOperationCancelRequestCompleted }
+                            Attributes::NexusOperationCancelRequestFailedEventAttributes(_) => { EventType::NexusOperationCancelRequestFailed }
                         }
                     }
                 }
@@ -2333,6 +2372,11 @@ pub mod temporal {
                 tonic::include_proto!("temporal.api.replication.v1");
             }
         }
+        pub mod rules {
+            pub mod v1 {
+                tonic::include_proto!("temporal.api.rules.v1");
+            }
+        }
         pub mod schedule {
             #[allow(rustdoc::invalid_html_tags)]
             pub mod v1 {

package/sdk-core/test-utils/src/lib.rs

@@ -11,23 +11,32 @@ pub mod workflows;
 pub use temporal_sdk_core::replay::HistoryForReplay;
 
 use crate::stream::{Stream, TryStreamExt};
-use anyhow::Context;
+use anyhow::{Context, Error, bail};
 use assert_matches::assert_matches;
 use futures_util::{StreamExt, future, stream, stream::FuturesUnordered};
 use parking_lot::Mutex;
 use prost::Message;
 use rand::Rng;
 use std::{
-    convert::TryFrom, env, future::Future, net::SocketAddr, path::PathBuf, sync::Arc,
-    time::Duration,
+    convert::TryFrom,
+    env,
+    future::Future,
+    net::SocketAddr,
+    path::PathBuf,
+    sync::Arc,
+    time::{Duration, Instant},
 };
 use temporal_client::{
-    Client, ClientTlsConfig, NamespacedClient, RetryClient, TlsConfig, WfClientExt,
-    WorkflowClientTrait, WorkflowExecutionInfo, WorkflowHandle, WorkflowOptions,
+    Client, ClientTlsConfig, GetWorkflowResultOpts, NamespacedClient, RetryClient, TlsConfig,
+    WfClientExt, WorkflowClientTrait, WorkflowExecutionInfo, WorkflowExecutionResult,
+    WorkflowHandle, WorkflowOptions,
 };
 use temporal_sdk::{
     IntoActivityFunc, Worker, WorkflowFunction,
-    interceptors::{FailOnNondeterminismInterceptor, WorkerInterceptor},
+    interceptors::{
+        FailOnNondeterminismInterceptor, InterceptorWithNext, ReturnWorkflowExitValueInterceptor,
+        WorkerInterceptor,
+    },
 };
 #[cfg(feature = "ephemeral-server")]
 use temporal_sdk_core::ephemeral_server::{EphemeralExe, EphemeralExeVersion};
@@ -41,9 +50,11 @@ use temporal_sdk_core_api::{
     Worker as CoreWorker,
     errors::PollError,
     telemetry::{
-        Logger, OtelCollectorOptionsBuilder, PrometheusExporterOptionsBuilder, TelemetryOptions,
-        TelemetryOptionsBuilder, metrics::CoreMeter,
+        Logger, OtelCollectorOptionsBuilder, PrometheusExporterOptions,
+        PrometheusExporterOptionsBuilder, TelemetryOptions, TelemetryOptionsBuilder,
+        metrics::CoreMeter,
     },
+    worker::WorkerVersioningStrategy,
 };
 use temporal_sdk_core_protos::{
     DEFAULT_ACTIVITY_TYPE,
@@ -61,7 +72,7 @@ use temporal_sdk_core_protos::{
         workflowservice::v1::StartWorkflowExecutionResponse,
     },
 };
-use tokio::sync::OnceCell;
+use tokio::{sync::OnceCell, task::AbortHandle};
 use url::Url;
 
 pub const NAMESPACE: &str = "default";
@@ -70,6 +81,7 @@ pub const TEST_Q: &str = "q";
 pub const INTEG_SERVER_TARGET_ENV_VAR: &str = "TEMPORAL_SERVICE_ADDRESS";
 pub const INTEG_NAMESPACE_ENV_VAR: &str = "TEMPORAL_NAMESPACE";
 pub const INTEG_USE_TLS_ENV_VAR: &str = "TEMPORAL_USE_TLS";
+pub const INTEG_API_KEY: &str = "TEMPORAL_API_KEY_PATH";
 /// This env var is set (to any value) if temporal CLI dev server is in use
 pub const INTEG_TEMPORAL_DEV_SERVER_USED_ENV_VAR: &str = "INTEG_TEMPORAL_DEV_SERVER_ON";
 /// This env var is set (to any value) if the test server is in use
@@ -107,7 +119,9 @@ pub fn integ_worker_config(tq: &str) -> WorkerConfigBuilder {
         .max_outstanding_activities(100_usize)
         .max_outstanding_local_activities(100_usize)
         .max_outstanding_workflow_tasks(100_usize)
-        .worker_build_id("test_build_id");
+        .versioning_strategy(WorkerVersioningStrategy::None {
+            build_id: "test_build_id".to_owned(),
+        });
     b
 }
 
@@ -179,6 +193,7 @@ pub struct CoreWfStarter {
     pub workflow_options: WorkflowOptions,
     initted_worker: OnceCell<InitializedWorker>,
     runtime_override: Option<Arc<CoreRuntime>>,
+    client_override: Option<Arc<RetryClient<Client>>>,
 }
 struct InitializedWorker {
     worker: Arc<dyn CoreWorker>,
@@ -188,14 +203,23 @@ struct InitializedWorker {
 impl CoreWfStarter {
     pub fn new(test_name: &str) -> Self {
         init_integ_telem();
-        Self::_new(test_name, None)
+        Self::_new(test_name, None, None)
     }
 
     pub fn new_with_runtime(test_name: &str, runtime: CoreRuntime) -> Self {
-        Self::_new(test_name, Some(runtime))
+        Self::_new(test_name, Some(runtime), None)
+    }
+
+    pub fn new_with_client(test_name: &str, client: RetryClient<Client>) -> Self {
+        init_integ_telem();
+        Self::_new(test_name, None, Some(client))
     }
 
-    fn _new(test_name: &str, runtime_override: Option<CoreRuntime>) -> Self {
+    fn _new(
+        test_name: &str,
+        runtime_override: Option<CoreRuntime>,
+        client_override: Option<RetryClient<Client>>,
+    ) -> Self {
         let task_q_salt = rand_6_chars();
         let task_queue = format!("{test_name}_{task_q_salt}");
         let mut worker_config = integ_worker_config(&task_queue);
@@ -208,6 +232,7 @@ impl CoreWfStarter {
             initted_worker: OnceCell::new(),
             workflow_options: Default::default(),
             runtime_override: runtime_override.map(Arc::new),
+            client_override: client_override.map(Arc::new),
         }
     }
 
@@ -219,6 +244,7 @@ impl CoreWfStarter {
             worker_config: self.worker_config.clone(),
             workflow_options: self.workflow_options.clone(),
             runtime_override: self.runtime_override.clone(),
+            client_override: self.client_override.clone(),
             initted_worker: Default::default(),
         }
     }
@@ -246,7 +272,7 @@ impl CoreWfStarter {
 
     /// Start the workflow defined by the builder and return run id
     pub async fn start_wf(&mut self) -> String {
-        self.start_wf_with_id(self.task_queue_name.clone()).await
+        self.start_wf_with_id(self.get_wf_id().to_owned()).await
     }
 
     /// Starts the workflow using the worker
@@ -315,6 +341,32 @@ impl CoreWfStarter {
         &self.task_queue_name
     }
 
+    /// Fetch the history of the default workflow for this starter. IE: The one that would
+    /// be started by [CoreWfStarter::start_wf].
+    pub async fn get_history(&self) -> History {
+        self.initted_worker
+            .get()
+            .expect("Starter must be initialized")
+            .client
+            .get_workflow_execution_history(self.get_wf_id().to_string(), None, vec![])
+            .await
+            .unwrap()
+            .history
+            .unwrap()
+    }
+
+    pub async fn wait_for_default_wf_finish(
+        &self,
+    ) -> Result<WorkflowExecutionResult<Vec<Payload>>, Error> {
+        self.initted_worker
+            .get()
+            .unwrap()
+            .client
+            .get_untyped_workflow_handle(self.get_wf_id().to_string(), "")
+            .get_workflow_result(GetWorkflowResultOpts { follow_runs: false })
+            .await
+    }
+
     async fn get_or_init(&mut self) -> &InitializedWorker {
         self.initted_worker
             .get_or_init(|| async {
@@ -327,15 +379,19 @@ impl CoreWfStarter {
                     .worker_config
                     .build()
                     .expect("Worker config must be valid");
-                let client = Arc::new(
-                    get_integ_server_options()
-                        .connect(
-                            cfg.namespace.clone(),
-                            rt.telemetry().get_temporal_metric_meter(),
-                        )
-                        .await
-                        .expect("Must connect"),
-                );
+                let client = if let Some(client) = self.client_override.take() {
+                    client
+                } else {
+                    Arc::new(
+                        get_integ_server_options()
+                            .connect(
+                                cfg.namespace.clone(),
+                                rt.telemetry().get_temporal_metric_meter(),
+                            )
+                            .await
+                            .expect("Must connect"),
+                    )
+                };
                 let worker = init_worker(rt, cfg, client.clone()).expect("Worker inits cleanly");
                 InitializedWorker {
                     worker: Arc::new(worker),
@@ -483,13 +539,13 @@ impl TestWorker {
             Arc::new(self.inner.shutdown_handle()),
         );
         // Automatically use results-based complete detection if we have a client
-        if self.fetch_results {
-            if let Some(c) = self.client.clone() {
-                iceptor.condition = TestWorkerShutdownCond::GetResults(
-                    std::mem::take(&mut self.started_workflows.lock()),
-                    c,
-                );
-            }
+        if self.fetch_results
+            && let Some(c) = self.client.clone()
+        {
+            iceptor.condition = TestWorkerShutdownCond::GetResults(
+                std::mem::take(&mut self.started_workflows.lock()),
+                c,
+            );
         }
         iceptor.next = next_interceptor.map(|i| Box::new(i) as Box<dyn WorkerInterceptor>);
         let get_results_waiter = iceptor.wait_all_wfs();
@@ -611,16 +667,18 @@ impl WorkerInterceptor for TestWorkerCompletionIceptor {
 
 /// Returns the client options used to connect to the server used for integration tests.
 pub fn get_integ_server_options() -> ClientOptions {
-    let temporal_server_address = match env::var(INTEG_SERVER_TARGET_ENV_VAR) {
-        Ok(addr) => addr,
-        Err(_) => "http://localhost:7233".to_owned(),
-    };
+    let temporal_server_address = env::var(INTEG_SERVER_TARGET_ENV_VAR)
+        .unwrap_or_else(|_| "http://localhost:7233".to_owned());
     let url = Url::try_from(&*temporal_server_address).unwrap();
     let mut cb = ClientOptionsBuilder::default();
     cb.identity("integ_tester".to_string())
         .target_url(url)
         .client_name("temporal-core".to_string())
         .client_version("0.1.0".to_string());
+    if let Ok(key_file) = env::var(INTEG_API_KEY) {
+        let content = std::fs::read_to_string(key_file).unwrap();
+        cb.api_key(Some(content));
+    }
     if let Some(tls) = get_integ_tls_config() {
         cb.tls_cfg(tls);
     };
@@ -827,7 +885,10 @@ where
 
 #[async_trait::async_trait(?Send)]
 pub trait WorkflowHandleExt {
-    async fn fetch_history_and_replay(&self, worker: &mut Worker) -> Result<(), anyhow::Error>;
+    async fn fetch_history_and_replay(
+        &self,
+        worker: &mut Worker,
+    ) -> Result<Option<Payload>, anyhow::Error>;
 }
 
 #[async_trait::async_trait(?Send)]
@@ -835,7 +896,10 @@ impl<R> WorkflowHandleExt for WorkflowHandle<RetryClient<Client>, R>
 where
     R: FromPayloadsExt,
 {
-    async fn fetch_history_and_replay(&self, worker: &mut Worker) -> Result<(), anyhow::Error> {
+    async fn fetch_history_and_replay(
+        &self,
+        worker: &mut Worker,
+    ) -> Result<Option<Payload>, anyhow::Error> {
         let wf_id = self.info().workflow_id.clone();
         let run_id = self.info().run_id.clone();
         let history = self
@@ -847,9 +911,13 @@ where
         let with_id = HistoryForReplay::new(history, wf_id);
         let replay_worker = init_core_replay_preloaded(worker.task_queue(), [with_id]);
         worker.with_new_core_worker(replay_worker);
-        worker.set_worker_interceptor(FailOnNondeterminismInterceptor {});
-        worker.run().await.unwrap();
-        Ok(())
+        let retval_icept = ReturnWorkflowExitValueInterceptor::default();
+        let retval_handle = retval_icept.get_result_handle();
+        let mut top_icept = InterceptorWithNext::new(Box::new(FailOnNondeterminismInterceptor {}));
+        top_icept.set_next(Box::new(retval_icept));
+        worker.set_worker_interceptor(top_icept);
+        worker.run().await?;
+        Ok(retval_handle.get().cloned())
     }
 }
 
@@ -880,3 +948,53 @@ pub fn rand_6_chars() -> String {
         .map(char::from)
         .collect()
 }
+
+pub static ANY_PORT: &str = "127.0.0.1:0";
+
+pub fn prom_metrics(
+    options_override: Option<PrometheusExporterOptions>,
+) -> (TelemetryOptions, SocketAddr, AbortOnDrop) {
+    let prom_exp_opts = options_override.unwrap_or_else(|| {
+        PrometheusExporterOptionsBuilder::default()
+            .socket_addr(ANY_PORT.parse().unwrap())
+            .build()
+            .unwrap()
+    });
+    let mut telemopts = get_integ_telem_options();
+    let prom_info = start_prometheus_metric_exporter(prom_exp_opts).unwrap();
+    telemopts.metrics = Some(prom_info.meter as Arc<dyn CoreMeter>);
+    (
+        telemopts,
+        prom_info.bound_addr,
+        AbortOnDrop {
+            ah: prom_info.abort_handle,
+        },
+    )
+}
+
+pub struct AbortOnDrop {
+    ah: AbortHandle,
+}
+
+impl Drop for AbortOnDrop {
+    fn drop(&mut self) {
+        self.ah.abort();
+    }
+}
+
+pub async fn eventually<F, Fut, T, E>(func: F, timeout: Duration) -> Result<T, anyhow::Error>
+where
+    F: Fn() -> Fut,
+    Fut: Future<Output = Result<T, E>>,
+{
+    let start = Instant::now();
+    loop {
+        if start.elapsed() > timeout {
+            bail!("Eventually hit timeout");
+        }
+        if let Ok(v) = func().await {
+            return Ok(v);
+        }
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    }
+}