@temporalio/core-bridge 1.5.2 → 1.6.0

package/sdk-core/sdk-core-protos/src/lib.rs

@@ -67,7 +67,7 @@ pub mod coresdk {
                     fmt_tt(&self.task_token),
                 )?;
                 if let Some(r) = self.result.as_ref().and_then(|r| r.status.as_ref()) {
-                    write!(f, ", {}", r)?;
+                    write!(f, ", {r}")?;
                 } else {
                     write!(f, ", missing result")?;
                 }
@@ -125,13 +125,13 @@ pub mod coresdk {
                 write!(f, "ActivityExecutionResult(")?;
                 match self {
                     aer::Status::Completed(v) => {
-                        write!(f, "{})", v)
+                        write!(f, "{v})")
                     }
                     aer::Status::Failed(v) => {
-                        write!(f, "{})", v)
+                        write!(f, "{v})")
                     }
                     aer::Status::Cancelled(v) => {
-                        write!(f, "{})", v)
+                        write!(f, "{v})")
                     }
                     aer::Status::WillCompleteAsync(_) => {
                         write!(f, "Will complete async)")
@@ -144,7 +144,7 @@ pub mod coresdk {
             fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
                 write!(f, "Success(")?;
                 if let Some(ref v) = self.result {
-                    write!(f, "{}", v)?;
+                    write!(f, "{v}")?;
                 }
                 write!(f, ")")
             }
@@ -154,7 +154,7 @@ pub mod coresdk {
             fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
                 write!(f, "Failure(")?;
                 if let Some(ref v) = self.failure {
-                    write!(f, "{}", v)?;
+                    write!(f, "{v}")?;
                 }
                 write!(f, ")")
             }
@@ -164,7 +164,7 @@ pub mod coresdk {
             fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
                 write!(f, "Cancellation(")?;
                 if let Some(ref v) = self.failure {
-                    write!(f, "{}", v)?;
+                    write!(f, "{v}")?;
                 }
                 write!(f, ")")
             }
@@ -197,11 +197,15 @@ pub mod coresdk {
                 matches!(self.status, Some(activity_resolution::Status::Failed(_)))
             }
 
-            pub fn timed_out(&self) -> bool {
-                matches!(self.status, Some(activity_resolution::Status::Failed(Failure {
-                    failure: Some(ref f)
-                })) if f.is_timeout()
-                    || f.cause.as_ref().map(|c| c.is_timeout()).unwrap_or_default())
+            pub fn timed_out(&self) -> Option<crate::temporal::api::enums::v1::TimeoutType> {
+                match self.status {
+                    Some(activity_resolution::Status::Failed(Failure {
+                        failure: Some(ref f),
+                    })) => f
+                        .is_timeout()
+                        .or_else(|| f.cause.as_ref().and_then(|c| c.is_timeout())),
+                    _ => None,
+                }
             }
 
             pub fn cancelled(&self) -> bool {
@@ -243,10 +247,6 @@ pub mod coresdk {
         }
     }
 
-    pub mod bridge {
-        tonic::include_proto!("coresdk.bridge");
-    }
-
     pub mod common {
         tonic::include_proto!("coresdk.common");
         use super::external_data::LocalActivityMarkerData;
@@ -317,7 +317,7 @@ pub mod coresdk {
     }
 
     pub mod external_data {
-        use prost_types::{Duration, Timestamp};
+        use prost_wkt_types::{Duration, Timestamp};
         use serde::{Deserialize, Deserializer, Serialize, Serializer};
         tonic::include_proto!("coresdk.external_data");
 
@@ -404,7 +404,7 @@ pub mod coresdk {
                 query::v1::WorkflowQuery,
             },
         };
-        use prost_types::Timestamp;
+        use prost_wkt_types::Timestamp;
         use std::{
             collections::HashMap,
             fmt::{Display, Formatter},
@@ -491,7 +491,7 @@ pub mod coresdk {
 
         impl Display for EvictionReason {
             fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-                write!(f, "{:?}", self)
+                write!(f, "{self:?}")
             }
         }
 
@@ -517,7 +517,7 @@ pub mod coresdk {
             fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
                 match &self.variant {
                     None => write!(f, "empty"),
-                    Some(v) => write!(f, "{}", v),
+                    Some(v) => write!(f, "{v}"),
                 }
             }
         }
@@ -673,7 +673,7 @@ pub mod coresdk {
             fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
                 match &self.variant {
                     None => write!(f, "Empty"),
-                    Some(v) => write!(f, "{}", v),
+                    Some(v) => write!(f, "{v}"),
                 }
             }
         }
@@ -1027,7 +1027,7 @@ pub mod coresdk {
             )?;
             match &self.status {
                 None => write!(f, "empty")?,
-                Some(s) => write!(f, "{}", s)?,
+                Some(s) => write!(f, "{s}")?,
             };
             write!(f, ")")
         }
@@ -1042,7 +1042,7 @@ pub mod coresdk {
                     write!(f, "Success(")?;
                     let mut written = 0;
                     for c in commands {
-                        write!(f, "{} ", c)?;
+                        write!(f, "{c} ")?;
                         written += 1;
                         if written >= 10 && written < commands.len() {
                             write!(f, "... {} more", commands.len() - written)?;
@@ -1107,8 +1107,11 @@ pub mod coresdk {
     }
 
     impl Failure {
-        pub fn is_timeout(&self) -> bool {
-            matches!(self.failure_info, Some(FailureInfo::TimeoutFailureInfo(_)))
+        pub fn is_timeout(&self) -> Option<crate::temporal::api::enums::v1::TimeoutType> {
+            match &self.failure_info {
+                Some(FailureInfo::TimeoutFailureInfo(ti)) => Some(ti.timeout_type()),
+                _ => None,
+            }
         }
 
         pub fn application_failure(message: String, non_retryable: bool) -> Self {
@@ -1124,6 +1127,26 @@ pub mod coresdk {
             }
         }
 
+        pub fn application_failure_from_error(ae: anyhow::Error, non_retryable: bool) -> Self {
+            Self {
+                failure_info: Some(FailureInfo::ApplicationFailureInfo(
+                    ApplicationFailureInfo {
+                        non_retryable,
+                        ..Default::default()
+                    },
+                )),
+                ..ae.chain()
+                    .rfold(None, |cause, e| {
+                        Some(Self {
+                            message: e.to_string(),
+                            cause: cause.map(Box::new),
+                            ..Default::default()
+                        })
+                    })
+                    .unwrap_or_default()
+            }
+        }
+
         /// Extracts an ApplicationFailureInfo from a Failure instance if it exists
         pub fn maybe_application_failure(&self) -> Option<&ApplicationFailureInfo> {
             if let Failure {
@@ -1194,7 +1217,7 @@ pub mod coresdk {
 
     impl From<anyhow::Error> for Failure {
         fn from(ae: anyhow::Error) -> Self {
-            Failure::application_failure(ae.to_string(), false)
+            Failure::application_failure_from_error(ae, false)
         }
     }
 
@@ -1550,6 +1573,7 @@ pub mod temporal {
         }
         pub mod common {
             pub mod v1 {
+                use base64::{prelude::BASE64_STANDARD, Engine};
                 use std::{
                     collections::HashMap,
                     fmt::{Display, Formatter},
@@ -1586,11 +1610,11 @@ pub mod temporal {
                             write!(
                                 f,
                                 "[{}..{}]",
-                                base64::encode(windows.next().unwrap_or_default()),
-                                base64::encode(windows.next_back().unwrap_or_default())
+                                BASE64_STANDARD.encode(windows.next().unwrap_or_default()),
+                                BASE64_STANDARD.encode(windows.next_back().unwrap_or_default())
                             )
                         } else {
-                            write!(f, "[{}]", base64::encode(&self.data))
+                            write!(f, "[{}]", BASE64_STANDARD.encode(&self.data))
                         }
                     }
                 }
@@ -1771,6 +1795,11 @@ pub mod temporal {
                 }
             }
         }
+        pub mod interaction {
+            pub mod v1 {
+                tonic::include_proto!("temporal.api.interaction.v1");
+            }
+        }
         pub mod namespace {
             pub mod v1 {
                 tonic::include_proto!("temporal.api.namespace.v1");
@@ -1816,11 +1845,6 @@ pub mod temporal {
                 tonic::include_proto!("temporal.api.testservice.v1");
             }
         }
-        pub mod update {
-            pub mod v1 {
-                tonic::include_proto!("temporal.api.update.v1");
-            }
-        }
         pub mod version {
             pub mod v1 {
                 tonic::include_proto!("temporal.api.version.v1");
@@ -1930,3 +1954,23 @@ pub mod grpc {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::temporal::api::failure::v1::Failure;
+    use anyhow::anyhow;
+
+    #[test]
+    fn anyhow_to_failure_conversion() {
+        let no_causes: Failure = anyhow!("no causes").into();
+        assert_eq!(no_causes.cause, None);
+        assert_eq!(no_causes.message, "no causes");
+        let orig = anyhow!("fail 1");
+        let mid = orig.context("fail 2");
+        let top = mid.context("fail 3");
+        let as_fail: Failure = top.into();
+        assert_eq!(as_fail.message, "fail 3");
+        assert_eq!(as_fail.cause.as_ref().unwrap().message, "fail 2");
+        assert_eq!(as_fail.cause.unwrap().cause.unwrap().message, "fail 1");
+    }
+}

package/sdk-core/sdk-core-protos/src/task_token.rs

@@ -1,8 +1,18 @@
+use base64::{prelude::BASE64_STANDARD, Engine};
 use std::fmt::{Debug, Display, Formatter};
 
 static LOCAL_ACT_TASK_TOKEN_PREFIX: &[u8] = b"local_act_";
 
-#[derive(Hash, Eq, PartialEq, Clone, derive_more::From, derive_more::Into)]
+#[derive(
+    Hash,
+    Eq,
+    PartialEq,
+    Clone,
+    derive_more::From,
+    derive_more::Into,
+    serde::Serialize,
+    serde::Deserialize,
+)]
 /// Type-safe wrapper for task token bytes
 pub struct TaskToken(pub Vec<u8>);
 
@@ -34,5 +44,5 @@ impl Debug for TaskToken {
 }
 
 pub fn fmt_tt(tt: &[u8]) -> String {
-    base64::encode(tt)
+    BASE64_STANDARD.encode(tt)
 }

package/sdk-core/test-utils/Cargo.toml

@@ -11,7 +11,8 @@ path = "src/histfetch.rs"
 [dependencies]
 anyhow = "1.0"
 async-trait = "0.1"
-base64 = "0.13"
+base64 = "0.21"
+bytes = "1.3"
 futures = "0.3"
 log = "0.4"
 once_cell = "1.16"
@@ -19,6 +20,7 @@ parking_lot = "0.12"
 prost = "0.11"
 prost-types = "0.11"
 rand = "0.8"
+rmp-serde = "1.1"
 serde_json = "1.0"
 temporal-client = { path = "../client" }
 temporal-sdk = { path = "../sdk" }

package/sdk-core/test-utils/src/histfetch.rs

@@ -23,6 +23,6 @@ async fn main() -> Result<(), anyhow::Error> {
         .expect("history field must be populated");
     // Serialize history to file
     let byteified = hist.encode_to_vec();
-    tokio::fs::write(format!("{}_history.bin", wf_id), &byteified).await?;
+    tokio::fs::write(format!("{wf_id}_history.bin"), &byteified).await?;
     Ok(())
 }

package/sdk-core/test-utils/src/lib.rs

@@ -5,8 +5,14 @@
 extern crate tracing;
 
 pub mod canned_histories;
+pub mod wf_input_saver;
+pub mod workflows;
 
-use crate::stream::{Stream, TryStreamExt};
+use crate::{
+    stream::{Stream, TryStreamExt},
+    wf_input_saver::stream_to_file,
+};
+use base64::{prelude::BASE64_STANDARD, Engine};
 use futures::{future, stream, stream::FuturesUnordered, StreamExt};
 use parking_lot::Mutex;
 use prost::Message;
@@ -45,7 +51,7 @@ use temporal_sdk_core_protos::{
     },
     temporal::api::{common::v1::Payload, history::v1::History},
 };
-use tokio::sync::OnceCell;
+use tokio::sync::{mpsc::unbounded_channel, OnceCell};
 use url::Url;
 
 pub const NAMESPACE: &str = "default";
@@ -61,6 +67,15 @@ pub const INTEG_TEST_SERVER_USED_ENV_VAR: &str = "INTEG_TEST_SERVER_ON";
 const OTEL_URL_ENV_VAR: &str = "TEMPORAL_INTEG_OTEL_URL";
 /// If set, enable direct scraping of prom metrics on the specified port
 const PROM_ENABLE_ENV_VAR: &str = "TEMPORAL_INTEG_PROM_PORT";
+#[macro_export]
+macro_rules! prost_dur {
+    ($dur_call:ident $args:tt) => {
+        std::time::Duration::$dur_call$args
+            .try_into()
+            .expect("test duration fits")
+    };
+}
+
 /// Create a worker instance which will use the provided test name to base the task queue and wf id
 /// upon. Returns the instance and the task queue name (which is also the workflow id).
 pub async fn init_core_and_create_wf(test_name: &str) -> CoreWfStarter {
@@ -82,6 +97,7 @@ pub fn init_core_replay_stream<I>(test_name: &str, histories: I) -> Arc<dyn Core
 where
     I: Stream<Item = HistoryForReplay> + Send + 'static,
 {
+    init_integ_telem();
     let worker_cfg = WorkerConfigBuilder::default()
         .namespace(NAMESPACE)
         .task_queue(test_name)
@@ -134,7 +150,8 @@ pub struct CoreWfStarter {
     /// Used for both the task queue and workflow id
     task_queue_name: String,
     pub worker_config: WorkerConfig,
-    wft_timeout: Option<Duration>,
+    /// Options to use when starting workflow(s)
+    pub workflow_options: WorkflowOptions,
     initted_worker: OnceCell<InitializedWorker>,
 }
 struct InitializedWorker {
@@ -144,14 +161,10 @@ struct InitializedWorker {
 
 impl CoreWfStarter {
     pub fn new(test_name: &str) -> Self {
-        let rand_bytes: Vec<u8> = rand::thread_rng().sample_iter(&Standard).take(6).collect();
-        let task_q_salt = base64::encode(rand_bytes);
-        let task_queue = format!("{}_{}", test_name, task_q_salt);
-        Self::new_tq_name(&task_queue)
-    }
-
-    pub fn new_tq_name(task_queue: &str) -> Self {
         init_integ_telem();
+        let rand_bytes: Vec<u8> = rand::thread_rng().sample_iter(&Standard).take(6).collect();
+        let task_q_salt = BASE64_STANDARD.encode(rand_bytes);
+        let task_queue = format!("{test_name}_{task_q_salt}");
         Self {
             task_queue_name: task_queue.to_owned(),
             worker_config: WorkerConfigBuilder::default()
@@ -161,8 +174,8 @@ impl CoreWfStarter {
                 .max_cached_workflows(1000_usize)
                 .build()
                 .unwrap(),
-            wft_timeout: None,
             initted_worker: OnceCell::new(),
+            workflow_options: Default::default(),
         }
     }
 
@@ -189,13 +202,27 @@ impl CoreWfStarter {
     }
 
     /// Start the workflow defined by the builder and return run id
-    pub async fn start_wf(&self) -> String {
-        self.start_wf_with_id(self.task_queue_name.clone(), WorkflowOptions::default())
+    pub async fn start_wf(&mut self) -> String {
+        self.start_wf_with_id(self.task_queue_name.clone()).await
+    }
+
+    pub async fn start_with_worker(
+        &self,
+        wf_name: impl Into<String>,
+        worker: &mut TestWorker,
+    ) -> String {
+        worker
+            .submit_wf(
+                self.task_queue_name.clone(),
+                wf_name.into(),
+                vec![],
+                self.workflow_options.clone(),
+            )
             .await
+            .unwrap()
     }
 
-    pub async fn start_wf_with_id(&self, workflow_id: String, mut opts: WorkflowOptions) -> String {
-        opts.task_timeout = opts.task_timeout.or(self.wft_timeout);
+    pub async fn start_wf_with_id(&self, workflow_id: String) -> String {
         self.initted_worker
             .get()
             .expect(
@@ -209,7 +236,7 @@ impl CoreWfStarter {
                 workflow_id,
                 self.task_queue_name.clone(),
                 None,
-                opts,
+                self.workflow_options.clone(),
             )
             .await
             .unwrap()
@@ -274,8 +301,13 @@ impl CoreWfStarter {
         self
     }
 
-    pub fn wft_timeout(&mut self, timeout: Duration) -> &mut Self {
-        self.wft_timeout = Some(timeout);
+    pub fn enable_wf_state_input_recording(&mut self) -> &mut Self {
+        let (ser_tx, ser_rx) = unbounded_channel();
+        let worker_cfg_clone = self.worker_config.clone();
+        tokio::spawn(async move {
+            stream_to_file(&worker_cfg_clone, ser_rx).await.unwrap();
+        });
+        self.worker_config.wf_state_inputs = Some(ser_tx);
         self
     }
 

package/sdk-core/test-utils/src/wf_input_saver.rs

@@ -0,0 +1,50 @@
+use anyhow::anyhow;
+use bytes::BytesMut;
+use futures::{stream::BoxStream, SinkExt, StreamExt};
+use prost::bytes::Bytes;
+use std::path::Path;
+use temporal_sdk_core_api::worker::WorkerConfig;
+use tokio::{fs::File, sync::mpsc::UnboundedReceiver};
+use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec};
+
+pub struct WFStateReplayData {
+    pub config: WorkerConfig,
+    pub inputs: BoxStream<'static, Result<BytesMut, std::io::Error>>,
+}
+
+pub async fn stream_to_file(
+    config: &WorkerConfig,
+    mut rcv: UnboundedReceiver<Vec<u8>>,
+) -> Result<(), anyhow::Error> {
+    let file = File::create("wf_inputs").await?;
+    let mut transport = FramedWrite::new(file, ldc());
+    // First write the worker config, since things like cache size affect how many evictions there
+    // will be, etc.
+    transport.send(rmp_serde::to_vec(config)?.into()).await?;
+    while let Some(v) = rcv.recv().await {
+        transport.send(Bytes::from(v)).await?;
+    }
+    Ok(())
+}
+
+pub async fn read_from_file(path: impl AsRef<Path>) -> Result<WFStateReplayData, anyhow::Error> {
+    let file = File::open(path).await?;
+    let mut framed_read = FramedRead::new(file, ldc());
+    // Deserialize the worker config first
+    let config = framed_read
+        .next()
+        .await
+        .ok_or_else(|| anyhow!("Replay data file is empty"))??;
+    let config = rmp_serde::from_slice(config.as_ref())?;
+
+    Ok(WFStateReplayData {
+        config,
+        inputs: framed_read.boxed(),
+    })
+}
+
+fn ldc() -> LengthDelimitedCodec {
+    LengthDelimitedCodec::builder()
+        .length_field_type::<u16>()
+        .new_codec()
+}

package/sdk-core/test-utils/src/workflows.rs

@@ -0,0 +1,29 @@
+use crate::prost_dur;
+use std::time::Duration;
+use temporal_sdk::{ActivityOptions, LocalActivityOptions, WfContext, WorkflowResult};
+use temporal_sdk_core_protos::{coresdk::AsJsonPayloadExt, temporal::api::common::v1::RetryPolicy};
+
+pub async fn la_problem_workflow(ctx: WfContext) -> WorkflowResult<()> {
+    ctx.local_activity(LocalActivityOptions {
+        activity_type: "delay".to_string(),
+        input: "hi".as_json_payload().expect("serializes fine"),
+        retry_policy: RetryPolicy {
+            initial_interval: Some(prost_dur!(from_micros(15))),
+            backoff_coefficient: 1_000.,
+            maximum_interval: Some(prost_dur!(from_millis(1500))),
+            maximum_attempts: 4,
+            non_retryable_error_types: vec![],
+        },
+        timer_backoff_threshold: Some(Duration::from_secs(1)),
+        ..Default::default()
+    })
+    .await;
+    ctx.activity(ActivityOptions {
+        activity_type: "delay".to_string(),
+        start_to_close_timeout: Some(Duration::from_secs(20)),
+        input: "hi!".as_json_payload().expect("serializes fine"),
+        ..Default::default()
+    })
+    .await;
+    Ok(().into())
+}

package/sdk-core/tests/fuzzy_workflow.rs

@@ -0,0 +1,130 @@
+use futures_util::{sink, stream::FuturesUnordered, FutureExt, StreamExt};
+use rand::{prelude::Distribution, rngs::SmallRng, Rng, SeedableRng};
+use std::{future, time::Duration};
+use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowOptions};
+use temporal_sdk::{ActContext, ActivityOptions, LocalActivityOptions, WfContext, WorkflowResult};
+use temporal_sdk_core_protos::coresdk::{AsJsonPayloadExt, FromJsonPayloadExt, IntoPayloadsExt};
+use temporal_sdk_core_test_utils::CoreWfStarter;
+use tokio_util::sync::CancellationToken;
+
+const FUZZY_SIG: &str = "fuzzy_sig";
+
+#[derive(serde::Serialize, serde::Deserialize, Copy, Clone)]
+enum FuzzyWfAction {
+    Shutdown,
+    DoAct,
+    DoLocalAct,
+}
+
+struct FuzzyWfActionSampler;
+impl Distribution<FuzzyWfAction> for FuzzyWfActionSampler {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> FuzzyWfAction {
+        let v: u8 = rng.gen_range(1..=2);
+        match v {
+            1 => FuzzyWfAction::DoAct,
+            2 => FuzzyWfAction::DoLocalAct,
+            _ => unreachable!(),
+        }
+    }
+}
+
+async fn echo(_ctx: ActContext, echo_me: String) -> Result<String, anyhow::Error> {
+    Ok(echo_me)
+}
+
+async fn fuzzy_wf_def(ctx: WfContext) -> WorkflowResult<()> {
+    let sigchan = ctx
+        .make_signal_channel(FUZZY_SIG)
+        .map(|sd| FuzzyWfAction::from_json_payload(&sd.input[0]).expect("Can deserialize signal"));
+    let done = CancellationToken::new();
+    let done_setter = done.clone();
+
+    sigchan
+        .take_until(done.cancelled())
+        .for_each_concurrent(None, |action| {
+            let fut = match action {
+                FuzzyWfAction::DoAct => ctx
+                    .activity(ActivityOptions {
+                        activity_type: "echo_activity".to_string(),
+                        start_to_close_timeout: Some(Duration::from_secs(5)),
+                        input: "hi!".as_json_payload().expect("serializes fine"),
+                        ..Default::default()
+                    })
+                    .map(|_| ())
+                    .boxed(),
+                FuzzyWfAction::DoLocalAct => ctx
+                    .local_activity(LocalActivityOptions {
+                        activity_type: "echo_activity".to_string(),
+                        start_to_close_timeout: Some(Duration::from_secs(5)),
+                        input: "hi!".as_json_payload().expect("serializes fine"),
+                        ..Default::default()
+                    })
+                    .map(|_| ())
+                    .boxed(),
+                FuzzyWfAction::Shutdown => {
+                    done_setter.cancel();
+                    future::ready(()).boxed()
+                }
+            };
+            fut
+        })
+        .await;
+
+    Ok(().into())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn fuzzy_workflow() {
+    let num_workflows = 200;
+    let wf_name = "fuzzy_wf";
+    let mut starter = CoreWfStarter::new("fuzzy_workflow");
+    starter.max_wft(25).max_cached_workflows(25).max_at(25);
+    // .enable_wf_state_input_recording();
+    let mut worker = starter.worker().await;
+    worker.register_wf(wf_name.to_owned(), fuzzy_wf_def);
+    worker.register_activity("echo_activity", echo);
+    let client = starter.get_client().await;
+
+    let mut workflow_handles = vec![];
+    for i in 0..num_workflows {
+        let wfid = format!("{wf_name}_{i}");
+        let rid = worker
+            .submit_wf(
+                wfid.clone(),
+                wf_name.to_owned(),
+                vec![],
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+        workflow_handles.push(client.get_untyped_workflow_handle(wfid, rid));
+    }
+
+    let rng = SmallRng::seed_from_u64(523189);
+    let mut actions: Vec<FuzzyWfAction> = rng.sample_iter(FuzzyWfActionSampler).take(15).collect();
+    actions.push(FuzzyWfAction::Shutdown);
+
+    let sig_sender = async {
+        for action in actions {
+            let sends: FuturesUnordered<_> = (0..num_workflows)
+                .map(|i| {
+                    client.signal_workflow_execution(
+                        format!("{wf_name}_{i}"),
+                        "".to_string(),
+                        FUZZY_SIG.to_string(),
+                        [action.as_json_payload().expect("Serializes ok")].into_payloads(),
+                        None,
+                    )
+                })
+                .collect();
+            sends
+                .map(|_| Ok(()))
+                .forward(sink::drain())
+                .await
+                .expect("Sending signals works");
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    };
+    let (r1, _) = tokio::join!(worker.run_until_done(), sig_sender);
+    r1.unwrap();
+}