temporalio 0.0.1 → 0.1.0

data/bridge/sdk-core/test-utils/src/histfetch.rs

@@ -23,6 +23,6 @@ async fn main() -> Result<(), anyhow::Error> {
         .expect("history field must be populated");
     // Serialize history to file
     let byteified = hist.encode_to_vec();
-    tokio::fs::write(format!("{}_history.bin", wf_id), &byteified).await?;
+    tokio::fs::write(format!("{wf_id}_history.bin"), &byteified).await?;
     Ok(())
 }

data/bridge/sdk-core/test-utils/src/lib.rs

@@ -5,8 +5,14 @@
 extern crate tracing;
 
 pub mod canned_histories;
+pub mod wf_input_saver;
+pub mod workflows;
 
-use crate::stream::TryStreamExt;
+use crate::{
+    stream::{Stream, TryStreamExt},
+    wf_input_saver::stream_to_file,
+};
+use base64::{prelude::BASE64_STANDARD, Engine};
 use futures::{future, stream, stream::FuturesUnordered, StreamExt};
 use parking_lot::Mutex;
 use prost::Message;
@@ -18,14 +24,24 @@ use std::{
 use temporal_client::{
     Client, RetryClient, WorkflowClientTrait, WorkflowExecutionInfo, WorkflowOptions,
 };
-use temporal_sdk::{interceptors::WorkerInterceptor, IntoActivityFunc, Worker, WorkflowFunction};
+use temporal_sdk::{
+    interceptors::{FailOnNondeterminismInterceptor, WorkerInterceptor},
+    IntoActivityFunc, Worker, WorkflowFunction,
+};
 use temporal_sdk_core::{
     ephemeral_server::{EphemeralExe, EphemeralExeVersion},
-    init_replay_worker, init_worker, telemetry_init, ClientOptions, ClientOptionsBuilder, Logger,
-    MetricsExporter, OtelCollectorOptions, TelemetryOptions, TelemetryOptionsBuilder,
-    TraceExporter, WorkerConfig, WorkerConfigBuilder,
+    init_replay_worker, init_worker,
+    replay::HistoryForReplay,
+    ClientOptions, ClientOptionsBuilder, CoreRuntime, WorkerConfig, WorkerConfigBuilder,
+};
+use temporal_sdk_core_api::errors::{PollActivityError, PollWfError};
+use temporal_sdk_core_api::{
+    telemetry::{
+        Logger, MetricsExporter, OtelCollectorOptions, TelemetryOptions, TelemetryOptionsBuilder,
+        TraceExportConfig, TraceExporter,
+    },
+    Worker as CoreWorker,
 };
-use temporal_sdk_core_api::Worker as CoreWorker;
 use temporal_sdk_core_protos::{
     coresdk::{
         workflow_commands::{
@@ -35,16 +51,17 @@ use temporal_sdk_core_protos::{
         workflow_completion::WorkflowActivationCompletion,
     },
     temporal::api::{common::v1::Payload, history::v1::History},
+    DEFAULT_ACTIVITY_TYPE,
 };
-use tokio::sync::OnceCell;
+use tokio::sync::{mpsc::unbounded_channel, OnceCell};
 use url::Url;
 
 pub const NAMESPACE: &str = "default";
 pub const TEST_Q: &str = "q";
 /// The env var used to specify where the integ tests should point
 pub const INTEG_SERVER_TARGET_ENV_VAR: &str = "TEMPORAL_SERVICE_ADDRESS";
-/// This env var is set (to any value) if temporalite is in use
-pub const INTEG_TEMPORALITE_USED_ENV_VAR: &str = "INTEG_TEMPORALITE_ON";
+/// This env var is set (to any value) if temporal CLI dev server is in use
+pub const INTEG_TEMPORAL_DEV_SERVER_USED_ENV_VAR: &str = "INTEG_TEMPORAL_DEV_SERVER_ON";
 /// This env var is set (to any value) if the test server is in use
 pub const INTEG_TEST_SERVER_USED_ENV_VAR: &str = "INTEG_TEST_SERVER_ON";
 
@@ -52,6 +69,15 @@ pub const INTEG_TEST_SERVER_USED_ENV_VAR: &str = "INTEG_TEST_SERVER_ON";
 const OTEL_URL_ENV_VAR: &str = "TEMPORAL_INTEG_OTEL_URL";
 /// If set, enable direct scraping of prom metrics on the specified port
 const PROM_ENABLE_ENV_VAR: &str = "TEMPORAL_INTEG_PROM_PORT";
+#[macro_export]
+macro_rules! prost_dur {
+    ($dur_call:ident $args:tt) => {
+        std::time::Duration::$dur_call$args
+            .try_into()
+            .expect("test duration fits")
+    };
+}
+
 /// Create a worker instance which will use the provided test name to base the task queue and wf id
 /// upon. Returns the instance and the task queue name (which is also the workflow id).
 pub async fn init_core_and_create_wf(test_name: &str) -> CoreWfStarter {
@@ -61,21 +87,44 @@ pub async fn init_core_and_create_wf(test_name: &str) -> CoreWfStarter {
     starter
 }
 
-/// Create a worker replay instance preloaded with a provided history. Returns the worker impl
-/// and the task queue name as in [init_core_and_create_wf].
-pub fn init_core_replay_preloaded(
-    test_name: &str,
-    history: &History,
-) -> (Arc<dyn CoreWorker>, String) {
-    telemetry_init(&get_integ_telem_options()).expect("Telemetry inits cleanly");
+/// Create a worker replay instance preloaded with provided histories. Returns the worker impl.
+pub fn init_core_replay_preloaded<I>(test_name: &str, histories: I) -> Arc<dyn CoreWorker>
+where
+    I: IntoIterator<Item = HistoryForReplay> + 'static,
+    <I as IntoIterator>::IntoIter: Send,
+{
+    init_core_replay_stream(test_name, stream::iter(histories))
+}
+pub fn init_core_replay_stream<I>(test_name: &str, histories: I) -> Arc<dyn CoreWorker>
+where
+    I: Stream<Item = HistoryForReplay> + Send + 'static,
+{
+    init_integ_telem();
     let worker_cfg = WorkerConfigBuilder::default()
         .namespace(NAMESPACE)
         .task_queue(test_name)
         .worker_build_id("test_bin_id")
         .build()
         .expect("Configuration options construct properly");
-    let worker = init_replay_worker(worker_cfg, history).expect("Replay worker must init properly");
-    (Arc::new(worker), test_name.to_string())
+    let worker =
+        init_replay_worker(worker_cfg, histories).expect("Replay worker must init properly");
+    Arc::new(worker)
+}
+pub fn replay_sdk_worker<I>(histories: I) -> Worker
+where
+    I: IntoIterator<Item = HistoryForReplay> + 'static,
+    <I as IntoIterator>::IntoIter: Send,
+{
+    replay_sdk_worker_stream(stream::iter(histories))
+}
+pub fn replay_sdk_worker_stream<I>(histories: I) -> Worker
+where
+    I: Stream<Item = HistoryForReplay> + Send + 'static,
+{
+    let core = init_core_replay_stream("replay_worker_test", histories);
+    let mut worker = Worker::new_from_core(core, "replay_q".to_string());
+    worker.set_worker_interceptor(Box::new(FailOnNondeterminismInterceptor {}));
+    worker
 }
 
 /// Load history from a file containing the protobuf serialization of it
@@ -87,13 +136,24 @@ pub async fn history_from_proto_binary(path_from_root: &str) -> Result<History,
     Ok(History::decode(&*bytes)?)
 }
 
+static INTEG_TESTS_RT: once_cell::sync::OnceCell<CoreRuntime> = once_cell::sync::OnceCell::new();
+pub fn init_integ_telem() {
+    INTEG_TESTS_RT.get_or_init(|| {
+        let telemetry_options = get_integ_telem_options();
+        let rt =
+            CoreRuntime::new_assume_tokio(telemetry_options).expect("Core runtime inits cleanly");
+        let _ = tracing::subscriber::set_global_default(rt.trace_subscriber());
+        rt
+    });
+}
+
 /// Implements a builder pattern to help integ tests initialize core and create workflows
 pub struct CoreWfStarter {
     /// Used for both the task queue and workflow id
     task_queue_name: String,
-    telemetry_options: TelemetryOptions,
     pub worker_config: WorkerConfig,
-    wft_timeout: Option<Duration>,
+    /// Options to use when starting workflow(s)
+    pub workflow_options: WorkflowOptions,
     initted_worker: OnceCell<InitializedWorker>,
 }
 struct InitializedWorker {
@@ -103,16 +163,12 @@ struct InitializedWorker {
 
 impl CoreWfStarter {
     pub fn new(test_name: &str) -> Self {
+        init_integ_telem();
         let rand_bytes: Vec<u8> = rand::thread_rng().sample_iter(&Standard).take(6).collect();
-        let task_q_salt = base64::encode(rand_bytes);
-        let task_queue = format!("{}_{}", test_name, task_q_salt);
-        Self::new_tq_name(&task_queue)
-    }
-
-    pub fn new_tq_name(task_queue: &str) -> Self {
+        let task_q_salt = BASE64_STANDARD.encode(rand_bytes);
+        let task_queue = format!("{test_name}_{task_q_salt}");
         Self {
             task_queue_name: task_queue.to_owned(),
-            telemetry_options: get_integ_telem_options(),
             worker_config: WorkerConfigBuilder::default()
                 .namespace(NAMESPACE)
                 .task_queue(task_queue)
@@ -120,8 +176,8 @@ impl CoreWfStarter {
                 .max_cached_workflows(1000_usize)
                 .build()
                 .unwrap(),
-            wft_timeout: None,
             initted_worker: OnceCell::new(),
+            workflow_options: Default::default(),
         }
     }
 
@@ -148,13 +204,27 @@ impl CoreWfStarter {
     }
 
     /// Start the workflow defined by the builder and return run id
-    pub async fn start_wf(&self) -> String {
-        self.start_wf_with_id(self.task_queue_name.clone(), WorkflowOptions::default())
+    pub async fn start_wf(&mut self) -> String {
+        self.start_wf_with_id(self.task_queue_name.clone()).await
+    }
+
+    pub async fn start_with_worker(
+        &self,
+        wf_name: impl Into<String>,
+        worker: &mut TestWorker,
+    ) -> String {
+        worker
+            .submit_wf(
+                self.task_queue_name.clone(),
+                wf_name.into(),
+                vec![],
+                self.workflow_options.clone(),
+            )
             .await
+            .unwrap()
     }
 
-    pub async fn start_wf_with_id(&self, workflow_id: String, mut opts: WorkflowOptions) -> String {
-        opts.task_timeout = opts.task_timeout.or(self.wft_timeout);
+    pub async fn start_wf_with_id(&self, workflow_id: String) -> String {
         self.initted_worker
             .get()
             .expect(
@@ -168,7 +238,7 @@ impl CoreWfStarter {
                 workflow_id,
                 self.task_queue_name.clone(),
                 None,
-                opts,
+                self.workflow_options.clone(),
             )
             .await
             .unwrap()
@@ -181,19 +251,21 @@ impl CoreWfStarter {
         &mut self,
         wf_id: impl Into<String>,
         run_id: impl Into<String>,
-        // TODO: Need not be passed in
         worker: &mut Worker,
     ) -> Result<(), anyhow::Error> {
+        let wf_id = wf_id.into();
         // Fetch history and replay it
         let history = self
             .get_client()
             .await
-            .get_workflow_execution_history(wf_id.into(), Some(run_id.into()), vec![])
+            .get_workflow_execution_history(wf_id.clone(), Some(run_id.into()), vec![])
             .await?
             .history
             .expect("history field must be populated");
-        let (replay_worker, _) = init_core_replay_preloaded(worker.task_queue(), &history);
+        let with_id = HistoryForReplay::new(history, wf_id);
+        let replay_worker = init_core_replay_preloaded(worker.task_queue(), [with_id]);
         worker.with_new_core_worker(replay_worker);
+        worker.set_worker_interceptor(Box::new(FailOnNondeterminismInterceptor {}));
         worker.run().await.unwrap();
         Ok(())
     }
@@ -231,22 +303,36 @@ impl CoreWfStarter {
         self
     }
 
-    pub fn wft_timeout(&mut self, timeout: Duration) -> &mut Self {
-        self.wft_timeout = Some(timeout);
+    pub fn no_remote_activities(&mut self) -> &mut Self {
+        self.worker_config.no_remote_activities = true;
+        self
+    }
+
+    pub fn enable_wf_state_input_recording(&mut self) -> &mut Self {
+        let (ser_tx, ser_rx) = unbounded_channel();
+        let worker_cfg_clone = self.worker_config.clone();
+        tokio::spawn(async move {
+            stream_to_file(&worker_cfg_clone, ser_rx).await.unwrap();
+        });
+        self.worker_config.wf_state_inputs = Some(ser_tx);
         self
     }
 
     async fn get_or_init(&mut self) -> &InitializedWorker {
         self.initted_worker
             .get_or_init(|| async {
-                telemetry_init(&self.telemetry_options).expect("Telemetry inits cleanly");
                 let client = Arc::new(
                     get_integ_server_options()
                         .connect(self.worker_config.namespace.clone(), None, None)
                         .await
                         .expect("Must connect"),
                 );
-                let worker = init_worker(self.worker_config.clone(), client.clone());
+                let worker = init_worker(
+                    INTEG_TESTS_RT.get().unwrap(),
+                    self.worker_config.clone(),
+                    client.clone(),
+                )
+                .expect("Worker inits cleanly");
                 InitializedWorker {
                     worker: Arc::new(worker),
                     client,
@@ -442,7 +528,6 @@ impl WorkerInterceptor for TestWorkerCompletionIceptor {
 
 /// Returns the client options used to connect to the server used for integration tests.
 pub fn get_integ_server_options() -> ClientOptions {
-    telemetry_init(&get_integ_telem_options()).expect("Telemetry inits cleanly");
     let temporal_server_address = match env::var(INTEG_SERVER_TARGET_ENV_VAR) {
         Ok(addr) => addr,
         Err(_) => "http://localhost:7233".to_owned(),
@@ -459,6 +544,8 @@ pub fn get_integ_server_options() -> ClientOptions {
 
 pub fn get_integ_telem_options() -> TelemetryOptions {
     let mut ob = TelemetryOptionsBuilder::default();
+    let filter_string =
+        env::var("RUST_LOG").unwrap_or_else(|_| "temporal_sdk_core=INFO".to_string());
     if let Some(url) = env::var(OTEL_URL_ENV_VAR)
         .ok()
         .map(|x| x.parse::<Url>().unwrap())
@@ -466,8 +553,12 @@ pub fn get_integ_telem_options() -> TelemetryOptions {
         let opts = OtelCollectorOptions {
             url,
             headers: Default::default(),
+            metric_periodicity: None,
         };
-        ob.tracing(TraceExporter::Otel(opts.clone()));
+        ob.tracing(TraceExportConfig {
+            filter: filter_string.clone(),
+            exporter: TraceExporter::Otel(opts.clone()),
+        });
         ob.metrics(MetricsExporter::Otel(opts));
     }
     if let Some(addr) = env::var(PROM_ENABLE_ENV_VAR)
@@ -476,15 +567,16 @@ pub fn get_integ_telem_options() -> TelemetryOptions {
     {
         ob.metrics(MetricsExporter::Prometheus(addr));
     }
-    ob.tracing_filter(env::var("RUST_LOG").unwrap_or_else(|_| "temporal_sdk_core=INFO".to_string()))
-        .logging(Logger::Console)
-        .build()
-        .unwrap()
+    ob.logging(Logger::Console {
+        filter: filter_string,
+    })
+    .build()
+    .unwrap()
 }
 
 pub fn default_cached_download() -> EphemeralExe {
     EphemeralExe::CachedDownload {
-        version: EphemeralExeVersion::Default {
+        version: EphemeralExeVersion::SDKDefault {
             sdk_name: "sdk-rust".to_string(),
             sdk_version: "0.1.0".to_string(),
         },
@@ -503,8 +595,7 @@ pub fn schedule_activity_cmd(
     ScheduleActivity {
         seq,
         activity_id: activity_id.to_string(),
-        activity_type: "test_activity".to_string(),
-        namespace: NAMESPACE.to_owned(),
+        activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
         task_queue: task_q.to_owned(),
         schedule_to_start_timeout: Some(activity_timeout.try_into().expect("duration fits")),
         start_to_close_timeout: Some(activity_timeout.try_into().expect("duration fits")),
@@ -525,7 +616,7 @@ pub fn schedule_local_activity_cmd(
     ScheduleLocalActivity {
         seq,
         activity_id: activity_id.to_string(),
-        activity_type: "test_activity".to_string(),
+        activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
         schedule_to_start_timeout: Some(activity_timeout.try_into().expect("duration fits")),
         start_to_close_timeout: Some(activity_timeout.try_into().expect("duration fits")),
         schedule_to_close_timeout: Some(activity_timeout.try_into().expect("duration fits")),
@@ -596,3 +687,23 @@ where
         .unwrap();
     }
 }
+
+/// Initiate shutdown, drain the pollers, and wait for shutdown to complete.
+pub async fn drain_pollers_and_shutdown(worker: &Arc<dyn CoreWorker>) {
+    worker.initiate_shutdown();
+    tokio::join!(
+        async {
+            assert!(matches!(
+                worker.poll_activity_task().await.unwrap_err(),
+                PollActivityError::ShutDown
+            ));
+        },
+        async {
+            assert!(matches!(
+                worker.poll_workflow_activation().await.unwrap_err(),
+                PollWfError::ShutDown,
+            ));
+        }
+    );
+    worker.shutdown().await;
+}

data/bridge/sdk-core/test-utils/src/wf_input_saver.rs

@@ -0,0 +1,50 @@
+use anyhow::anyhow;
+use bytes::BytesMut;
+use futures::{stream::BoxStream, SinkExt, StreamExt};
+use prost::bytes::Bytes;
+use std::path::Path;
+use temporal_sdk_core_api::worker::WorkerConfig;
+use tokio::{fs::File, sync::mpsc::UnboundedReceiver};
+use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec};
+
+pub struct WFStateReplayData {
+    pub config: WorkerConfig,
+    pub inputs: BoxStream<'static, Result<BytesMut, std::io::Error>>,
+}
+
+pub async fn stream_to_file(
+    config: &WorkerConfig,
+    mut rcv: UnboundedReceiver<Vec<u8>>,
+) -> Result<(), anyhow::Error> {
+    let file = File::create("wf_inputs").await?;
+    let mut transport = FramedWrite::new(file, ldc());
+    // First write the worker config, since things like cache size affect how many evictions there
+    // will be, etc.
+    transport.send(rmp_serde::to_vec(config)?.into()).await?;
+    while let Some(v) = rcv.recv().await {
+        transport.send(Bytes::from(v)).await?;
+    }
+    Ok(())
+}
+
+pub async fn read_from_file(path: impl AsRef<Path>) -> Result<WFStateReplayData, anyhow::Error> {
+    let file = File::open(path).await?;
+    let mut framed_read = FramedRead::new(file, ldc());
+    // Deserialize the worker config first
+    let config = framed_read
+        .next()
+        .await
+        .ok_or_else(|| anyhow!("Replay data file is empty"))??;
+    let config = rmp_serde::from_slice(config.as_ref())?;
+
+    Ok(WFStateReplayData {
+        config,
+        inputs: framed_read.boxed(),
+    })
+}
+
+fn ldc() -> LengthDelimitedCodec {
+    LengthDelimitedCodec::builder()
+        .length_field_type::<u16>()
+        .new_codec()
+}

data/bridge/sdk-core/test-utils/src/workflows.rs

@@ -0,0 +1,29 @@
+use crate::prost_dur;
+use std::time::Duration;
+use temporal_sdk::{ActivityOptions, LocalActivityOptions, WfContext, WorkflowResult};
+use temporal_sdk_core_protos::{coresdk::AsJsonPayloadExt, temporal::api::common::v1::RetryPolicy};
+
+pub async fn la_problem_workflow(ctx: WfContext) -> WorkflowResult<()> {
+    ctx.local_activity(LocalActivityOptions {
+        activity_type: "delay".to_string(),
+        input: "hi".as_json_payload().expect("serializes fine"),
+        retry_policy: RetryPolicy {
+            initial_interval: Some(prost_dur!(from_micros(15))),
+            backoff_coefficient: 1_000.,
+            maximum_interval: Some(prost_dur!(from_millis(1500))),
+            maximum_attempts: 4,
+            non_retryable_error_types: vec![],
+        },
+        timer_backoff_threshold: Some(Duration::from_secs(1)),
+        ..Default::default()
+    })
+    .await;
+    ctx.activity(ActivityOptions {
+        activity_type: "delay".to_string(),
+        start_to_close_timeout: Some(Duration::from_secs(20)),
+        input: "hi!".as_json_payload().expect("serializes fine"),
+        ..Default::default()
+    })
+    .await;
+    Ok(().into())
+}

data/bridge/sdk-core/tests/fuzzy_workflow.rs

@@ -0,0 +1,130 @@
+use futures_util::{sink, stream::FuturesUnordered, FutureExt, StreamExt};
+use rand::{prelude::Distribution, rngs::SmallRng, Rng, SeedableRng};
+use std::{future, time::Duration};
+use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowOptions};
+use temporal_sdk::{ActContext, ActivityOptions, LocalActivityOptions, WfContext, WorkflowResult};
+use temporal_sdk_core_protos::coresdk::{AsJsonPayloadExt, FromJsonPayloadExt, IntoPayloadsExt};
+use temporal_sdk_core_test_utils::CoreWfStarter;
+use tokio_util::sync::CancellationToken;
+
+const FUZZY_SIG: &str = "fuzzy_sig";
+
+#[derive(serde::Serialize, serde::Deserialize, Copy, Clone)]
+enum FuzzyWfAction {
+    Shutdown,
+    DoAct,
+    DoLocalAct,
+}
+
+struct FuzzyWfActionSampler;
+impl Distribution<FuzzyWfAction> for FuzzyWfActionSampler {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> FuzzyWfAction {
+        let v: u8 = rng.gen_range(1..=2);
+        match v {
+            1 => FuzzyWfAction::DoAct,
+            2 => FuzzyWfAction::DoLocalAct,
+            _ => unreachable!(),
+        }
+    }
+}
+
+async fn echo(_ctx: ActContext, echo_me: String) -> Result<String, anyhow::Error> {
+    Ok(echo_me)
+}
+
+async fn fuzzy_wf_def(ctx: WfContext) -> WorkflowResult<()> {
+    let sigchan = ctx
+        .make_signal_channel(FUZZY_SIG)
+        .map(|sd| FuzzyWfAction::from_json_payload(&sd.input[0]).expect("Can deserialize signal"));
+    let done = CancellationToken::new();
+    let done_setter = done.clone();
+
+    sigchan
+        .take_until(done.cancelled())
+        .for_each_concurrent(None, |action| {
+            let fut = match action {
+                FuzzyWfAction::DoAct => ctx
+                    .activity(ActivityOptions {
+                        activity_type: "echo_activity".to_string(),
+                        start_to_close_timeout: Some(Duration::from_secs(5)),
+                        input: "hi!".as_json_payload().expect("serializes fine"),
+                        ..Default::default()
+                    })
+                    .map(|_| ())
+                    .boxed(),
+                FuzzyWfAction::DoLocalAct => ctx
+                    .local_activity(LocalActivityOptions {
+                        activity_type: "echo_activity".to_string(),
+                        start_to_close_timeout: Some(Duration::from_secs(5)),
+                        input: "hi!".as_json_payload().expect("serializes fine"),
+                        ..Default::default()
+                    })
+                    .map(|_| ())
+                    .boxed(),
+                FuzzyWfAction::Shutdown => {
+                    done_setter.cancel();
+                    future::ready(()).boxed()
+                }
+            };
+            fut
+        })
+        .await;
+
+    Ok(().into())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn fuzzy_workflow() {
+    let num_workflows = 200;
+    let wf_name = "fuzzy_wf";
+    let mut starter = CoreWfStarter::new("fuzzy_workflow");
+    starter.max_wft(25).max_cached_workflows(25).max_at(25);
+    // .enable_wf_state_input_recording();
+    let mut worker = starter.worker().await;
+    worker.register_wf(wf_name.to_owned(), fuzzy_wf_def);
+    worker.register_activity("echo_activity", echo);
+    let client = starter.get_client().await;
+
+    let mut workflow_handles = vec![];
+    for i in 0..num_workflows {
+        let wfid = format!("{wf_name}_{i}");
+        let rid = worker
+            .submit_wf(
+                wfid.clone(),
+                wf_name.to_owned(),
+                vec![],
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+        workflow_handles.push(client.get_untyped_workflow_handle(wfid, rid));
+    }
+
+    let rng = SmallRng::seed_from_u64(523189);
+    let mut actions: Vec<FuzzyWfAction> = rng.sample_iter(FuzzyWfActionSampler).take(15).collect();
+    actions.push(FuzzyWfAction::Shutdown);
+
+    let sig_sender = async {
+        for action in actions {
+            let sends: FuturesUnordered<_> = (0..num_workflows)
+                .map(|i| {
+                    client.signal_workflow_execution(
+                        format!("{wf_name}_{i}"),
+                        "".to_string(),
+                        FUZZY_SIG.to_string(),
+                        [action.as_json_payload().expect("Serializes ok")].into_payloads(),
+                        None,
+                    )
+                })
+                .collect();
+            sends
+                .map(|_| Ok(()))
+                .forward(sink::drain())
+                .await
+                .expect("Sending signals works");
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    };
+    let (r1, _) = tokio::join!(worker.run_until_done(), sig_sender);
+    r1.unwrap();
+}