temporalio 0.0.2 → 0.1.1

data/bridge/sdk-core/test-utils/src/histfetch.rs

@@ -23,6 +23,6 @@ async fn main() -> Result<(), anyhow::Error> {
         .expect("history field must be populated");
     // Serialize history to file
     let byteified = hist.encode_to_vec();
-    tokio::fs::write(format!("{}_history.bin", wf_id), &byteified).await?;
+    tokio::fs::write(format!("{wf_id}_history.bin"), &byteified).await?;
     Ok(())
 }

data/bridge/sdk-core/test-utils/src/lib.rs

@@ -5,8 +5,14 @@
 extern crate tracing;
 
 pub mod canned_histories;
+pub mod wf_input_saver;
+pub mod workflows;
 
-use crate::stream::{Stream, TryStreamExt};
+use crate::{
+    stream::{Stream, TryStreamExt},
+    wf_input_saver::stream_to_file,
+};
+use base64::{prelude::BASE64_STANDARD, Engine};
 use futures::{future, stream, stream::FuturesUnordered, StreamExt};
 use parking_lot::Mutex;
 use prost::Message;
@@ -28,6 +34,7 @@ use temporal_sdk_core::{
     replay::HistoryForReplay,
     ClientOptions, ClientOptionsBuilder, CoreRuntime, WorkerConfig, WorkerConfigBuilder,
 };
+use temporal_sdk_core_api::errors::{PollActivityError, PollWfError};
 use temporal_sdk_core_api::{
     telemetry::{
         Logger, MetricsExporter, OtelCollectorOptions, TelemetryOptions, TelemetryOptionsBuilder,
@@ -44,16 +51,17 @@ use temporal_sdk_core_protos::{
         workflow_completion::WorkflowActivationCompletion,
     },
     temporal::api::{common::v1::Payload, history::v1::History},
+    DEFAULT_ACTIVITY_TYPE,
 };
-use tokio::sync::OnceCell;
+use tokio::sync::{mpsc::unbounded_channel, OnceCell};
 use url::Url;
 
 pub const NAMESPACE: &str = "default";
 pub const TEST_Q: &str = "q";
 /// The env var used to specify where the integ tests should point
 pub const INTEG_SERVER_TARGET_ENV_VAR: &str = "TEMPORAL_SERVICE_ADDRESS";
-/// This env var is set (to any value) if temporalite is in use
-pub const INTEG_TEMPORALITE_USED_ENV_VAR: &str = "INTEG_TEMPORALITE_ON";
+/// This env var is set (to any value) if temporal CLI dev server is in use
+pub const INTEG_TEMPORAL_DEV_SERVER_USED_ENV_VAR: &str = "INTEG_TEMPORAL_DEV_SERVER_ON";
 /// This env var is set (to any value) if the test server is in use
 pub const INTEG_TEST_SERVER_USED_ENV_VAR: &str = "INTEG_TEST_SERVER_ON";
 
@@ -61,6 +69,15 @@ pub const INTEG_TEST_SERVER_USED_ENV_VAR: &str = "INTEG_TEST_SERVER_ON";
 const OTEL_URL_ENV_VAR: &str = "TEMPORAL_INTEG_OTEL_URL";
 /// If set, enable direct scraping of prom metrics on the specified port
 const PROM_ENABLE_ENV_VAR: &str = "TEMPORAL_INTEG_PROM_PORT";
+#[macro_export]
+macro_rules! prost_dur {
+    ($dur_call:ident $args:tt) => {
+        std::time::Duration::$dur_call$args
+            .try_into()
+            .expect("test duration fits")
+    };
+}
+
 /// Create a worker instance which will use the provided test name to base the task queue and wf id
 /// upon. Returns the instance and the task queue name (which is also the workflow id).
 pub async fn init_core_and_create_wf(test_name: &str) -> CoreWfStarter {
@@ -82,6 +99,7 @@ pub fn init_core_replay_stream<I>(test_name: &str, histories: I) -> Arc<dyn Core
 where
     I: Stream<Item = HistoryForReplay> + Send + 'static,
 {
+    init_integ_telem();
     let worker_cfg = WorkerConfigBuilder::default()
         .namespace(NAMESPACE)
         .task_queue(test_name)
@@ -134,7 +152,8 @@ pub struct CoreWfStarter {
     /// Used for both the task queue and workflow id
     task_queue_name: String,
     pub worker_config: WorkerConfig,
-    wft_timeout: Option<Duration>,
+    /// Options to use when starting workflow(s)
+    pub workflow_options: WorkflowOptions,
     initted_worker: OnceCell<InitializedWorker>,
 }
 struct InitializedWorker {
@@ -144,14 +163,10 @@ struct InitializedWorker {
 
 impl CoreWfStarter {
     pub fn new(test_name: &str) -> Self {
-        let rand_bytes: Vec<u8> = rand::thread_rng().sample_iter(&Standard).take(6).collect();
-        let task_q_salt = base64::encode(rand_bytes);
-        let task_queue = format!("{}_{}", test_name, task_q_salt);
-        Self::new_tq_name(&task_queue)
-    }
-
-    pub fn new_tq_name(task_queue: &str) -> Self {
         init_integ_telem();
+        let rand_bytes: Vec<u8> = rand::thread_rng().sample_iter(&Standard).take(6).collect();
+        let task_q_salt = BASE64_STANDARD.encode(rand_bytes);
+        let task_queue = format!("{test_name}_{task_q_salt}");
         Self {
             task_queue_name: task_queue.to_owned(),
             worker_config: WorkerConfigBuilder::default()
@@ -161,8 +176,8 @@ impl CoreWfStarter {
                 .max_cached_workflows(1000_usize)
                 .build()
                 .unwrap(),
-            wft_timeout: None,
             initted_worker: OnceCell::new(),
+            workflow_options: Default::default(),
         }
     }
 
@@ -189,13 +204,27 @@ impl CoreWfStarter {
     }
 
     /// Start the workflow defined by the builder and return run id
-    pub async fn start_wf(&self) -> String {
-        self.start_wf_with_id(self.task_queue_name.clone(), WorkflowOptions::default())
+    pub async fn start_wf(&mut self) -> String {
+        self.start_wf_with_id(self.task_queue_name.clone()).await
+    }
+
+    pub async fn start_with_worker(
+        &self,
+        wf_name: impl Into<String>,
+        worker: &mut TestWorker,
+    ) -> String {
+        worker
+            .submit_wf(
+                self.task_queue_name.clone(),
+                wf_name.into(),
+                vec![],
+                self.workflow_options.clone(),
+            )
             .await
+            .unwrap()
     }
 
-    pub async fn start_wf_with_id(&self, workflow_id: String, mut opts: WorkflowOptions) -> String {
-        opts.task_timeout = opts.task_timeout.or(self.wft_timeout);
+    pub async fn start_wf_with_id(&self, workflow_id: String) -> String {
         self.initted_worker
             .get()
             .expect(
@@ -209,7 +238,7 @@ impl CoreWfStarter {
                 workflow_id,
                 self.task_queue_name.clone(),
                 None,
-                opts,
+                self.workflow_options.clone(),
             )
             .await
             .unwrap()
@@ -274,8 +303,18 @@ impl CoreWfStarter {
         self
     }
 
-    pub fn wft_timeout(&mut self, timeout: Duration) -> &mut Self {
-        self.wft_timeout = Some(timeout);
+    pub fn no_remote_activities(&mut self) -> &mut Self {
+        self.worker_config.no_remote_activities = true;
+        self
+    }
+
+    pub fn enable_wf_state_input_recording(&mut self) -> &mut Self {
+        let (ser_tx, ser_rx) = unbounded_channel();
+        let worker_cfg_clone = self.worker_config.clone();
+        tokio::spawn(async move {
+            stream_to_file(&worker_cfg_clone, ser_rx).await.unwrap();
+        });
+        self.worker_config.wf_state_inputs = Some(ser_tx);
         self
     }
 
@@ -537,7 +576,7 @@ pub fn get_integ_telem_options() -> TelemetryOptions {
 
 pub fn default_cached_download() -> EphemeralExe {
     EphemeralExe::CachedDownload {
-        version: EphemeralExeVersion::Default {
+        version: EphemeralExeVersion::SDKDefault {
             sdk_name: "sdk-rust".to_string(),
             sdk_version: "0.1.0".to_string(),
         },
@@ -556,7 +595,7 @@ pub fn schedule_activity_cmd(
     ScheduleActivity {
         seq,
         activity_id: activity_id.to_string(),
-        activity_type: "test_activity".to_string(),
+        activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
         task_queue: task_q.to_owned(),
         schedule_to_start_timeout: Some(activity_timeout.try_into().expect("duration fits")),
         start_to_close_timeout: Some(activity_timeout.try_into().expect("duration fits")),
@@ -577,7 +616,7 @@ pub fn schedule_local_activity_cmd(
     ScheduleLocalActivity {
         seq,
         activity_id: activity_id.to_string(),
-        activity_type: "test_activity".to_string(),
+        activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
         schedule_to_start_timeout: Some(activity_timeout.try_into().expect("duration fits")),
         start_to_close_timeout: Some(activity_timeout.try_into().expect("duration fits")),
         schedule_to_close_timeout: Some(activity_timeout.try_into().expect("duration fits")),
@@ -648,3 +687,23 @@ where
         .unwrap();
     }
 }
+
+/// Initiate shutdown, drain the pollers, and wait for shutdown to complete.
+pub async fn drain_pollers_and_shutdown(worker: &Arc<dyn CoreWorker>) {
+    worker.initiate_shutdown();
+    tokio::join!(
+        async {
+            assert!(matches!(
+                worker.poll_activity_task().await.unwrap_err(),
+                PollActivityError::ShutDown
+            ));
+        },
+        async {
+            assert!(matches!(
+                worker.poll_workflow_activation().await.unwrap_err(),
+                PollWfError::ShutDown,
+            ));
+        }
+    );
+    worker.shutdown().await;
+}

data/bridge/sdk-core/test-utils/src/wf_input_saver.rs

@@ -0,0 +1,50 @@
+use anyhow::anyhow;
+use bytes::BytesMut;
+use futures::{stream::BoxStream, SinkExt, StreamExt};
+use prost::bytes::Bytes;
+use std::path::Path;
+use temporal_sdk_core_api::worker::WorkerConfig;
+use tokio::{fs::File, sync::mpsc::UnboundedReceiver};
+use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec};
+
+pub struct WFStateReplayData {
+    pub config: WorkerConfig,
+    pub inputs: BoxStream<'static, Result<BytesMut, std::io::Error>>,
+}
+
+pub async fn stream_to_file(
+    config: &WorkerConfig,
+    mut rcv: UnboundedReceiver<Vec<u8>>,
+) -> Result<(), anyhow::Error> {
+    let file = File::create("wf_inputs").await?;
+    let mut transport = FramedWrite::new(file, ldc());
+    // First write the worker config, since things like cache size affect how many evictions there
+    // will be, etc.
+    transport.send(rmp_serde::to_vec(config)?.into()).await?;
+    while let Some(v) = rcv.recv().await {
+        transport.send(Bytes::from(v)).await?;
+    }
+    Ok(())
+}
+
+pub async fn read_from_file(path: impl AsRef<Path>) -> Result<WFStateReplayData, anyhow::Error> {
+    let file = File::open(path).await?;
+    let mut framed_read = FramedRead::new(file, ldc());
+    // Deserialize the worker config first
+    let config = framed_read
+        .next()
+        .await
+        .ok_or_else(|| anyhow!("Replay data file is empty"))??;
+    let config = rmp_serde::from_slice(config.as_ref())?;
+
+    Ok(WFStateReplayData {
+        config,
+        inputs: framed_read.boxed(),
+    })
+}
+
+fn ldc() -> LengthDelimitedCodec {
+    LengthDelimitedCodec::builder()
+        .length_field_type::<u16>()
+        .new_codec()
+}

data/bridge/sdk-core/test-utils/src/workflows.rs

@@ -0,0 +1,29 @@
+use crate::prost_dur;
+use std::time::Duration;
+use temporal_sdk::{ActivityOptions, LocalActivityOptions, WfContext, WorkflowResult};
+use temporal_sdk_core_protos::{coresdk::AsJsonPayloadExt, temporal::api::common::v1::RetryPolicy};
+
+pub async fn la_problem_workflow(ctx: WfContext) -> WorkflowResult<()> {
+    ctx.local_activity(LocalActivityOptions {
+        activity_type: "delay".to_string(),
+        input: "hi".as_json_payload().expect("serializes fine"),
+        retry_policy: RetryPolicy {
+            initial_interval: Some(prost_dur!(from_micros(15))),
+            backoff_coefficient: 1_000.,
+            maximum_interval: Some(prost_dur!(from_millis(1500))),
+            maximum_attempts: 4,
+            non_retryable_error_types: vec![],
+        },
+        timer_backoff_threshold: Some(Duration::from_secs(1)),
+        ..Default::default()
+    })
+    .await;
+    ctx.activity(ActivityOptions {
+        activity_type: "delay".to_string(),
+        start_to_close_timeout: Some(Duration::from_secs(20)),
+        input: "hi!".as_json_payload().expect("serializes fine"),
+        ..Default::default()
+    })
+    .await;
+    Ok(().into())
+}

data/bridge/sdk-core/tests/fuzzy_workflow.rs

@@ -0,0 +1,130 @@
+use futures_util::{sink, stream::FuturesUnordered, FutureExt, StreamExt};
+use rand::{prelude::Distribution, rngs::SmallRng, Rng, SeedableRng};
+use std::{future, time::Duration};
+use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowOptions};
+use temporal_sdk::{ActContext, ActivityOptions, LocalActivityOptions, WfContext, WorkflowResult};
+use temporal_sdk_core_protos::coresdk::{AsJsonPayloadExt, FromJsonPayloadExt, IntoPayloadsExt};
+use temporal_sdk_core_test_utils::CoreWfStarter;
+use tokio_util::sync::CancellationToken;
+
+const FUZZY_SIG: &str = "fuzzy_sig";
+
+#[derive(serde::Serialize, serde::Deserialize, Copy, Clone)]
+enum FuzzyWfAction {
+    Shutdown,
+    DoAct,
+    DoLocalAct,
+}
+
+struct FuzzyWfActionSampler;
+impl Distribution<FuzzyWfAction> for FuzzyWfActionSampler {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> FuzzyWfAction {
+        let v: u8 = rng.gen_range(1..=2);
+        match v {
+            1 => FuzzyWfAction::DoAct,
+            2 => FuzzyWfAction::DoLocalAct,
+            _ => unreachable!(),
+        }
+    }
+}
+
+async fn echo(_ctx: ActContext, echo_me: String) -> Result<String, anyhow::Error> {
+    Ok(echo_me)
+}
+
+async fn fuzzy_wf_def(ctx: WfContext) -> WorkflowResult<()> {
+    let sigchan = ctx
+        .make_signal_channel(FUZZY_SIG)
+        .map(|sd| FuzzyWfAction::from_json_payload(&sd.input[0]).expect("Can deserialize signal"));
+    let done = CancellationToken::new();
+    let done_setter = done.clone();
+
+    sigchan
+        .take_until(done.cancelled())
+        .for_each_concurrent(None, |action| {
+            let fut = match action {
+                FuzzyWfAction::DoAct => ctx
+                    .activity(ActivityOptions {
+                        activity_type: "echo_activity".to_string(),
+                        start_to_close_timeout: Some(Duration::from_secs(5)),
+                        input: "hi!".as_json_payload().expect("serializes fine"),
+                        ..Default::default()
+                    })
+                    .map(|_| ())
+                    .boxed(),
+                FuzzyWfAction::DoLocalAct => ctx
+                    .local_activity(LocalActivityOptions {
+                        activity_type: "echo_activity".to_string(),
+                        start_to_close_timeout: Some(Duration::from_secs(5)),
+                        input: "hi!".as_json_payload().expect("serializes fine"),
+                        ..Default::default()
+                    })
+                    .map(|_| ())
+                    .boxed(),
+                FuzzyWfAction::Shutdown => {
+                    done_setter.cancel();
+                    future::ready(()).boxed()
+                }
+            };
+            fut
+        })
+        .await;
+
+    Ok(().into())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn fuzzy_workflow() {
+    let num_workflows = 200;
+    let wf_name = "fuzzy_wf";
+    let mut starter = CoreWfStarter::new("fuzzy_workflow");
+    starter.max_wft(25).max_cached_workflows(25).max_at(25);
+    // .enable_wf_state_input_recording();
+    let mut worker = starter.worker().await;
+    worker.register_wf(wf_name.to_owned(), fuzzy_wf_def);
+    worker.register_activity("echo_activity", echo);
+    let client = starter.get_client().await;
+
+    let mut workflow_handles = vec![];
+    for i in 0..num_workflows {
+        let wfid = format!("{wf_name}_{i}");
+        let rid = worker
+            .submit_wf(
+                wfid.clone(),
+                wf_name.to_owned(),
+                vec![],
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+        workflow_handles.push(client.get_untyped_workflow_handle(wfid, rid));
+    }
+
+    let rng = SmallRng::seed_from_u64(523189);
+    let mut actions: Vec<FuzzyWfAction> = rng.sample_iter(FuzzyWfActionSampler).take(15).collect();
+    actions.push(FuzzyWfAction::Shutdown);
+
+    let sig_sender = async {
+        for action in actions {
+            let sends: FuturesUnordered<_> = (0..num_workflows)
+                .map(|i| {
+                    client.signal_workflow_execution(
+                        format!("{wf_name}_{i}"),
+                        "".to_string(),
+                        FUZZY_SIG.to_string(),
+                        [action.as_json_payload().expect("Serializes ok")].into_payloads(),
+                        None,
+                    )
+                })
+                .collect();
+            sends
+                .map(|_| Ok(()))
+                .forward(sink::drain())
+                .await
+                .expect("Sending signals works");
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    };
+    let (r1, _) = tokio::join!(worker.run_until_done(), sig_sender);
+    r1.unwrap();
+}

data/bridge/sdk-core/tests/{load_tests.rs → heavy_tests.rs}

@@ -1,17 +1,17 @@
-use assert_matches::assert_matches;
 use futures::{future::join_all, sink, stream::FuturesUnordered, StreamExt};
 use std::time::{Duration, Instant};
 use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowOptions};
-use temporal_sdk::{ActContext, ActivityOptions, WfContext};
+use temporal_sdk::{ActContext, ActivityOptions, WfContext, WorkflowResult};
 use temporal_sdk_core_protos::coresdk::{
-    activity_result::ActivityExecutionResult, activity_task::activity_task as act_task,
-    workflow_commands::ActivityCancellationType, ActivityTaskCompletion, AsJsonPayloadExt,
+    workflow_commands::ActivityCancellationType, AsJsonPayloadExt,
 };
-use temporal_sdk_core_test_utils::CoreWfStarter;
+use temporal_sdk_core_test_utils::{workflows::la_problem_workflow, CoreWfStarter};
+
+mod fuzzy_workflow;
 
 #[tokio::test]
 async fn activity_load() {
-    const CONCURRENCY: usize = 1000;
+    const CONCURRENCY: usize = 512;
 
     let mut starter = CoreWfStarter::new("activity_load");
     starter
@@ -23,18 +23,16 @@ async fn activity_load() {
 
     let activity_id = "act-1";
     let activity_timeout = Duration::from_secs(8);
-    let payload_dat = b"hello".to_vec();
     let task_queue = starter.get_task_queue().to_owned();
 
-    let pd = payload_dat.clone();
     let wf_fn = move |ctx: WfContext| {
         let task_queue = task_queue.clone();
-        let payload_dat = pd.clone();
-
+        let payload = "yo".as_json_payload().unwrap();
         async move {
             let activity = ActivityOptions {
                 activity_id: Some(activity_id.to_string()),
                 activity_type: "test_activity".to_string(),
+                input: payload.clone(),
                 task_queue,
                 schedule_to_start_timeout: Some(activity_timeout),
                 start_to_close_timeout: Some(activity_timeout),
@@ -44,7 +42,7 @@ async fn activity_load() {
                 ..Default::default()
             };
             let res = ctx.activity(activity).await.unwrap_ok_payload();
-            assert_eq!(res.data, payload_dat);
+            assert_eq!(res.data, payload.data);
             Ok(().into())
         }
     };
@@ -52,9 +50,13 @@ async fn activity_load() {
     let starting = Instant::now();
     let wf_type = "activity_load";
     worker.register_wf(wf_type.to_owned(), wf_fn);
+    worker.register_activity(
+        "test_activity",
+        |_ctx: ActContext, echo: String| async move { Ok(echo) },
+    );
     join_all((0..CONCURRENCY).map(|i| {
         let worker = &worker;
-        let wf_id = format!("activity_load_{}", i);
+        let wf_id = format!("activity_load_{i}");
         async move {
             worker
                 .submit_wf(
@@ -71,42 +73,8 @@ async fn activity_load() {
     dbg!(starting.elapsed());
 
     let running = Instant::now();
-    let core = starter.get_worker().await;
-
-    // Poll for and complete all activities
-    let c2 = core.clone();
-    let all_acts = async move {
-        let mut act_complete_futs = vec![];
-        for _ in 0..CONCURRENCY {
-            let task = c2.poll_activity_task().await.unwrap();
-            assert_matches!(
-                task.variant,
-                Some(act_task::Variant::Start(ref start_activity)) => {
-                    assert_eq!(start_activity.activity_type, "test_activity")
-                }
-            );
-            let pd = payload_dat.clone();
-            let core = c2.clone();
-            act_complete_futs.push(tokio::spawn(async move {
-                core.complete_activity_task(ActivityTaskCompletion {
-                    task_token: task.task_token,
-                    result: Some(ActivityExecutionResult::ok(pd.into())),
-                })
-                .await
-                .unwrap()
-            }));
-        }
-        join_all(act_complete_futs)
-            .await
-            .into_iter()
-            .for_each(|h| h.unwrap());
-    };
-    tokio::join! {
-        async {
-            worker.run_until_done().await.unwrap();
-        },
-        all_acts
-    };
+
+    worker.run_until_done().await.unwrap();
     dbg!(running.elapsed());
 }
 
@@ -153,7 +121,7 @@ async fn workflow_load() {
 
     let mut workflow_handles = vec![];
     for i in 0..num_workflows {
-        let wfid = format!("{}_{}", wf_name, i);
+        let wfid = format!("{wf_name}_{i}");
         let rid = worker
             .submit_wf(
                 wfid.clone(),
@@ -171,7 +139,7 @@ async fn workflow_load() {
             let sends: FuturesUnordered<_> = (0..num_workflows)
                 .map(|i| {
                     client.signal_workflow_execution(
-                        format!("{}_{}", wf_name, i),
+                        format!("{wf_name}_{i}"),
                         "".to_string(),
                         SIGNAME.to_string(),
                         None,
@@ -187,5 +155,111 @@ async fn workflow_load() {
             tokio::time::sleep(Duration::from_secs(2)).await;
         }
     };
-    tokio::select! { r1 = worker.run_until_done() => {r1.unwrap()}, _ = sig_sender => {}};
+    tokio::select! { r1 = worker.run_until_done() => {r1.unwrap()}, _ = sig_sender => {}}
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn evict_while_la_running_no_interference() {
+    let wf_name = "evict_while_la_running_no_interference";
+    let mut starter = CoreWfStarter::new(wf_name);
+    starter.max_local_at(20);
+    starter.max_cached_workflows(20);
+    // Though it doesn't make sense to set wft higher than cached workflows, leaving this commented
+    // introduces more instability that can be useful in the test.
+    // starter.max_wft(20);
+    let mut worker = starter.worker().await;
+
+    worker.register_wf(wf_name.to_owned(), la_problem_workflow);
+    worker.register_activity("delay", |_: ActContext, _: String| async {
+        tokio::time::sleep(Duration::from_secs(15)).await;
+        Ok(())
+    });
+
+    let client = starter.get_client().await;
+    let subfs = FuturesUnordered::new();
+    for i in 1..100 {
+        let wf_id = format!("{wf_name}-{i}");
+        let run_id = worker
+            .submit_wf(
+                &wf_id,
+                wf_name.to_owned(),
+                vec![],
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+        let cw = worker.core_worker.clone();
+        let client = client.clone();
+        subfs.push(async move {
+            // Evict the workflow
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            cw.request_workflow_eviction(&run_id);
+            // Wake up workflow by sending signal
+            client
+                .signal_workflow_execution(
+                    wf_id,
+                    run_id.clone(),
+                    "whaatever".to_string(),
+                    None,
+                    None,
+                )
+                .await
+                .unwrap();
+        });
+    }
+    let runf = async {
+        worker.run_until_done().await.unwrap();
+    };
+    tokio::join!(subfs.collect::<Vec<_>>(), runf);
+}
+
+pub async fn many_parallel_timers_longhist(ctx: WfContext) -> WorkflowResult<()> {
+    for _ in 0..120 {
+        let mut futs = vec![];
+        for _ in 0..100 {
+            futs.push(ctx.timer(Duration::from_millis(100)));
+        }
+        join_all(futs).await;
+    }
+    Ok(().into())
+}
+
+#[tokio::test]
+async fn can_paginate_long_history() {
+    let wf_name = "can_paginate_long_history";
+    let mut starter = CoreWfStarter::new(wf_name);
+    starter.no_remote_activities();
+    // Do not use sticky queues so we are forced to paginate once history gets long
+    starter.max_cached_workflows(0);
+
+    let mut worker = starter.worker().await;
+    worker.register_wf(wf_name.to_owned(), many_parallel_timers_longhist);
+    let run_id = worker
+        .submit_wf(
+            wf_name.to_owned(),
+            wf_name.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    let client = starter.get_client().await;
+    tokio::spawn(async move {
+        loop {
+            for _ in 0..10 {
+                client
+                    .signal_workflow_execution(
+                        wf_name.to_owned(),
+                        run_id.clone(),
+                        "sig".to_string(),
+                        None,
+                        None,
+                    )
+                    .await
+                    .unwrap();
+            }
+            tokio::time::sleep(Duration::from_secs(3)).await;
+        }
+    });
+    worker.run_until_done().await.unwrap();
 }