@temporalio/core-bridge 1.8.6 → 1.9.0

package/sdk-core/test-utils/src/lib.rs

@@ -5,9 +5,13 @@
 extern crate tracing;
 
 pub mod canned_histories;
+pub mod interceptors;
 pub mod wf_input_saver;
 pub mod workflows;
 
+use anyhow::Context;
+pub use temporal_sdk_core::replay::HistoryForReplay;
+
 use crate::{
     stream::{Stream, TryStreamExt},
     wf_input_saver::stream_to_file,
@@ -29,17 +33,19 @@ use temporal_sdk::{
     interceptors::{FailOnNondeterminismInterceptor, WorkerInterceptor},
     IntoActivityFunc, Worker, WorkflowFunction,
 };
+#[cfg(feature = "ephemeral-server")]
+use temporal_sdk_core::ephemeral_server::{EphemeralExe, EphemeralExeVersion};
 use temporal_sdk_core::{
-    ephemeral_server::{EphemeralExe, EphemeralExeVersion},
     init_replay_worker, init_worker,
-    replay::HistoryForReplay,
+    replay::ReplayWorkerInput,
+    telemetry::{build_otlp_metric_exporter, start_prometheus_metric_exporter},
     ClientOptions, ClientOptionsBuilder, CoreRuntime, WorkerConfigBuilder,
 };
 use temporal_sdk_core_api::{
     errors::{PollActivityError, PollWfError},
     telemetry::{
-        Logger, MetricsExporter, OtelCollectorOptions, TelemetryOptions, TelemetryOptionsBuilder,
-        TraceExportConfig, TraceExporter,
+        metrics::CoreMeter, Logger, OtelCollectorOptionsBuilder, PrometheusExporterOptionsBuilder,
+        TelemetryOptions, TelemetryOptionsBuilder,
     },
     Worker as CoreWorker,
 };
@@ -51,7 +57,10 @@ use temporal_sdk_core_protos::{
         },
         workflow_completion::WorkflowActivationCompletion,
     },
-    temporal::api::{common::v1::Payload, history::v1::History},
+    temporal::api::{
+        common::v1::Payload, history::v1::History,
+        workflowservice::v1::StartWorkflowExecutionResponse,
+    },
     DEFAULT_ACTIVITY_TYPE,
 };
 use tokio::sync::{mpsc::unbounded_channel, OnceCell};
@@ -82,7 +91,7 @@ macro_rules! prost_dur {
 }
 
 /// Create a worker instance which will use the provided test name to base the task queue and wf id
-/// upon. Returns the instance and the task queue name (which is also the workflow id).
+/// upon. Returns the instance.
 pub async fn init_core_and_create_wf(test_name: &str) -> CoreWfStarter {
     let mut starter = CoreWfStarter::new(test_name);
     let _ = starter.get_worker().await;
@@ -109,8 +118,8 @@ where
         .worker_build_id("test_bin_id")
         .build()
         .expect("Configuration options construct properly");
-    let worker =
-        init_replay_worker(worker_cfg, histories).expect("Replay worker must init properly");
+    let worker = init_replay_worker(ReplayWorkerInput::new(worker_cfg, histories))
+        .expect("Replay worker must init properly");
     Arc::new(worker)
 }
 pub fn replay_sdk_worker<I>(histories: I) -> Worker
@@ -126,7 +135,7 @@ where
 {
     let core = init_core_replay_stream("replay_worker_test", histories);
     let mut worker = Worker::new_from_core(core, "replay_q".to_string());
-    worker.set_worker_interceptor(Box::new(FailOnNondeterminismInterceptor {}));
+    worker.set_worker_interceptor(FailOnNondeterminismInterceptor {});
     worker
 }
 
@@ -145,7 +154,9 @@ pub fn init_integ_telem() {
         let telemetry_options = get_integ_telem_options();
         let rt =
             CoreRuntime::new_assume_tokio(telemetry_options).expect("Core runtime inits cleanly");
-        let _ = tracing::subscriber::set_global_default(rt.trace_subscriber());
+        if let Some(sub) = rt.telemetry().trace_subscriber() {
+            let _ = tracing::subscriber::set_global_default(sub);
+        }
         rt
     });
 }
@@ -194,6 +205,18 @@ impl CoreWfStarter {
         }
     }
 
+    /// Create a new starter with no initialized worker or runtime override. Useful for starting a
+    /// new worker on the same queue.
+    pub fn clone_no_worker(&self) -> Self {
+        Self {
+            task_queue_name: self.task_queue_name.clone(),
+            worker_config: self.worker_config.clone(),
+            workflow_options: self.workflow_options.clone(),
+            runtime_override: None,
+            initted_worker: Default::default(),
+        }
+    }
+
     pub async fn worker(&mut self) -> TestWorker {
         let w = self.get_worker().await;
         let tq = w.get_config().task_queue.clone();
@@ -236,6 +259,23 @@ impl CoreWfStarter {
             .unwrap()
     }
 
+    pub async fn eager_start_with_worker(
+        &self,
+        wf_name: impl Into<String>,
+        worker: &mut TestWorker,
+    ) -> StartWorkflowExecutionResponse {
+        assert!(self.workflow_options.enable_eager_workflow_start);
+        worker
+            .eager_submit_wf(
+                self.task_queue_name.clone(),
+                wf_name.into(),
+                vec![],
+                self.workflow_options.clone(),
+            )
+            .await
+            .unwrap()
+    }
+
     pub async fn start_wf_with_id(&self, workflow_id: String) -> String {
         let iw = self.initted_worker.get().expect(
             "Worker must be initted before starting a workflow.\
@@ -275,7 +315,7 @@ impl CoreWfStarter {
         let with_id = HistoryForReplay::new(history, wf_id);
         let replay_worker = init_core_replay_preloaded(worker.task_queue(), [with_id]);
         worker.with_new_core_worker(replay_worker);
-        worker.set_worker_interceptor(Box::new(FailOnNondeterminismInterceptor {}));
+        worker.set_worker_interceptor(FailOnNondeterminismInterceptor {});
         worker.run().await.unwrap();
         Ok(())
     }
@@ -366,23 +406,17 @@ pub struct TestWorker {
     /// If set true (default), and a client is available, we will fetch workflow results to
     /// determine when they have all completed.
     pub fetch_results: bool,
-    iceptor: Option<TestWorkerCompletionIceptor>,
 }
 impl TestWorker {
     /// Create a new test worker
     pub fn new(core_worker: Arc<dyn CoreWorker>, task_queue: impl Into<String>) -> Self {
         let inner = Worker::new_from_core(core_worker.clone(), task_queue);
-        let iceptor = TestWorkerCompletionIceptor::new(
-            TestWorkerShutdownCond::NoAutoShutdown,
-            Arc::new(inner.shutdown_handle()),
-        );
         Self {
             inner,
             core_worker,
             client: None,
             started_workflows: Mutex::new(vec![]),
             fetch_results: true,
-            iceptor: Some(iceptor),
         }
     }
 
@@ -443,6 +477,39 @@ impl TestWorker {
         }
     }
 
+    /// Similar to `submit_wf` but checking that the server returns the first
+    /// workflow task in the client response.
+    /// Note that this does not guarantee that the worker will execute this task eagerly.
+    pub async fn eager_submit_wf(
+        &self,
+        workflow_id: impl Into<String>,
+        workflow_type: impl Into<String>,
+        input: Vec<Payload>,
+        options: WorkflowOptions,
+    ) -> Result<StartWorkflowExecutionResponse, anyhow::Error> {
+        let c = self.client.as_ref().context("client needed for eager wf")?;
+        let wfid = workflow_id.into();
+        let res = c
+            .start_workflow(
+                input,
+                self.inner.task_queue().to_string(),
+                wfid.clone(),
+                workflow_type.into(),
+                None,
+                options,
+            )
+            .await?;
+        res.eager_workflow_task
+            .as_ref()
+            .context("no eager workflow task")?;
+        self.started_workflows.lock().push(WorkflowExecutionInfo {
+            namespace: c.namespace().to_string(),
+            workflow_id: wfid,
+            run_id: Some(res.run_id.clone()),
+        });
+        Ok(res)
+    }
+
     /// Runs until all expected workflows have completed
     pub async fn run_until_done(&mut self) -> Result<(), anyhow::Error> {
         self.run_until_done_intercepted(Option::<TestWorkerCompletionIceptor>::None)
@@ -452,9 +519,12 @@ impl TestWorker {
     /// See [Self::run_until_done], but allows configuration of some low-level interception.
     pub async fn run_until_done_intercepted(
         &mut self,
-        interceptor: Option<impl WorkerInterceptor + 'static>,
+        next_interceptor: Option<impl WorkerInterceptor + 'static>,
     ) -> Result<(), anyhow::Error> {
-        let mut iceptor = self.iceptor.take().unwrap();
+        let mut iceptor = TestWorkerCompletionIceptor::new(
+            TestWorkerShutdownCond::NoAutoShutdown,
+            Arc::new(self.inner.shutdown_handle()),
+        );
         // Automatically use results-based complete detection if we have a client
         if self.fetch_results {
             if let Some(c) = self.client.clone() {
@@ -464,9 +534,9 @@ impl TestWorker {
                 );
             }
         }
-        iceptor.next = interceptor.map(|i| Box::new(i) as Box<dyn WorkerInterceptor>);
+        iceptor.next = next_interceptor.map(|i| Box::new(i) as Box<dyn WorkerInterceptor>);
         let get_results_waiter = iceptor.wait_all_wfs();
-        self.inner.set_worker_interceptor(Box::new(iceptor));
+        self.inner.set_worker_interceptor(iceptor);
         tokio::try_join!(self.inner.run(), get_results_waiter)?;
         Ok(())
     }
@@ -482,7 +552,6 @@ pub enum TestWorkerShutdownCond {
 pub struct TestWorkerCompletionIceptor {
     condition: TestWorkerShutdownCond,
     shutdown_handle: Arc<dyn Fn()>,
-    every_activation: Option<BoxDynActivationHook>,
     next: Option<Box<dyn WorkerInterceptor>>,
 }
 impl TestWorkerCompletionIceptor {
@@ -490,7 +559,6 @@ impl TestWorkerCompletionIceptor {
         Self {
             condition,
             shutdown_handle,
-            every_activation: None,
             next: None,
         }
     }
@@ -523,11 +591,8 @@ impl TestWorkerCompletionIceptor {
 #[async_trait::async_trait(?Send)]
 impl WorkerInterceptor for TestWorkerCompletionIceptor {
     async fn on_workflow_activation_completion(&self, completion: &WorkflowActivationCompletion) {
-        if let Some(func) = self.every_activation.as_ref() {
-            func(completion);
-        }
         if completion.has_execution_ending() {
-            info!("Workflow {} says it's finishing", &completion.run_id);
+            debug!("Workflow {} says it's finishing", &completion.run_id);
         }
         if let Some(n) = self.next.as_ref() {
             n.on_workflow_activation_completion(completion).await;
@@ -585,22 +650,24 @@ pub fn get_integ_telem_options() -> TelemetryOptions {
         .ok()
         .map(|x| x.parse::<Url>().unwrap())
     {
-        let opts = OtelCollectorOptions {
-            url,
-            headers: Default::default(),
-            metric_periodicity: None,
-        };
-        ob.tracing(TraceExportConfig {
-            filter: filter_string.clone(),
-            exporter: TraceExporter::Otel(opts.clone()),
-        });
-        ob.metrics(MetricsExporter::Otel(opts));
+        let opts = OtelCollectorOptionsBuilder::default()
+            .url(url)
+            .build()
+            .unwrap();
+        ob.metrics(Arc::new(build_otlp_metric_exporter(opts).unwrap()) as Arc<dyn CoreMeter>);
     }
     if let Some(addr) = env::var(PROM_ENABLE_ENV_VAR)
         .ok()
         .map(|x| SocketAddr::new([127, 0, 0, 1].into(), x.parse().unwrap()))
     {
-        ob.metrics(MetricsExporter::Prometheus(addr));
+        let prom_info = start_prometheus_metric_exporter(
+            PrometheusExporterOptionsBuilder::default()
+                .socket_addr(addr)
+                .build()
+                .unwrap(),
+        )
+        .unwrap();
+        ob.metrics(prom_info.meter as Arc<dyn CoreMeter>);
     }
     ob.logging(Logger::Console {
         filter: filter_string,
@@ -609,6 +676,7 @@ pub fn get_integ_telem_options() -> TelemetryOptions {
     .unwrap()
 }
 
+#[cfg(feature = "ephemeral-server")]
 pub fn default_cached_download() -> EphemeralExe {
     EphemeralExe::CachedDownload {
         version: EphemeralExeVersion::SDKDefault {

package/sdk-core/tests/integ_tests/metrics_tests.rs

@@ -1,8 +1,15 @@
 use assert_matches::assert_matches;
-use std::{sync::Arc, time::Duration};
+use std::{net::SocketAddr, sync::Arc, time::Duration};
 use temporal_client::{WorkflowClientTrait, WorkflowOptions, WorkflowService};
-use temporal_sdk_core::{init_worker, CoreRuntime};
-use temporal_sdk_core_api::{telemetry::MetricsExporter, worker::WorkerConfigBuilder, Worker};
+use temporal_sdk_core::{init_worker, telemetry::start_prometheus_metric_exporter, CoreRuntime};
+use temporal_sdk_core_api::{
+    telemetry::{
+        metrics::{CoreMeter, MetricAttributes, MetricParameters},
+        PrometheusExporterOptionsBuilder, TelemetryOptions,
+    },
+    worker::WorkerConfigBuilder,
+    Worker,
+};
 use temporal_sdk_core_protos::{
     coresdk::{
         activity_result::ActivityExecutionResult,
@@ -23,7 +30,7 @@ use temporal_sdk_core_protos::{
 use temporal_sdk_core_test_utils::{
     get_integ_server_options, get_integ_telem_options, CoreWfStarter, NAMESPACE,
 };
-use tokio::{join, sync::Barrier};
+use tokio::{join, sync::Barrier, task::AbortHandle};
 
 static ANY_PORT: &str = "127.0.0.1:0";
 
@@ -31,15 +38,41 @@ async fn get_text(endpoint: String) -> String {
     reqwest::get(endpoint).await.unwrap().text().await.unwrap()
 }
 
+struct AbortOnDrop {
+    ah: AbortHandle,
+}
+impl Drop for AbortOnDrop {
+    fn drop(&mut self) {
+        self.ah.abort();
+    }
+}
+
+fn prom_metrics() -> (TelemetryOptions, SocketAddr, AbortOnDrop) {
+    let mut telemopts = get_integ_telem_options();
+    let prom_info = start_prometheus_metric_exporter(
+        PrometheusExporterOptionsBuilder::default()
+            .socket_addr(ANY_PORT.parse().unwrap())
+            .build()
+            .unwrap(),
+    )
+    .unwrap();
+    telemopts.metrics = Some(prom_info.meter as Arc<dyn CoreMeter>);
+    (
+        telemopts,
+        prom_info.bound_addr,
+        AbortOnDrop {
+            ah: prom_info.abort_handle,
+        },
+    )
+}
+
 #[tokio::test]
 async fn prometheus_metrics_exported() {
-    let mut telemopts = get_integ_telem_options();
-    telemopts.metrics = Some(MetricsExporter::Prometheus(ANY_PORT.parse().unwrap()));
+    let (telemopts, addr, _aborter) = prom_metrics();
     let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
-    let addr = rt.telemetry().prom_port().unwrap();
     let opts = get_integ_server_options();
     let mut raw_client = opts
-        .connect_no_namespace(rt.metric_meter().as_deref(), None)
+        .connect_no_namespace(rt.telemetry().get_temporal_metric_meter(), None)
         .await
         .unwrap();
     assert!(raw_client.get_client().capabilities().is_some());
@@ -56,15 +89,26 @@ async fn prometheus_metrics_exported() {
     assert!(body.contains(
         "temporal_request_latency_count{operation=\"GetSystemInfo\",service_name=\"temporal-core-sdk\"} 1"
     ));
+    // Verify counter names are appropriate (don't end w/ '_total')
+    assert!(body.contains("temporal_request{"));
+    // Verify non-temporal metrics meter does not prefix
+    let mm = rt.telemetry().get_metric_meter().unwrap();
+    let g = mm.inner.gauge(MetricParameters::from("mygauge"));
+    g.record(
+        42,
+        &MetricAttributes::OTel {
+            kvs: Arc::new(vec![]),
+        },
+    );
+    let body = get_text(format!("http://{addr}/metrics")).await;
+    assert!(body.contains("\nmygauge 42"));
 }
 
 #[tokio::test]
 async fn one_slot_worker_reports_available_slot() {
-    let mut telemopts = get_integ_telem_options();
+    let (telemopts, addr, _aborter) = prom_metrics();
     let tq = "one_slot_worker_tq";
-    telemopts.metrics = Some(MetricsExporter::Prometheus(ANY_PORT.parse().unwrap()));
     let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
-    let addr = rt.telemetry().prom_port().unwrap();
 
     let worker_cfg = WorkerConfigBuilder::default()
         .namespace(NAMESPACE)
@@ -161,6 +205,16 @@ async fn one_slot_worker_reports_available_slot() {
              service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
              worker_type=\"WorkflowWorker\"}} 2"
         )));
+        assert!(body.contains(&format!(
+            "temporal_worker_task_slots_available{{namespace=\"{NAMESPACE}\",\
+             service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
+             worker_type=\"ActivityWorker\"}} 1"
+        )));
+        assert!(body.contains(&format!(
+            "temporal_worker_task_slots_available{{namespace=\"{NAMESPACE}\",\
+             service_name=\"temporal-core-sdk\",task_queue=\"one_slot_worker_tq\",\
+             worker_type=\"LocalActivityWorker\"}} 1"
+        )));
 
         // Start a workflow so that a task will get delivered
         client
@@ -182,7 +236,7 @@ async fn one_slot_worker_reports_available_slot() {
         wf_task_barr.wait().await;
 
         // At this point the workflow task is outstanding, so there should be 0 slots, and
-        // the activities haven't started, so there should be 1 each.
+        // the activities haven't started, so there should still be 1 each.
         let body = get_text(format!("http://{addr}/metrics")).await;
         assert!(body.contains(&format!(
             "temporal_worker_task_slots_available{{namespace=\"{NAMESPACE}\",\
@@ -264,10 +318,8 @@ async fn query_of_closed_workflow_doesnt_tick_terminal_metric(
     )]
     completion: workflow_command::Variant,
 ) {
-    let mut telemopts = get_integ_telem_options();
-    telemopts.metrics = Some(MetricsExporter::Prometheus(ANY_PORT.parse().unwrap()));
+    let (telemopts, addr, _aborter) = prom_metrics();
     let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
-    let addr = rt.telemetry().prom_port().unwrap();
     let mut starter =
         CoreWfStarter::new_with_runtime("query_of_closed_workflow_doesnt_tick_terminal_metric", rt);
     // Disable cache to ensure replay happens completely
@@ -275,7 +327,22 @@ async fn query_of_closed_workflow_doesnt_tick_terminal_metric(
     let worker = starter.get_worker().await;
     let run_id = starter.start_wf().await;
     let task = worker.poll_workflow_activation().await.unwrap();
+    // Fail wf task
+    worker
+        .complete_workflow_activation(WorkflowActivationCompletion::fail(
+            task.run_id,
+            "whatever".into(),
+        ))
+        .await
+        .unwrap();
+    // Handle cache eviction
+    let task = worker.poll_workflow_activation().await.unwrap();
+    worker
+        .complete_workflow_activation(WorkflowActivationCompletion::empty(task.run_id))
+        .await
+        .unwrap();
     // Immediately complete the workflow
+    let task = worker.poll_workflow_activation().await.unwrap();
     worker
         .complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
             task.run_id,
@@ -370,3 +437,31 @@ async fn query_of_closed_workflow_doesnt_tick_terminal_metric(
         .expect("Must find matching metric");
     assert!(matching_line.ends_with('1'));
 }
+
+#[test]
+fn runtime_new() {
+    let mut rt = CoreRuntime::new(
+        get_integ_telem_options(),
+        tokio::runtime::Builder::new_multi_thread(),
+    )
+    .unwrap();
+    let handle = rt.tokio_handle();
+    let _rt = handle.enter();
+    let (telemopts, addr, _aborter) = prom_metrics();
+    rt.telemetry_mut()
+        .attach_late_init_metrics(telemopts.metrics.unwrap());
+    let opts = get_integ_server_options();
+    handle.block_on(async {
+        let mut raw_client = opts
+            .connect_no_namespace(rt.telemetry().get_temporal_metric_meter(), None)
+            .await
+            .unwrap();
+        assert!(raw_client.get_client().capabilities().is_some());
+        let _ = raw_client
+            .list_namespaces(ListNamespacesRequest::default())
+            .await
+            .unwrap();
+        let body = get_text(format!("http://{addr}/metrics")).await;
+        assert!(body.contains("temporal_request"));
+    });
+}

package/sdk-core/tests/integ_tests/queries_tests.rs

@@ -1,6 +1,7 @@
 use assert_matches::assert_matches;
 use futures::{prelude::stream::FuturesUnordered, FutureExt, StreamExt};
-use std::time::Duration;
+use futures_util::future::join_all;
+use std::time::{Duration, Instant};
 use temporal_client::WorkflowClientTrait;
 use temporal_sdk_core_protos::{
     coresdk::{
@@ -13,6 +14,7 @@ use temporal_sdk_core_protos::{
 use temporal_sdk_core_test_utils::{
     drain_pollers_and_shutdown, init_core_and_create_wf, WorkerTestHelpers,
 };
+use tokio::join;
 
 #[tokio::test]
 async fn simple_query_legacy() {
@@ -102,7 +104,7 @@ async fn simple_query_legacy() {
         let task = core.poll_workflow_activation().await.unwrap();
         core.complete_execution(&task.run_id).await;
     };
-    let (q_resp, _) = tokio::join!(query_fut, workflow_completions_future);
+    let (q_resp, _) = join!(query_fut, workflow_completions_future);
     // Ensure query response is as expected
     assert_eq!(&q_resp.unwrap()[0].data, query_resp);
 }
@@ -308,7 +310,176 @@ async fn fail_legacy_query() {
         let task = core.poll_workflow_activation().await.unwrap();
         core.complete_execution(&task.run_id).await;
     };
-    let (q_resp, _) = tokio::join!(query_fut, workflow_completions_future);
+    let (q_resp, _) = join!(query_fut, workflow_completions_future);
     // Ensure query response is a failure and has the right message
     assert_eq!(q_resp.message(), query_err);
 }
+
+#[tokio::test]
+async fn multiple_concurrent_queries_no_new_history() {
+    let mut starter = init_core_and_create_wf("multiple_concurrent_queries_no_new_history").await;
+    let core = starter.get_worker().await;
+    let workflow_id = starter.get_task_queue().to_string();
+    let started = Instant::now();
+    let task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        task.run_id.clone(),
+        vec![], // complete with no commands so that there will be no new history
+    ))
+    .await
+    .unwrap();
+    let client = starter.get_client().await;
+    let num_queries = 10;
+    let query_futs = (1..=num_queries).map(|_| async {
+        client
+            .query_workflow_execution(
+                workflow_id.to_string(),
+                task.run_id.to_string(),
+                WorkflowQuery {
+                    query_type: "myquery".to_string(),
+                    query_args: Some(b"hi".into()),
+                    header: None,
+                },
+            )
+            .await
+            .unwrap();
+    });
+    let complete_fut = async {
+        for _ in 1..=num_queries {
+            let task = core.poll_workflow_activation().await.unwrap();
+            let query = assert_matches!(
+                task.jobs.as_slice(),
+                [WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::QueryWorkflow(q)),
+                }] => q
+            );
+            core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+                task.run_id,
+                QueryResult {
+                    query_id: query.query_id.clone(),
+                    variant: Some(
+                        QuerySuccess {
+                            response: Some("done".into()),
+                        }
+                        .into(),
+                    ),
+                }
+                .into(),
+            ))
+            .await
+            .unwrap();
+        }
+    };
+    join!(join_all(query_futs), complete_fut);
+    // No need to properly finish
+    client
+        .terminate_workflow_execution(workflow_id, None)
+        .await
+        .unwrap();
+    // This test should not take a long time. Things can still work, but if it takes a long time
+    // that means we aren't buffering properly, and tasks are getting redelivered after timeout.
+    if started.elapsed() > Duration::from_secs(9) {
+        panic!("Should not have taken this long");
+    }
+}
+
+#[tokio::test]
+async fn query_superseded_by_newer_wft_is_discarded() {
+    let mut starter = init_core_and_create_wf("query_superseded_by_newer_wft_is_discarded").await;
+    let core = starter.get_worker().await;
+    let workflow_id = starter.get_task_queue().to_string();
+    let task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        task.run_id.clone(),
+        vec![], // complete with no commands so that there will be no new history
+    ))
+    .await
+    .unwrap();
+    let client = starter.get_client().await;
+    // Send two queries so that one of them is buffered
+    let query_futs = (1..=2).map(|_| async {
+        client
+            .query_workflow_execution(
+                workflow_id.to_string(),
+                task.run_id.to_string(),
+                WorkflowQuery {
+                    query_type: "myquery".to_string(),
+                    query_args: Some(b"hi".into()),
+                    header: None,
+                },
+            )
+            .await
+            .unwrap();
+    });
+    let complete_fut = async {
+        let task = core.poll_workflow_activation().await.unwrap();
+        let query = assert_matches!(
+            task.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::QueryWorkflow(q)),
+            }] => q
+        );
+        // While handling the first query, signal the workflow so a new WFT is generated and the
+        // second query is still in the buffer
+        client
+            .signal_workflow_execution(
+                workflow_id.to_string(),
+                task.run_id.to_string(),
+                "blah".to_string(),
+                None,
+                None,
+            )
+            .await
+            .unwrap();
+        tokio::time::sleep(Duration::from_millis(500)).await;
+        core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+            task.run_id,
+            QueryResult {
+                query_id: query.query_id.clone(),
+                variant: Some(
+                    QuerySuccess {
+                        response: Some("done".into()),
+                    }
+                    .into(),
+                ),
+            }
+            .into(),
+        ))
+        .await
+        .unwrap();
+        // We should get the signal activation since the in-buffer query should've been failed
+        let task = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            task.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+            }]
+        );
+        core.complete_execution(&task.run_id).await;
+        // Query will get retried by server since we fail the task w/ the stale query
+        let task = core.poll_workflow_activation().await.unwrap();
+        let query = assert_matches!(
+            task.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::QueryWorkflow(q)),
+            }] => q
+        );
+        core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+            task.run_id,
+            QueryResult {
+                query_id: query.query_id.clone(),
+                variant: Some(
+                    QuerySuccess {
+                        response: Some("done".into()),
+                    }
+                    .into(),
+                ),
+            }
+            .into(),
+        ))
+        .await
+        .unwrap();
+    };
+    join!(join_all(query_futs), complete_fut);
+    drain_pollers_and_shutdown(&core).await;
+}