@temporalio/core-bridge 1.13.2 → 1.14.0

package/sdk-core/{core → crates/sdk-core}/src/core_tests/workflow_tasks.rs

@@ -27,46 +27,47 @@ use std::{
     },
     time::Duration,
 };
-use temporal_sdk_core_api::{
+use temporalio_client::MESSAGE_TOO_LARGE_KEY;
+use temporalio_common::{
     Worker as WorkerTrait,
     errors::PollError,
-    worker::{
-        PollerBehavior, SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext,
-        SlotSupplier, SlotSupplierPermit, WorkflowSlotKind,
-    },
-};
-use temporal_sdk_core_protos::{
-    canned_histories,
-    coresdk::{
-        activity_result::{self as ar, ActivityResolution, activity_resolution},
-        common::VersioningIntent,
-        workflow_activation::{
-            FireTimer, InitializeWorkflow, ResolveActivity, UpdateRandomSeed,
-            WorkflowActivationJob, remove_from_cache::EvictionReason, workflow_activation_job,
+    protos::{
+        canned_histories,
+        coresdk::{
+            activity_result::{self as ar, ActivityResolution, activity_resolution},
+            common::VersioningIntent,
+            workflow_activation::{
+                FireTimer, InitializeWorkflow, ResolveActivity, UpdateRandomSeed,
+                WorkflowActivationJob, remove_from_cache::EvictionReason, workflow_activation_job,
+            },
+            workflow_commands::{
+                ActivityCancellationType, CancelTimer, CompleteWorkflowExecution,
+                ContinueAsNewWorkflowExecution, FailWorkflowExecution, RequestCancelActivity,
+                ScheduleActivity, SetPatchMarker, StartChildWorkflowExecution, UpdateResponse,
+                update_response::Response, workflow_command,
+            },
+            workflow_completion::WorkflowActivationCompletion,
         },
-        workflow_commands::{
-            ActivityCancellationType, CancelTimer, CompleteWorkflowExecution,
-            ContinueAsNewWorkflowExecution, FailWorkflowExecution, RequestCancelActivity,
-            ScheduleActivity, SetPatchMarker, StartChildWorkflowExecution, UpdateResponse,
-            update_response::Response, workflow_command,
+        default_act_sched, default_wes_attribs,
+        temporal::api::{
+            command::v1::command::Attributes,
+            common::v1::{Payload, RetryPolicy},
+            enums::v1::{CommandType, EventType, WorkflowTaskFailedCause},
+            failure::v1::Failure,
+            history::v1::{
+                TimerFiredEventAttributes, WorkflowPropertiesModifiedExternallyEventAttributes,
+                history_event,
+            },
+            workflowservice::v1::{
+                GetWorkflowExecutionHistoryResponse, RespondWorkflowTaskCompletedResponse,
+            },
         },
-        workflow_completion::WorkflowActivationCompletion,
+        test_utils::start_timer_cmd,
     },
-    default_act_sched, default_wes_attribs,
-    temporal::api::{
-        command::v1::command::Attributes,
-        common::v1::{Payload, RetryPolicy},
-        enums::v1::{CommandType, EventType, WorkflowTaskFailedCause},
-        failure::v1::Failure,
-        history::v1::{
-            TimerFiredEventAttributes, WorkflowPropertiesModifiedExternallyEventAttributes,
-            history_event,
-        },
-        workflowservice::v1::{
-            GetWorkflowExecutionHistoryResponse, RespondWorkflowTaskCompletedResponse,
-        },
+    worker::{
+        PollerBehavior, SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext,
+        SlotSupplier, SlotSupplierPermit, WorkerTaskTypes, WorkflowSlotKind,
     },
-    test_utils::start_timer_cmd,
 };
 use tokio::{
     join,
@@ -1286,6 +1287,40 @@ async fn fail_wft_then_recover() {
     core.shutdown().await;
 }
 
+#[tokio::test]
+async fn default_wft_fail_cause_is_worker_unhandled() {
+    let t = canned_histories::single_timer("1");
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::AllHistory],
+        mock_worker_client(),
+    );
+    mh.num_expected_fails = 1;
+    mh.expect_fail_wft_matcher = Box::new(|_, cause, _| {
+        matches!(
+            cause,
+            WorkflowTaskFailedCause::WorkflowWorkerUnhandledFailure
+        )
+    });
+    let mock = build_mock_pollers(mh);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::fail(
+        act.run_id,
+        Failure {
+            message: "Simulated workflow failure".to_string(),
+            ..Default::default()
+        },
+        None,
+    ))
+    .await
+    .unwrap();
+
+    core.shutdown().await;
+}
+
 #[tokio::test]
 async fn poll_response_triggers_wf_error() {
     let mut t = TestHistoryBuilder::default();
@@ -2636,7 +2671,7 @@ async fn poller_wont_run_ahead_of_task_slots() {
             .max_cached_workflows(10_usize)
             .max_outstanding_workflow_tasks(10_usize)
             .workflow_task_poller_behavior(PollerBehavior::SimpleMaximum(10_usize))
-            .no_remote_activities(true)
+            .task_types(WorkerTaskTypes::workflow_only())
             .build()
             .unwrap(),
         mock_client,
@@ -2696,7 +2731,7 @@ async fn poller_wont_poll_until_lang_polls() {
 
     let worker = Worker::new_test(
         test_worker_cfg()
-            .no_remote_activities(true)
+            .task_types(WorkerTaskTypes::workflow_only())
             .build()
             .unwrap(),
         mock_client,
@@ -2841,7 +2876,7 @@ async fn slot_provider_cant_hand_out_more_permits_than_cache_size() {
                     .build(),
             ))
             .workflow_task_poller_behavior(PollerBehavior::SimpleMaximum(10_usize))
-            .no_remote_activities(true)
+            .task_types(WorkerTaskTypes::workflow_only())
             .build()
             .unwrap(),
         mock_client,
@@ -2989,14 +3024,15 @@ async fn both_normal_and_sticky_pollers_poll_concurrently() {
             .max_outstanding_workflow_tasks(2_usize)
             .workflow_task_poller_behavior(PollerBehavior::SimpleMaximum(2_usize))
             .nonsticky_to_sticky_poll_ratio(0.2)
-            .no_remote_activities(true)
+            .task_types(WorkerTaskTypes::workflow_only())
             .build()
             .unwrap(),
         Some("stickytq".to_string()),
         Arc::new(mock_client),
         None,
         None,
-    );
+    )
+    .unwrap();
 
     for _ in 1..50 {
         let activation = worker.poll_workflow_activation().await.unwrap();
@@ -3035,3 +3071,54 @@ async fn both_normal_and_sticky_pollers_poll_concurrently() {
         "At peak, there should be exactly 2 pollers active at the same time"
     );
 }
+
+#[tokio::test]
+async fn grpc_message_too_large_doesnt_spam_task_fails() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [
+            ResponseType::AllHistory,
+            ResponseType::AllHistory,
+            ResponseType::AllHistory,
+            ResponseType::AllHistory,
+            ResponseType::AllHistory,
+            ResponseType::AllHistory,
+        ],
+        mock_worker_client(),
+    );
+    mh.num_expected_fails = 1;
+    let mut times = 1;
+    mh.completion_mock_fn = Some(Box::new(move |_| {
+        if times <= 5 {
+            let mut err = tonic::Status::new(tonic::Code::ResourceExhausted, "message too large");
+            // This key is what we look for
+            err.metadata_mut().insert(MESSAGE_TOO_LARGE_KEY, 1.into());
+            times += 1;
+            Err(err)
+        } else {
+            Ok(Default::default())
+        }
+    }));
+
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    // Since the mock makes us fail 5 times, we should succeed on the sixth
+    for _ in 1..=5 {
+        let act = core.poll_workflow_activation().await.unwrap();
+        core.complete_workflow_activation(WorkflowActivationCompletion::empty(&act.run_id))
+            .await
+            .unwrap();
+        core.handle_eviction().await;
+    }
+    let act = core.poll_workflow_activation().await.unwrap();
+    core.complete_execution(&act.run_id).await;
+    core.drain_pollers_and_shutdown().await;
+    // Mock only expects 1 task failure, and would fail here if we spammed
+}

package/sdk-core/{core → crates/sdk-core}/src/debug_client.rs

@@ -6,7 +6,7 @@ use hyper::http::HeaderValue;
 use prost::Message;
 use reqwest::{self, header::HeaderMap};
 use std::time::Duration;
-use temporal_sdk_core_protos::temporal::api::history::v1::History;
+use temporalio_common::protos::temporal::api::history::v1::History;
 use url::Url;
 
 /// A client for interacting with the VSCode debug plugin

package/sdk-core/{core → crates/sdk-core}/src/ephemeral_server/mod.rs

@@ -10,7 +10,7 @@ use std::{
     io,
     path::{Path, PathBuf},
 };
-use temporal_client::ClientOptionsBuilder;
+use temporalio_client::ClientOptions;
 use tokio::{
     task::spawn_blocking,
     time::{Duration, sleep},
@@ -224,12 +224,12 @@ impl EphemeralServer {
         // Try to connect every 100ms for 5s
         // TODO(cretz): Some other way, e.g. via stdout, to know whether the
         // server is up?
-        let client_options = ClientOptionsBuilder::default()
+        let client_options = ClientOptions::builder()
             .identity("online_checker".to_owned())
             .target_url(Url::parse(&target_url)?)
             .client_name("online-checker".to_owned())
             .client_version("0.1.0".to_owned())
-            .build()?;
+            .build();
         let mut last_error = None;
         for _ in 0..50 {
             sleep(Duration::from_millis(100)).await;

package/sdk-core/crates/sdk-core/src/histfetch.rs

@@ -0,0 +1,33 @@
+//! Use this binary to fetch histories as proto-encoded binary. The first argument must be a
+//! workflow ID. A run id may optionally be provided as the second arg. The history is written to
+//! `{workflow_id}_history.bin`.
+//!
+//! We can use `clap` if this needs more arguments / other stuff later on.
+
+use prost::Message;
+use temporalio_client::{ClientOptions, WorkflowClientTrait};
+use url::Url;
+
+#[tokio::main]
+async fn main() -> Result<(), anyhow::Error> {
+    let url = Url::try_from("http://localhost:7233").unwrap();
+    let copts = ClientOptions::builder()
+        .client_name("histfetch")
+        .client_version("0.0")
+        .target_url(url)
+        .build();
+    let client = copts.connect("default", None).await?;
+    let wf_id = std::env::args()
+        .nth(1)
+        .expect("must provide workflow id as only argument");
+    let run_id = std::env::args().nth(2);
+    let hist = client
+        .get_workflow_execution_history(wf_id.clone(), run_id, vec![])
+        .await?
+        .history
+        .expect("history field must be populated");
+    // Serialize history to file
+    let byteified = hist.encode_to_vec();
+    tokio::fs::write(format!("{wf_id}_history.bin"), &byteified).await?;
+    Ok(())
+}

package/sdk-core/{core → crates/sdk-core}/src/internal_flags.rs

@@ -6,7 +6,7 @@ use std::{
     collections::{BTreeSet, HashSet},
     iter,
 };
-use temporal_sdk_core_protos::temporal::api::{
+use temporalio_common::protos::temporal::api::{
     history::v1::WorkflowTaskCompletedEventAttributes, sdk::v1::WorkflowTaskCompletedMetadata,
     workflowservice::v1::get_system_info_response,
 };
@@ -35,7 +35,7 @@ pub enum CoreInternalFlags {
     /// multiple workflow coroutines generated a terminal command). This has the
     /// consequence that all non-terminal commands are sent to the server, even
     /// if in the sequence delivered by lang they came after a terminal command.
-    /// See https://github.com/temporalio/features/issues/481.
+    /// See <https://github.com/temporalio/features/issues/481>.
     MoveTerminalCommands = 3,
     /// We received a value higher than this code can understand.
     TooHigh = u32::MAX,
@@ -233,8 +233,9 @@ impl CoreInternalFlags {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use temporal_sdk_core_protos::temporal::api::workflowservice::v1::get_system_info_response::Capabilities;
+    use temporalio_common::protos::temporal::api::workflowservice::v1::get_system_info_response::Capabilities;
 
+    #[allow(clippy::derivable_impls)] // Only want this in test
     impl Default for InternalFlags {
         fn default() -> Self {
             Self::Disabled

package/sdk-core/{core → crates/sdk-core}/src/lib.rs

@@ -12,6 +12,8 @@ extern crate tracing;
 extern crate core;
 
 mod abstractions;
+#[cfg(feature = "antithesis_assertions")]
+mod antithesis;
 #[cfg(feature = "debug-plugin")]
 pub mod debug_client;
 #[cfg(feature = "ephemeral-server")]
@@ -30,21 +32,18 @@ mod core_tests;
 #[macro_use]
 pub mod test_help;
 
-pub(crate) use temporal_sdk_core_api::errors;
+pub(crate) use temporalio_common::errors;
 
 pub use pollers::{
-    Client, ClientOptions, ClientOptionsBuilder, ClientTlsConfig, RetryClient, RetryConfig,
-    TlsConfig, WorkflowClientTrait,
+    Client, ClientOptions, ClientTlsOptions, RetryClient, RetryOptions, TlsOptions,
+    WorkflowClientTrait,
 };
-pub use temporal_sdk_core_api as api;
-pub use temporal_sdk_core_protos as protos;
-pub use temporal_sdk_core_protos::TaskToken;
+pub use temporalio_common::protos::TaskToken;
 pub use url::Url;
 pub use worker::{
-    FixedSizeSlotSupplier, RealSysInfo, ResourceBasedSlotsOptions,
-    ResourceBasedSlotsOptionsBuilder, ResourceBasedTuner, ResourceSlotOptions, SlotSupplierOptions,
-    TunerBuilder, TunerHolder, TunerHolderOptions, TunerHolderOptionsBuilder, Worker, WorkerConfig,
-    WorkerConfigBuilder,
+    FixedSizeSlotSupplier, ResourceBasedSlotsOptions, ResourceBasedSlotsOptionsBuilder,
+    ResourceBasedTuner, ResourceSlotOptions, SlotSupplierOptions, TunerBuilder, TunerHolder,
+    TunerHolderOptions, TunerHolderOptionsBuilder, Worker, WorkerConfig, WorkerConfigBuilder,
 };
 
 /// Expose [WorkerClient] symbols
@@ -61,14 +60,14 @@ use crate::{
 };
 use anyhow::bail;
 use futures_util::Stream;
-use std::sync::{Arc, OnceLock};
-use temporal_client::{ConfiguredClient, NamespacedClient, SharedReplaceableClient};
-use temporal_sdk_core_api::{
+use std::{sync::Arc, time::Duration};
+use temporalio_client::{ConfiguredClient, NamespacedClient, SharedReplaceableClient};
+use temporalio_common::{
     Worker as WorkerTrait,
     errors::{CompleteActivityError, PollError},
+    protos::coresdk::ActivityHeartbeat,
     telemetry::TelemetryOptions,
 };
-use temporal_sdk_core_protos::coresdk::ActivityHeartbeat;
 
 /// Initialize a worker bound to a task queue.
 ///
@@ -89,40 +88,40 @@ pub fn init_worker<CT>(
 where
     CT: Into<sealed::AnyClient>,
 {
-    if worker_config.namespace.is_empty() {
+    let namespace = worker_config.namespace.clone();
+    if namespace.is_empty() {
         bail!("Worker namespace cannot be empty");
     }
 
     let client = RetryClient::new(
-        SharedReplaceableClient::new(init_worker_client(&worker_config, client)),
-        RetryConfig::default(),
+        SharedReplaceableClient::new(init_worker_client(
+            worker_config.namespace.clone(),
+            worker_config.client_identity_override.clone(),
+            client,
+        )),
+        RetryOptions::default(),
     );
     let client_ident = client.identity();
-    let sticky_q = sticky_q_name_for_worker(&client_ident, &worker_config);
+    let sticky_q = sticky_q_name_for_worker(&client_ident, worker_config.max_cached_workflows);
 
     if client_ident.is_empty() {
         bail!("Client identity cannot be empty. Either lang or user should be setting this value");
     }
 
-    let heartbeat_fn = worker_config
-        .heartbeat_interval
-        .map(|_| Arc::new(OnceLock::new()));
-
     let client_bag = Arc::new(WorkerClientBag::new(
         client,
-        worker_config.namespace.clone(),
-        client_ident,
+        namespace.clone(),
+        client_ident.clone(),
         worker_config.versioning_strategy.clone(),
-        heartbeat_fn.clone(),
     ));
 
-    Ok(Worker::new(
-        worker_config,
+    Worker::new(
+        worker_config.clone(),
         sticky_q,
-        client_bag,
+        client_bag.clone(),
         Some(&runtime.telemetry),
-        heartbeat_fn,
-    ))
+        runtime.heartbeat_interval,
+    )
 }
 
 /// Create a worker for replaying one or more existing histories. It will auto-shutdown as soon as
@@ -142,12 +141,16 @@ where
     rwi.into_core_worker()
 }
 
-pub(crate) fn init_worker_client<CT>(config: &WorkerConfig, client: CT) -> Client
+pub(crate) fn init_worker_client<CT>(
+    namespace: String,
+    client_identity_override: Option<String>,
+    client: CT,
+) -> Client
 where
     CT: Into<sealed::AnyClient>,
 {
-    let mut client = Client::new(*client.into().into_inner(), config.namespace.clone());
-    if let Some(ref id_override) = config.client_identity_override {
+    let mut client = Client::new(*client.into().into_inner(), namespace.clone());
+    if let Some(ref id_override) = client_identity_override {
         client.options_mut().identity.clone_from(id_override);
     }
     client
@@ -157,9 +160,9 @@ where
 /// workflows.
 pub(crate) fn sticky_q_name_for_worker(
     process_identity: &str,
-    config: &WorkerConfig,
+    max_cached_workflows: usize,
 ) -> Option<String> {
-    if config.max_cached_workflows > 0 {
+    if max_cached_workflows > 0 {
         Some(format!(
             "{}-{}",
             &process_identity,
@@ -172,7 +175,7 @@ pub(crate) fn sticky_q_name_for_worker(
 
 mod sealed {
     use super::*;
-    use temporal_client::{SharedReplaceableClient, TemporalServiceClient};
+    use temporalio_client::{SharedReplaceableClient, TemporalServiceClient};
 
     /// Allows passing different kinds of clients into things that want to be flexible. Motivating
     /// use-case was worker initialization.
@@ -234,6 +237,37 @@ pub struct CoreRuntime {
     telemetry: TelemetryInstance,
     runtime: Option<tokio::runtime::Runtime>,
     runtime_handle: tokio::runtime::Handle,
+    heartbeat_interval: Option<Duration>,
+}
+
+/// Holds telemetry options, as well as worker heartbeat_interval. Construct with [RuntimeOptionsBuilder]
+#[derive(Default, derive_builder::Builder)]
+#[builder(build_fn(validate = "Self::validate"))]
+#[non_exhaustive]
+pub struct RuntimeOptions {
+    /// Telemetry configuration options.
+    #[builder(default)]
+    telemetry_options: TelemetryOptions,
+    /// Optional worker heartbeat interval - This configures the heartbeat setting of all
+    /// workers created using this runtime.
+    ///
+    /// Interval must be between 1s and 60s, inclusive.
+    #[builder(default = "Some(Duration::from_secs(60))")]
+    heartbeat_interval: Option<Duration>,
+}
+
+impl RuntimeOptionsBuilder {
+    fn validate(&self) -> Result<(), String> {
+        if let Some(Some(interval)) = self.heartbeat_interval
+            && (interval < Duration::from_secs(1) || interval > Duration::from_secs(60))
+        {
+            return Err(format!(
+                "heartbeat_interval ({interval:?}) must be between 1s and 60s",
+            ));
+        }
+
+        Ok(())
+    }
 }
 
 /// Wraps a [tokio::runtime::Builder] to allow layering multiple on_thread_start functions
@@ -268,13 +302,13 @@ impl CoreRuntime {
     /// If a tokio runtime has already been initialized. To re-use an existing runtime, call
     /// [CoreRuntime::new_assume_tokio].
     pub fn new<F>(
-        telemetry_options: TelemetryOptions,
+        runtime_options: RuntimeOptions,
         mut tokio_builder: TokioRuntimeBuilder<F>,
     ) -> Result<Self, anyhow::Error>
     where
         F: Fn() + Send + Sync + 'static,
     {
-        let telemetry = telemetry_init(telemetry_options)?;
+        let telemetry = telemetry_init(runtime_options.telemetry_options)?;
         let subscriber = telemetry.trace_subscriber();
         let runtime = tokio_builder
             .inner
@@ -289,7 +323,8 @@ impl CoreRuntime {
             })
             .build()?;
         let _rg = runtime.enter();
-        let mut me = Self::new_assume_tokio_initialized_telem(telemetry);
+        let mut me =
+            Self::new_assume_tokio_initialized_telem(telemetry, runtime_options.heartbeat_interval);
         me.runtime = Some(runtime);
         Ok(me)
     }
@@ -299,9 +334,12 @@ impl CoreRuntime {
     ///
     /// # Panics
     /// If there is no currently active Tokio runtime
-    pub fn new_assume_tokio(telemetry_options: TelemetryOptions) -> Result<Self, anyhow::Error> {
-        let telemetry = telemetry_init(telemetry_options)?;
-        Ok(Self::new_assume_tokio_initialized_telem(telemetry))
+    pub fn new_assume_tokio(runtime_options: RuntimeOptions) -> Result<Self, anyhow::Error> {
+        let telemetry = telemetry_init(runtime_options.telemetry_options)?;
+        Ok(Self::new_assume_tokio_initialized_telem(
+            telemetry,
+            runtime_options.heartbeat_interval,
+        ))
     }
 
     /// Construct a runtime from an already-initialized telemetry instance, assuming a tokio runtime
@@ -309,7 +347,10 @@ impl CoreRuntime {
     ///
     /// # Panics
     /// If there is no currently active Tokio runtime
-    pub fn new_assume_tokio_initialized_telem(telemetry: TelemetryInstance) -> Self {
+    pub fn new_assume_tokio_initialized_telem(
+        telemetry: TelemetryInstance,
+        heartbeat_interval: Option<Duration>,
+    ) -> Self {
         let runtime_handle = tokio::runtime::Handle::current();
         if let Some(sub) = telemetry.trace_subscriber() {
             set_trace_subscriber_for_current_thread(sub);
@@ -318,6 +359,7 @@ impl CoreRuntime {
             telemetry,
             runtime: None,
             runtime_handle,
+            heartbeat_interval,
         }
     }
 

package/sdk-core/{core → crates/sdk-core}/src/pollers/mod.rs

@@ -3,9 +3,9 @@ mod poll_buffer;
 pub(crate) use poll_buffer::{
     ActivityTaskOptions, LongPollBuffer, WorkflowTaskOptions, WorkflowTaskPoller,
 };
-pub use temporal_client::{
-    Client, ClientOptions, ClientOptionsBuilder, ClientTlsConfig, RetryClient, RetryConfig,
-    TlsConfig, WorkflowClientTrait,
+pub use temporalio_client::{
+    Client, ClientOptions, ClientTlsOptions, RetryClient, RetryOptions, TlsOptions,
+    WorkflowClientTrait,
 };
 
 use crate::{
@@ -15,9 +15,11 @@ use crate::{
 use anyhow::{anyhow, bail};
 use futures_util::{Stream, stream};
 use std::{fmt::Debug, marker::PhantomData};
-use temporal_sdk_core_api::worker::{ActivitySlotKind, NexusSlotKind, SlotKind, WorkflowSlotKind};
-use temporal_sdk_core_protos::temporal::api::workflowservice::v1::{
-    PollActivityTaskQueueResponse, PollNexusTaskQueueResponse, PollWorkflowTaskQueueResponse,
+use temporalio_common::{
+    protos::temporal::api::workflowservice::v1::{
+        PollActivityTaskQueueResponse, PollNexusTaskQueueResponse, PollWorkflowTaskQueueResponse,
+    },
+    worker::{ActivitySlotKind, NexusSlotKind, SlotKind, WorkflowSlotKind},
 };
 use tokio::select;
 use tokio_util::sync::CancellationToken;