@temporalio/core-bridge 0.20.2 → 0.22.0

package/sdk-core/core/src/workflow/workflow_tasks/mod.rs

@@ -23,6 +23,7 @@ use futures::FutureExt;
 use parking_lot::Mutex;
 use std::{
     fmt::Debug,
+    future::Future,
     ops::Add,
     sync::Arc,
     time::{Duration, Instant},
@@ -57,7 +58,7 @@ pub struct WorkflowTaskManager {
     pending_activations: PendingActivations,
     /// Holds activations which are purely query activations needed to respond to legacy queries.
     /// Activations may only be added here for runs which do not have other pending activations.
-    pending_legacy_queries: SegQueue<WorkflowActivation>,
+    pending_queries: SegQueue<WorkflowActivation>,
     /// Holds poll wft responses from the server that need to be applied
     ready_buffered_wft: SegQueue<ValidPollWFTQResponse>,
     /// Used to wake blocked workflow task polling
@@ -74,9 +75,8 @@ pub struct WorkflowTaskManager {
 #[derive(Clone, Debug)]
 pub(crate) struct OutstandingTask {
     pub info: WorkflowTaskInfo,
-    /// If set the outstanding task has query from the old `query` field which must be fulfilled
-    /// upon finishing replay
-    pub legacy_query: Option<QueryWorkflow>,
+    /// Set if the outstanding task has quer(ies) which must be fulfilled upon finishing replay
+    pub pending_queries: Vec<QueryWorkflow>,
     start_time: Instant,
 }
 
@@ -86,17 +86,29 @@ pub(crate) enum OutstandingActivation {
     Normal {
         /// True if there is an eviction in the joblist
         contains_eviction: bool,
+        /// Number of jobs in the activation
+        num_jobs: usize,
     },
     /// An activation for a legacy query
     LegacyQuery,
 }
 
 impl OutstandingActivation {
+    const fn has_only_eviction(self) -> bool {
+        matches!(
+            self,
+            OutstandingActivation::Normal {
+                contains_eviction: true,
+                num_jobs: nj
+            }
+        if nj == 1)
+    }
     const fn has_eviction(self) -> bool {
         matches!(
             self,
             OutstandingActivation::Normal {
-                contains_eviction: true
+                contains_eviction: true,
+                ..
             }
         )
     }
@@ -152,9 +164,9 @@ pub(crate) enum ActivationAction {
 
 #[derive(Debug, Eq, PartialEq, Hash)]
 pub(crate) enum EvictionRequestResult {
-    EvictionIssued(Option<u32>),
+    EvictionRequested(Option<u32>),
     NotFound,
-    EvictionAlreadyOutstanding,
+    EvictionAlreadyRequested(Option<u32>),
 }
 
 macro_rules! machine_mut {
@@ -179,7 +191,7 @@ impl WorkflowTaskManager {
         Self {
             workflow_machines: WorkflowConcurrencyManager::new(),
             pending_activations: Default::default(),
-            pending_legacy_queries: Default::default(),
+            pending_queries: Default::default(),
             ready_buffered_wft: Default::default(),
             pending_activations_notifier,
             cache_manager: Mutex::new(WorkflowCacheManager::new(eviction_policy, metrics.clone())),
@@ -187,9 +199,69 @@ impl WorkflowTaskManager {
         }
     }
 
+    /// Returns number of currently cached workflows
+    pub fn cached_workflows(&self) -> usize {
+        self.workflow_machines.cached_workflows()
+    }
+
+    /// Resolves once there is either capacity in the cache, or there are no pending evictions.
+    /// Inversely: Waits while there are pending evictions and the cache is full.
+    /// Waiting while there are no pending evictions must be avoided because it would block forever,
+    /// since there is no way for the cache size to be reduced.
+    pub fn wait_for_cache_capacity(&self) -> Option<impl Future<Output = ()> + '_> {
+        let are_no_pending_evictions = || {
+            !self.pending_activations.is_some_eviction()
+                && !self.workflow_machines.are_outstanding_evictions()
+        };
+        if !are_no_pending_evictions() {
+            let wait_fut = {
+                self.cache_manager
+                    .lock()
+                    .wait_for_capacity(are_no_pending_evictions)?
+            };
+            return Some(wait_fut);
+        }
+        None
+    }
+
+    /// Add a new run (as just received from polling) to the cache. If doing so would overflow the
+    /// cache, an eviction is queued to make room and the passed-in task is buffered and `None` is
+    /// returned.
+    ///
+    /// If the task is for a run already in the cache, the poll response is returned right away
+    /// and should be issued.
+    pub async fn add_new_run_to_cache(
+        &self,
+        poll_resp: ValidPollWFTQResponse,
+    ) -> Option<ValidPollWFTQResponse> {
+        let run_id = &poll_resp.workflow_execution.run_id;
+        let maybe_evicted = self.cache_manager.lock().insert(run_id);
+
+        if let Some(evicted_run_id) = maybe_evicted {
+            self.request_eviction(
+                &evicted_run_id,
+                "Workflow cache full",
+                EvictionReason::CacheFull,
+            );
+            debug!(run_id=%poll_resp.workflow_execution.run_id,
+                   "Received a WFT for a new run while at the cache limit. Buffering the task.");
+            // Buffer the task
+            if let Some(not_buffered) = self
+                .workflow_machines
+                .buffer_resp_if_outstanding_work(poll_resp)
+            {
+                self.make_buffered_poll_ready(not_buffered);
+            }
+
+            return None;
+        }
+
+        Some(poll_resp)
+    }
+
     pub(crate) fn next_pending_activation(&self) -> Option<WorkflowActivation> {
-        // Dispatch pending legacy queries first
-        if let leg_q @ Some(_) = self.pending_legacy_queries.pop() {
+        // Dispatch pending queries first
+        if let leg_q @ Some(_) = self.pending_queries.pop() {
             return leg_q;
         }
         // It is important that we do not issue pending activations for any workflows which already
@@ -205,15 +277,24 @@ impl WorkflowTaskManager {
                 .workflow_machines
                 .access_sync(&pending_info.run_id, |wfm| wfm.machines.get_wf_activation())
                 .and_then(|mut act| {
-                    if let Some(reason) = pending_info.needs_eviction {
-                        act.append_evict_job(reason);
+                    // Only evict workflows after all other pending work is complete.
+                    if act.jobs.is_empty() {
+                        if let Some(reason) = pending_info.needs_eviction {
+                            act.append_evict_job(reason);
+                        }
+                    }
+                    if !act.jobs.is_empty() {
+                        self.insert_outstanding_activation(&act)?;
+                        self.cache_manager.lock().touch(&act.run_id);
+                        Ok(Some(act))
+                    } else {
+                        // If for whatever reason we triggered a pending activation but there wasn't
+                        // actually any work to be done, just ignore that.
+                        Ok(None)
                     }
-                    self.insert_outstanding_activation(&act)?;
-                    Ok(act)
                 })
             {
-                self.cache_manager.lock().touch(&act.run_id);
-                Some(act)
+                act
             } else {
                 self.request_eviction(
                     &pending_info.run_id,
@@ -256,6 +337,10 @@ impl WorkflowTaskManager {
         reason: EvictionReason,
     ) -> EvictionRequestResult {
         if self.workflow_machines.exists(run_id) {
+            let attempts = self
+                .workflow_machines
+                .get_task(run_id)
+                .map(|wt| wt.info.attempt);
             if !self.activation_has_eviction(run_id) {
                 let message = message.into();
                 debug!(%run_id, %message, "Eviction requested");
@@ -263,13 +348,9 @@ impl WorkflowTaskManager {
                 self.pending_activations
                     .notify_needs_eviction(run_id, message, reason);
                 self.pending_activations_notifier.notify_waiters();
-                EvictionRequestResult::EvictionIssued(
-                    self.workflow_machines
-                        .get_task(run_id)
-                        .map(|wt| wt.info.attempt),
-                )
+                EvictionRequestResult::EvictionRequested(attempts)
             } else {
-                EvictionRequestResult::EvictionAlreadyOutstanding
+                EvictionRequestResult::EvictionAlreadyRequested(attempts)
             }
         } else {
             warn!(%run_id, "Eviction requested for unknown run");
@@ -277,11 +358,8 @@ impl WorkflowTaskManager {
         }
     }
 
-    /// Evict a workflow from the cache by its run id and enqueue a pending activation to evict the
-    /// workflow. Any existing pending activations will be destroyed, and any outstanding
-    /// activations invalidated.
-    ///
-    /// Returns that workflow's task info if it was present.
+    /// Evict a workflow from the cache by its run id. Any existing pending activations will be
+    /// destroyed, and any outstanding activations invalidated.
     fn evict_run(&self, run_id: &str) {
         debug!(run_id=%run_id, "Evicting run");
 
@@ -333,33 +411,45 @@ impl WorkflowTaskManager {
             .take()
             .map(|q| query_to_job(LEGACY_QUERY_ID.to_string(), q));
 
-        let (info, mut next_activation) =
+        let (info, mut next_activation, mut pending_queries) =
             match self.instantiate_or_update_workflow(work, client).await {
-                Ok((info, next_activation)) => (info, next_activation),
+                Ok(res) => res,
                 Err(e) => {
                     return NewWfTaskOutcome::Evict(e);
                 }
             };
 
+        if !pending_queries.is_empty() && legacy_query.is_some() {
+            error!(
+                "Server issued both normal and legacy queries. This should not happen. Please \
+                 file a bug report."
+            );
+            return NewWfTaskOutcome::Evict(WorkflowUpdateError {
+                source: WFMachinesError::Fatal(
+                    "Server issued both normal and legacy query".to_string(),
+                ),
+                run_id: next_activation.run_id,
+            });
+        }
+
         // Immediately dispatch query activation if no other jobs
-        let legacy_query = if next_activation.jobs.is_empty() {
-            if let Some(lq) = legacy_query {
+        if let Some(lq) = legacy_query {
+            if next_activation.jobs.is_empty() {
                 debug!("Dispatching legacy query {}", &lq);
                 next_activation
                     .jobs
                     .push(workflow_activation_job::Variant::QueryWorkflow(lq).into());
+            } else {
+                pending_queries.push(lq);
             }
-            None
-        } else {
-            legacy_query
-        };
+        }
 
         self.workflow_machines
             .insert_wft(
                 &next_activation.run_id,
                 OutstandingTask {
                     info,
-                    legacy_query,
+                    pending_queries,
                     start_time: task_start_time,
                 },
             )
@@ -396,20 +486,23 @@ impl WorkflowTaskManager {
         mut commands: Vec<WFCommand>,
         local_activity_request_sink: impl FnOnce(Vec<LocalActRequest>) -> Vec<LocalActivityResolution>,
     ) -> Result<Option<ServerCommandsWithWorkflowInfo>, WorkflowUpdateError> {
-        // No-command replies to evictions can simply skip everything
-        if commands.is_empty() && self.activation_has_eviction(run_id) {
-            return Ok(None);
-        }
-
-        let (task_token, is_leg_query_task, start_time) =
+        // There used to be code here that would return right away if the run reply had no commands
+        // and the activation that was just completed only had an eviction in it. That was bad
+        // because we wouldn't have yet sent any previously buffered commands since there was a
+        // pending activation (the eviction) and then we would *skip* doing anything with them here,
+        // because there were no new commands. In general it seems best to avoid short-circuiting
+        // here.
+
+        let activation_was_only_eviction = self.activation_has_only_eviction(run_id);
+        let (task_token, has_pending_query, start_time) =
             if let Some(entry) = self.workflow_machines.get_task(run_id) {
                 (
                     entry.info.task_token.clone(),
-                    entry.legacy_query.is_some(),
+                    !entry.pending_queries.is_empty(),
                     entry.start_time,
                 )
             } else {
-                if !self.activation_has_eviction(run_id) {
+                if !activation_was_only_eviction {
                     // Don't bother warning if this was an eviction, since it's normal to issue
                     // eviction activations without an associated workflow task in that case.
                     warn!(
@@ -458,6 +551,7 @@ impl WorkflowTaskManager {
                 }
             }
 
+            let activation_was_eviction = self.activation_has_eviction(run_id);
             let (are_pending, server_cmds, local_activities, wft_timeout) = machine_mut!(
                 self,
                 run_id,
@@ -466,7 +560,13 @@ impl WorkflowTaskManager {
                         // Send commands from lang into the machines then check if the workflow run
                         // needs another activation and mark it if so
                         wfm.push_commands(commands).await?;
-                        let are_pending = wfm.apply_next_task_if_ready().await?;
+                        // Don't bother applying the next task if we're evicting at the end of
+                        // this activation
+                        let are_pending = if !activation_was_eviction {
+                            wfm.apply_next_task_if_ready().await?
+                        } else {
+                            false
+                        };
                         // We want to fetch the outgoing commands only after a next WFT may have
                         // been applied, as outgoing server commands may be affected.
                         let outgoing_cmds = wfm.get_server_commands();
@@ -506,36 +606,34 @@ impl WorkflowTaskManager {
             let must_heartbeat = self
                 .wait_for_local_acts_or_heartbeat(run_id, wft_heartbeat_deadline)
                 .await;
-            let is_query_playback = is_leg_query_task && query_responses.is_empty();
+            let has_query_responses = !query_responses.is_empty();
+            let is_query_playback = has_pending_query && !has_query_responses;
 
             // We only actually want to send commands back to the server if there are no more
             // pending activations and we are caught up on replay. We don't want to complete a wft
             // if we already saw the final event in the workflow, or if we are playing back for the
-            // express purpose of fulfilling a query
-            if !self.pending_activations.has_pending(run_id)
-                && !server_cmds.replaying
-                && !is_query_playback
-            {
-                Some(ServerCommandsWithWorkflowInfo {
-                    task_token,
-                    action: ActivationAction::WftComplete {
-                        // TODO: Don't force if also sending complete execution cmd
-                        force_new_wft: must_heartbeat,
-                        commands: server_cmds.commands,
-                        query_responses,
-                    },
-                })
-            } else if query_responses.is_empty() {
-                None
+            // express purpose of fulfilling a query. If the activation we sent was *only* an
+            // eviction, and there were no commands produced during iteration, don't send that
+            // either.
+            let no_commands_and_evicting =
+                server_cmds.commands.is_empty() && activation_was_only_eviction;
+            let to_be_sent = ServerCommandsWithWorkflowInfo {
+                task_token,
+                action: ActivationAction::WftComplete {
+                    // TODO: Don't force if also sending complete execution cmd
+                    force_new_wft: must_heartbeat,
+                    commands: server_cmds.commands,
+                    query_responses,
+                },
+            };
+            let should_respond = !(self.pending_activations.has_pending(run_id)
+                || server_cmds.replaying
+                || is_query_playback
+                || no_commands_and_evicting);
+            if should_respond || has_query_responses {
+                Some(to_be_sent)
             } else {
-                Some(ServerCommandsWithWorkflowInfo {
-                    task_token,
-                    action: ActivationAction::WftComplete {
-                        commands: vec![],
-                        query_responses,
-                        force_new_wft: false,
-                    },
-                })
+                None
             }
         };
         Ok(ret)
@@ -573,7 +671,8 @@ impl WorkflowTaskManager {
         } else {
             // Blow up any cached data associated with the workflow
             let should_report = match self.request_eviction(run_id, failstr, reason) {
-                EvictionRequestResult::EvictionIssued(Some(attempt)) => attempt <= 1,
+                EvictionRequestResult::EvictionRequested(Some(attempt))
+                | EvictionRequestResult::EvictionAlreadyRequested(Some(attempt)) => attempt <= 1,
                 _ => false,
             };
             if should_report {
@@ -592,7 +691,8 @@ impl WorkflowTaskManager {
         &self,
         poll_wf_resp: ValidPollWFTQResponse,
         client: Arc<WorkerClientBag>,
-    ) -> Result<(WorkflowTaskInfo, WorkflowActivation), WorkflowUpdateError> {
+    ) -> Result<(WorkflowTaskInfo, WorkflowActivation, Vec<QueryWorkflow>), WorkflowUpdateError>
+    {
         let run_id = poll_wf_resp.workflow_execution.run_id.clone();
 
         let wft_info = WorkflowTaskInfo {
@@ -606,11 +706,16 @@ impl WorkflowTaskManager {
             .get(0)
             .map(|ev| ev.event_id > 1)
             .unwrap_or_default();
+        let poll_resp_is_incremental =
+            poll_resp_is_incremental || poll_wf_resp.history.events.is_empty();
+
+        let mut did_miss_cache = !poll_resp_is_incremental;
 
         let page_token = if !self.workflow_machines.exists(&run_id) && poll_resp_is_incremental {
             debug!(run_id=?run_id, "Workflow task has partial history, but workflow is not in \
                    cache. Will fetch history");
             self.metrics.sticky_cache_miss();
+            did_miss_cache = true;
             NextPageToken::FetchFromStart
         } else {
             poll_wf_resp.next_page_token.into()
@@ -639,16 +744,26 @@ impl WorkflowTaskManager {
             .await
         {
             Ok(mut activation) => {
-                // If there are in-poll queries, insert jobs for those queries into the activation
+                // If there are in-poll queries, insert jobs for those queries into the activation,
+                // but only if we hit the cache. If we didn't, those queries will need to be dealt
+                // with once replay is over
+                let mut pending_queries = vec![];
                 if !poll_wf_resp.query_requests.is_empty() {
-                    let query_jobs = poll_wf_resp
-                        .query_requests
-                        .into_iter()
-                        .map(|q| workflow_activation_job::Variant::QueryWorkflow(q).into());
-                    activation.jobs.extend(query_jobs);
+                    if !did_miss_cache {
+                        let query_jobs = poll_wf_resp
+                            .query_requests
+                            .into_iter()
+                            .map(|q| workflow_activation_job::Variant::QueryWorkflow(q).into());
+                        activation.jobs.extend(query_jobs);
+                    } else {
+                        poll_wf_resp
+                            .query_requests
+                            .into_iter()
+                            .for_each(|q| pending_queries.push(q));
+                    }
                 }
 
-                Ok((wft_info, activation))
+                Ok((wft_info, activation, pending_queries))
             }
             Err(source) => Err(WorkflowUpdateError { source, run_id }),
         }
@@ -660,12 +775,14 @@ impl WorkflowTaskManager {
     /// eviction, which could be avoided if this is called too early.
     ///
     /// Returns true if WFT was marked completed internally
-    pub(crate) fn after_wft_report(&self, run_id: &str, did_complete_wft: bool) -> bool {
+    pub(crate) fn after_wft_report(&self, run_id: &str, reported_wft_to_server: bool) -> bool {
         let mut just_evicted = false;
 
-        if let Some(OutstandingActivation::Normal {
-            contains_eviction: true,
-        }) = self.workflow_machines.get_activation(run_id)
+        if self
+            .workflow_machines
+            .get_activation(run_id)
+            .map(|a| a.has_eviction())
+            .unwrap_or_default()
         {
             self.evict_run(run_id);
             just_evicted = true;
@@ -673,50 +790,48 @@ impl WorkflowTaskManager {
 
         // Workflows with no more pending activations (IE: They have completed a WFT) must be
         // removed from the outstanding tasks map
-        let retme = if !self.pending_activations.has_pending(run_id) {
-            if !just_evicted {
-                // Check if there was a legacy query which must be fulfilled, and if there is create
-                // a new pending activation for it.
-                if let Some(ref mut ot) = &mut *self
-                    .workflow_machines
-                    .get_task_mut(run_id)
-                    .expect("Machine must exist")
-                {
-                    if let Some(query) = ot.legacy_query.take() {
+        if !self.pending_activations.has_pending(run_id) && !just_evicted {
+            if let Some(ref mut ot) = &mut *self
+                .workflow_machines
+                .get_task_mut(run_id)
+                .expect("Machine must exist")
+            {
+                // Check if there was a pending query which must be fulfilled, and if there is
+                // create a new pending activation for it.
+                if !ot.pending_queries.is_empty() {
+                    for query in ot.pending_queries.drain(..) {
                         let na = create_query_activation(run_id.to_string(), [query]);
-                        self.pending_legacy_queries.push(na);
-                        self.pending_activations_notifier.notify_waiters();
-                        return false;
+                        self.pending_queries.push(na);
                     }
+                    self.pending_activations_notifier.notify_waiters();
+                    return false;
                 }
+            }
 
-                // Evict run id if cache is full. Non-sticky will always evict.
-                let maybe_evicted = self.cache_manager.lock().insert(run_id);
-                if let Some(evicted_run_id) = maybe_evicted {
-                    self.request_eviction(
-                        &evicted_run_id,
-                        "Workflow cache full",
-                        EvictionReason::CacheFull,
-                    );
-                }
+            // Evict run id if cache is full. Non-sticky will always evict.
+            let maybe_evicted = self.cache_manager.lock().insert(run_id);
+            if let Some(evicted_run_id) = maybe_evicted {
+                self.request_eviction(
+                    &evicted_run_id,
+                    "Workflow cache full",
+                    EvictionReason::CacheFull,
+                );
+            }
 
-                // If there was a buffered poll response from the server, it is now ready to
-                // be handled.
-                if let Some(buffd) = self.workflow_machines.take_buffered_poll(run_id) {
-                    self.make_buffered_poll_ready(buffd);
-                }
+            // If there was a buffered poll response from the server, it is now ready to
+            // be handled.
+            if let Some(buffd) = self.workflow_machines.take_buffered_poll(run_id) {
+                self.make_buffered_poll_ready(buffd);
             }
+        }
 
-            // The evict may or may not have already done this, but even when we aren't evicting
-            // we want to clear the outstanding workflow task since it's now complete.
-            self.workflow_machines
-                .complete_wft(run_id, did_complete_wft)
-                .is_some()
-        } else {
-            false
-        };
+        // If we reported to server, we always want to mark it complete.
+        let wft_marked_complete = self
+            .workflow_machines
+            .complete_wft(run_id, reported_wft_to_server)
+            .is_some();
         self.on_activation_done(run_id);
-        retme
+        wft_marked_complete
     }
 
     /// Must be called after *every* activation is replied to, regardless of whether or not we
@@ -725,10 +840,11 @@ impl WorkflowTaskManager {
     ///
     /// Any subsequent action that needs to be taken will be created as a new activation
     fn on_activation_done(&self, run_id: &str) {
-        if self.workflow_machines.delete_activation(run_id).is_some() {
-            self.pending_activations_notifier.notify_waiters();
-        }
-        // It's possible the activation is already removed due to completing an eviction
+        self.workflow_machines.delete_activation(run_id);
+        // It's important to use `notify_one` here to avoid possible races where we're waiting
+        // on a cache slot and fail to realize pending activations must be issued before a slot
+        // will free up.
+        self.pending_activations_notifier.notify_one();
     }
 
     /// Let a workflow know that something we've been waiting locally on has resolved, like a local
@@ -739,7 +855,8 @@ impl WorkflowTaskManager {
         run_id: &str,
         resolved: LocalResolution,
     ) -> Result<(), WorkflowUpdateError> {
-        self.workflow_machines
+        let result_was_important = self
+            .workflow_machines
             .access_sync(run_id, |wfm: &mut WorkflowManager| {
                 wfm.notify_of_local_result(resolved)
             })?
@@ -748,7 +865,9 @@ impl WorkflowTaskManager {
                 run_id: run_id.to_string(),
             })?;
 
-        self.needs_activation(run_id);
+        if result_was_important {
+            self.needs_activation(run_id);
+        }
         Ok(())
     }
 
@@ -765,6 +884,7 @@ impl WorkflowTaskManager {
         } else {
             OutstandingActivation::Normal {
                 contains_eviction: act.eviction_index().is_some(),
+                num_jobs: act.jobs.len(),
             }
         };
         match self
@@ -785,6 +905,13 @@ impl WorkflowTaskManager {
         }
     }
 
+    fn activation_has_only_eviction(&self, run_id: &str) -> bool {
+        self.workflow_machines
+            .get_activation(run_id)
+            .map(OutstandingActivation::has_only_eviction)
+            .unwrap_or_default()
+    }
+
     fn activation_has_eviction(&self, run_id: &str) -> bool {
         self.workflow_machines
             .get_activation(run_id)

package/sdk-core/core-api/src/worker.rs

@@ -20,6 +20,8 @@ pub struct WorkerConfig {
     /// The maximum allowed number of workflow tasks that will ever be given to this worker at one
     /// time. Note that one workflow task may require multiple activations - so the WFT counts as
     /// "outstanding" until all activations it requires have been completed.
+    ///
+    /// Cannot be larger than `max_cached_workflows`.
     #[builder(default = "100")]
     pub max_outstanding_workflow_tasks: usize,
     /// The maximum number of activity tasks that will ever be given to this worker concurrently
@@ -90,6 +92,13 @@ impl WorkerConfigBuilder {
         if self.max_concurrent_wft_polls == Some(0) {
             return Err("`max_concurrent_wft_polls` must be at least 1".to_owned());
         }
+        if self.max_outstanding_workflow_tasks > self.max_cached_workflows {
+            return Err(
+                "Maximum concurrent workflow tasks cannot exceed the maximum number of cached \
+                 workflows"
+                    .to_owned(),
+            );
+        }
         Ok(())
     }
 }

package/sdk-core/sdk/Cargo.toml

@@ -13,6 +13,7 @@ categories = ["development-tools"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-trait = "0.1"
 anyhow = "1.0"
 base64 = "0.13"
 crossbeam = "0.8"