@temporalio/core-bridge 0.20.1 → 0.21.1

package/sdk-core/core/src/workflow/workflow_tasks/mod.rs

@@ -57,7 +57,7 @@ pub struct WorkflowTaskManager {
     pending_activations: PendingActivations,
     /// Holds activations which are purely query activations needed to respond to legacy queries.
     /// Activations may only be added here for runs which do not have other pending activations.
-    pending_legacy_queries: SegQueue<WorkflowActivation>,
+    pending_queries: SegQueue<WorkflowActivation>,
     /// Holds poll wft responses from the server that need to be applied
     ready_buffered_wft: SegQueue<ValidPollWFTQResponse>,
     /// Used to wake blocked workflow task polling
@@ -74,9 +74,8 @@ pub struct WorkflowTaskManager {
 #[derive(Clone, Debug)]
 pub(crate) struct OutstandingTask {
     pub info: WorkflowTaskInfo,
-    /// If set the outstanding task has query from the old `query` field which must be fulfilled
-    /// upon finishing replay
-    pub legacy_query: Option<QueryWorkflow>,
+    /// Set if the outstanding task has quer(ies) which must be fulfilled upon finishing replay
+    pub pending_queries: Vec<QueryWorkflow>,
     start_time: Instant,
 }
 
@@ -150,6 +149,13 @@ pub(crate) enum ActivationAction {
     RespondLegacyQuery { result: QueryResult },
 }
 
+#[derive(Debug, Eq, PartialEq, Hash)]
+pub(crate) enum EvictionRequestResult {
+    EvictionIssued(Option<u32>),
+    NotFound,
+    EvictionAlreadyOutstanding,
+}
+
 macro_rules! machine_mut {
     ($myself:ident, $run_id:ident, $clos:expr) => {{
         $myself
@@ -172,7 +178,7 @@ impl WorkflowTaskManager {
         Self {
             workflow_machines: WorkflowConcurrencyManager::new(),
             pending_activations: Default::default(),
-            pending_legacy_queries: Default::default(),
+            pending_queries: Default::default(),
             ready_buffered_wft: Default::default(),
             pending_activations_notifier,
             cache_manager: Mutex::new(WorkflowCacheManager::new(eviction_policy, metrics.clone())),
@@ -181,8 +187,8 @@ impl WorkflowTaskManager {
     }
 
     pub(crate) fn next_pending_activation(&self) -> Option<WorkflowActivation> {
-        // Dispatch pending legacy queries first
-        if let leg_q @ Some(_) = self.pending_legacy_queries.pop() {
+        // Dispatch pending queries first
+        if let leg_q @ Some(_) = self.pending_queries.pop() {
             return leg_q;
         }
         // It is important that we do not issue pending activations for any workflows which already
@@ -201,12 +207,18 @@ impl WorkflowTaskManager {
                     if let Some(reason) = pending_info.needs_eviction {
                         act.append_evict_job(reason);
                     }
-                    self.insert_outstanding_activation(&act)?;
-                    Ok(act)
+                    // If for whatever reason we triggered a pending activation but there wasn't
+                    // actually any work to be done, just ignore that.
+                    if !act.jobs.is_empty() {
+                        self.insert_outstanding_activation(&act)?;
+                        self.cache_manager.lock().touch(&act.run_id);
+                        Ok(Some(act))
+                    } else {
+                        Ok(None)
+                    }
                 })
             {
-                self.cache_manager.lock().touch(&act.run_id);
-                Some(act)
+                act
             } else {
                 self.request_eviction(
                     &pending_info.run_id,
@@ -247,7 +259,7 @@ impl WorkflowTaskManager {
         run_id: &str,
         message: impl Into<String>,
         reason: EvictionReason,
-    ) -> Option<u32> {
+    ) -> EvictionRequestResult {
         if self.workflow_machines.exists(run_id) {
             if !self.activation_has_eviction(run_id) {
                 let message = message.into();
@@ -256,13 +268,17 @@ impl WorkflowTaskManager {
                 self.pending_activations
                     .notify_needs_eviction(run_id, message, reason);
                 self.pending_activations_notifier.notify_waiters();
+                EvictionRequestResult::EvictionIssued(
+                    self.workflow_machines
+                        .get_task(run_id)
+                        .map(|wt| wt.info.attempt),
+                )
+            } else {
+                EvictionRequestResult::EvictionAlreadyOutstanding
             }
-            self.workflow_machines
-                .get_task(run_id)
-                .map(|wt| wt.info.attempt)
         } else {
             warn!(%run_id, "Eviction requested for unknown run");
-            None
+            EvictionRequestResult::NotFound
         }
     }
 
@@ -304,9 +320,11 @@ impl WorkflowTaskManager {
             return NewWfTaskOutcome::TaskBuffered;
         };
 
+        let start_event_id = work.history.events.first().map(|e| e.event_id);
         debug!(
             task_token = %&work.task_token,
             history_length = %work.history.events.len(),
+            start_event_id = ?start_event_id,
             attempt = %work.attempt,
             run_id = %work.workflow_execution.run_id,
             "Applying new workflow task from server"
@@ -320,33 +338,45 @@ impl WorkflowTaskManager {
             .take()
             .map(|q| query_to_job(LEGACY_QUERY_ID.to_string(), q));
 
-        let (info, mut next_activation) =
+        let (info, mut next_activation, mut pending_queries) =
             match self.instantiate_or_update_workflow(work, client).await {
-                Ok((info, next_activation)) => (info, next_activation),
+                Ok(res) => res,
                 Err(e) => {
                     return NewWfTaskOutcome::Evict(e);
                 }
             };
 
+        if !pending_queries.is_empty() && legacy_query.is_some() {
+            error!(
+                "Server issued both normal and legacy queries. This should not happen. Please \
+                 file a bug report."
+            );
+            return NewWfTaskOutcome::Evict(WorkflowUpdateError {
+                source: WFMachinesError::Fatal(
+                    "Server issued both normal and legacy query".to_string(),
+                ),
+                run_id: next_activation.run_id,
+            });
+        }
+
         // Immediately dispatch query activation if no other jobs
-        let legacy_query = if next_activation.jobs.is_empty() {
-            if let Some(lq) = legacy_query {
+        if let Some(lq) = legacy_query {
+            if next_activation.jobs.is_empty() {
                 debug!("Dispatching legacy query {}", &lq);
                 next_activation
                     .jobs
                     .push(workflow_activation_job::Variant::QueryWorkflow(lq).into());
+            } else {
+                pending_queries.push(lq);
             }
-            None
-        } else {
-            legacy_query
-        };
+        }
 
         self.workflow_machines
             .insert_wft(
                 &next_activation.run_id,
                 OutstandingTask {
                     info,
-                    legacy_query,
+                    pending_queries,
                     start_time: task_start_time,
                 },
             )
@@ -388,11 +418,11 @@ impl WorkflowTaskManager {
             return Ok(None);
         }
 
-        let (task_token, is_leg_query_task, start_time) =
+        let (task_token, has_pending_query, start_time) =
             if let Some(entry) = self.workflow_machines.get_task(run_id) {
                 (
                     entry.info.task_token.clone(),
-                    entry.legacy_query.is_some(),
+                    !entry.pending_queries.is_empty(),
                     entry.start_time,
                 )
             } else {
@@ -493,7 +523,7 @@ impl WorkflowTaskManager {
             let must_heartbeat = self
                 .wait_for_local_acts_or_heartbeat(run_id, wft_heartbeat_deadline)
                 .await;
-            let is_query_playback = is_leg_query_task && query_responses.is_empty();
+            let is_query_playback = has_pending_query && query_responses.is_empty();
 
             // We only actually want to send commands back to the server if there are no more
             // pending activations and we are caught up on replay. We don't want to complete a wft
@@ -559,9 +589,10 @@ impl WorkflowTaskManager {
             FailedActivationOutcome::ReportLegacyQueryFailure(tt)
         } else {
             // Blow up any cached data associated with the workflow
-            let should_report = self
-                .request_eviction(run_id, failstr, reason)
-                .map_or(true, |attempt| attempt <= 1);
+            let should_report = match self.request_eviction(run_id, failstr, reason) {
+                EvictionRequestResult::EvictionIssued(Some(attempt)) => attempt <= 1,
+                _ => false,
+            };
             if should_report {
                 FailedActivationOutcome::Report(tt)
             } else {
@@ -578,7 +609,8 @@ impl WorkflowTaskManager {
         &self,
         poll_wf_resp: ValidPollWFTQResponse,
         client: Arc<WorkerClientBag>,
-    ) -> Result<(WorkflowTaskInfo, WorkflowActivation), WorkflowUpdateError> {
+    ) -> Result<(WorkflowTaskInfo, WorkflowActivation, Vec<QueryWorkflow>), WorkflowUpdateError>
+    {
         let run_id = poll_wf_resp.workflow_execution.run_id.clone();
 
         let wft_info = WorkflowTaskInfo {
@@ -592,11 +624,16 @@ impl WorkflowTaskManager {
             .get(0)
             .map(|ev| ev.event_id > 1)
             .unwrap_or_default();
+        let poll_resp_is_incremental =
+            poll_resp_is_incremental || poll_wf_resp.history.events.is_empty();
+
+        let mut did_miss_cache = !poll_resp_is_incremental;
 
         let page_token = if !self.workflow_machines.exists(&run_id) && poll_resp_is_incremental {
             debug!(run_id=?run_id, "Workflow task has partial history, but workflow is not in \
                    cache. Will fetch history");
             self.metrics.sticky_cache_miss();
+            did_miss_cache = true;
             NextPageToken::FetchFromStart
         } else {
             poll_wf_resp.next_page_token.into()
@@ -625,16 +662,26 @@ impl WorkflowTaskManager {
             .await
         {
             Ok(mut activation) => {
-                // If there are in-poll queries, insert jobs for those queries into the activation
+                // If there are in-poll queries, insert jobs for those queries into the activation,
+                // but only if we hit the cache. If we didn't, those queries will need to be dealt
+                // with once replay is over
+                let mut pending_queries = vec![];
                 if !poll_wf_resp.query_requests.is_empty() {
-                    let query_jobs = poll_wf_resp
-                        .query_requests
-                        .into_iter()
-                        .map(|q| workflow_activation_job::Variant::QueryWorkflow(q).into());
-                    activation.jobs.extend(query_jobs);
+                    if !did_miss_cache {
+                        let query_jobs = poll_wf_resp
+                            .query_requests
+                            .into_iter()
+                            .map(|q| workflow_activation_job::Variant::QueryWorkflow(q).into());
+                        activation.jobs.extend(query_jobs);
+                    } else {
+                        poll_wf_resp
+                            .query_requests
+                            .into_iter()
+                            .for_each(|q| pending_queries.push(q));
+                    }
                 }
 
-                Ok((wft_info, activation))
+                Ok((wft_info, activation, pending_queries))
             }
             Err(source) => Err(WorkflowUpdateError { source, run_id }),
         }
@@ -661,16 +708,18 @@ impl WorkflowTaskManager {
         // removed from the outstanding tasks map
         let retme = if !self.pending_activations.has_pending(run_id) {
             if !just_evicted {
-                // Check if there was a legacy query which must be fulfilled, and if there is create
-                // a new pending activation for it.
+                // Check if there was a pending query which must be fulfilled, and if there is
+                // create a new pending activation for it.
                 if let Some(ref mut ot) = &mut *self
                     .workflow_machines
                     .get_task_mut(run_id)
                     .expect("Machine must exist")
                 {
-                    if let Some(query) = ot.legacy_query.take() {
-                        let na = create_query_activation(run_id.to_string(), [query]);
-                        self.pending_legacy_queries.push(na);
+                    if !ot.pending_queries.is_empty() {
+                        for query in ot.pending_queries.drain(..) {
+                            let na = create_query_activation(run_id.to_string(), [query]);
+                            self.pending_queries.push(na);
+                        }
                         self.pending_activations_notifier.notify_waiters();
                         return false;
                     }
@@ -725,7 +774,8 @@ impl WorkflowTaskManager {
         run_id: &str,
         resolved: LocalResolution,
     ) -> Result<(), WorkflowUpdateError> {
-        self.workflow_machines
+        let result_was_important = self
+            .workflow_machines
             .access_sync(run_id, |wfm: &mut WorkflowManager| {
                 wfm.notify_of_local_result(resolved)
             })?
@@ -734,7 +784,9 @@ impl WorkflowTaskManager {
                 run_id: run_id.to_string(),
             })?;
 
-        self.needs_activation(run_id);
+        if result_was_important {
+            self.needs_activation(run_id);
+        }
         Ok(())
     }
 

package/sdk-core/sdk/src/lib.rs

@@ -389,7 +389,7 @@ impl ActivityHalf {
                 tokio::spawn(ACT_CANCEL_TOK.scope(ct, async move {
                     let mut inputs = start.input;
                     let arg = inputs.pop().unwrap_or_default();
-                    let output = (&act_fn.act_func)(arg).await;
+                    let output = (act_fn.act_func)(arg).await;
                     let result = match output {
                         Ok(res) => ActivityExecutionResult::ok(res),
                         Err(err) => match err.downcast::<ActivityCancelledError>() {

package/sdk-core/sdk-core-protos/src/history_info.rs

@@ -117,17 +117,13 @@ impl HistoryInfo {
     /// Remove events from the beginning of this history such that it looks like what would've been
     /// delivered on a sticky queue where the previously started task was the one before the last
     /// task in this history.
-    ///
-    /// This is not *fully* accurate in that it will include commands that were part of the last
-    /// WFT completion, which the server would typically not include, but it's good enough for
-    /// testing.
     pub fn make_incremental(&mut self) {
         let last_complete_ix = self
             .events
             .iter()
             .rposition(|he| he.event_type() == EventType::WorkflowTaskCompleted)
             .expect("Must be a WFT completed event in history");
-        self.events.drain(0..=last_complete_ix);
+        self.events.drain(0..last_complete_ix);
     }
 
     pub fn events(&self) -> &[HistoryEvent] {
@@ -223,7 +219,7 @@ mod tests {
     fn incremental_works() {
         let t = single_timer("timer1");
         let hi = t.get_one_wft(2).unwrap();
-        assert_eq!(hi.events().len(), 4);
-        assert_eq!(hi.events()[0].event_id, 5);
+        assert_eq!(hi.events().len(), 5);
+        assert_eq!(hi.events()[0].event_id, 4);
     }
 }

package/sdk-core/tests/integ_tests/workflow_tests/continue_as_new.rs

@@ -1,5 +1,5 @@
 use std::time::Duration;
-use temporal_client::{WorkflowClientTrait, WorkflowOptions};
+use temporal_client::WorkflowOptions;
 use temporal_sdk::{WfContext, WfExitValue, WorkflowResult};
 use temporal_sdk_core_protos::coresdk::workflow_commands::ContinueAsNewWorkflowExecution;
 use temporal_sdk_core_test_utils::CoreWfStarter;
@@ -33,13 +33,31 @@ async fn continue_as_new_happy_path() {
         )
         .await
         .unwrap();
+    // The four additional runs
+    worker.incr_expected_run_count(4);
     worker.run_until_done().await.unwrap();
+}
 
-    // Terminate the continued workflow
-    starter
-        .get_client()
-        .await
-        .terminate_workflow_execution(wf_name.to_owned(), None)
-        .await
-        .unwrap();
+#[tokio::test]
+async fn continue_as_new_multiple_concurrent() {
+    let wf_name = "continue_as_new_multiple_concurrent";
+    let mut starter = CoreWfStarter::new(wf_name);
+    starter.max_cached_workflows(3).max_wft(3);
+    let mut worker = starter.worker().await;
+    worker.register_wf(wf_name.to_string(), continue_as_new_wf);
+
+    let wf_names = (1..=20).map(|i| format!("{}-{}", wf_name, i));
+    for name in wf_names.clone() {
+        worker
+            .submit_wf(
+                name.to_string(),
+                wf_name.to_string(),
+                vec![[1].into()],
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+    }
+    worker.incr_expected_run_count(20 * 4);
+    worker.run_until_done().await.unwrap();
 }

package/src/errors.rs

@@ -10,6 +10,8 @@ pub static SHUTDOWN_ERROR: OnceCell<Root<JsFunction>> = OnceCell::new();
 pub static NO_WORKER_ERROR: OnceCell<Root<JsFunction>> = OnceCell::new();
 /// Something unexpected happened, considered fatal
 pub static UNEXPECTED_ERROR: OnceCell<Root<JsFunction>> = OnceCell::new();
+/// Used in different parts of the project to signal that something unexpected has happened
+pub static ILLEGAL_STATE_ERROR: OnceCell<Root<JsFunction>> = OnceCell::new();
 
 static ALREADY_REGISTERED_ERRORS: OnceCell<bool> = OnceCell::new();
 
@@ -70,9 +72,9 @@ pub fn register_errors(mut cx: FunctionContext) -> JsResult<JsUndefined> {
     let res = ALREADY_REGISTERED_ERRORS.set(true);
     if res.is_err() {
         // Don't do anything if errors are already registered
-        return Ok(cx.undefined())
+        return Ok(cx.undefined());
     }
-    
+
     let mapping = cx.argument::<JsObject>(0)?;
     let shutdown_error = mapping
         .get(&mut cx, "ShutdownError")?
@@ -90,11 +92,16 @@ pub fn register_errors(mut cx: FunctionContext) -> JsResult<JsUndefined> {
         .get(&mut cx, "UnexpectedError")?
         .downcast_or_throw::<JsFunction, FunctionContext>(&mut cx)?
         .root(&mut cx);
+    let illegal_state_error = mapping
+        .get(&mut cx, "IllegalStateError")?
+        .downcast_or_throw::<JsFunction, FunctionContext>(&mut cx)?
+        .root(&mut cx);
 
     TRANSPORT_ERROR.get_or_try_init(|| Ok(transport_error))?;
     SHUTDOWN_ERROR.get_or_try_init(|| Ok(shutdown_error))?;
     NO_WORKER_ERROR.get_or_try_init(|| Ok(no_worker_error))?;
     UNEXPECTED_ERROR.get_or_try_init(|| Ok(unexpected_error))?;
+    ILLEGAL_STATE_ERROR.get_or_try_init(|| Ok(illegal_state_error))?;
 
     Ok(cx.undefined())
 }

package/src/lib.rs

@@ -8,6 +8,7 @@ use once_cell::sync::OnceCell;
 use opentelemetry::trace::{FutureExt, SpanContext, TraceContextExt};
 use prost::Message;
 use std::{
+    cell::RefCell,
     fmt::Display,
     future::Future,
     sync::Arc,
@@ -135,7 +136,7 @@ struct Client {
     core_client: Arc<RawClient>,
 }
 
-type BoxedClient = JsBox<Client>;
+type BoxedClient = JsBox<RefCell<Option<Client>>>;
 impl Finalize for Client {}
 
 /// Worker struct, hold a reference for the channel sender responsible for sending requests from
@@ -291,10 +292,10 @@ fn start_bridge_loop(event_queue: Arc<EventQueue>, receiver: &mut UnboundedRecei
                         }
                         Ok(client) => {
                             send_result(event_queue.clone(), callback, |cx| {
-                                Ok(cx.boxed(Client {
+                                Ok(cx.boxed(RefCell::new(Some(Client {
                                     runtime,
                                     core_client: Arc::new(client),
-                                }))
+                                }))))
                             });
                         }
                     }
@@ -590,15 +591,23 @@ fn worker_new(mut cx: FunctionContext) -> JsResult<JsUndefined> {
     let callback = cx.argument::<JsFunction>(2)?;
 
     let config = worker_options.as_worker_config(&mut cx)?;
-
-    let request = Request::InitWorker {
-        client: client.core_client.clone(),
-        runtime: client.runtime.clone(),
-        config,
-        callback: callback.root(&mut cx),
-    };
-    if let Err(err) = client.runtime.sender.send(request) {
-        callback_with_unexpected_error(&mut cx, callback, err)?;
+    match &*client.borrow() {
+        None => {
+            callback_with_error(&mut cx, callback, move |cx| {
+                UNEXPECTED_ERROR.from_string(cx, "Tried to use closed Client".to_string())
+            })?;
+        }
+        Some(client) => {
+            let request = Request::InitWorker {
+                client: client.core_client.clone(),
+                runtime: client.runtime.clone(),
+                config,
+                callback: callback.root(&mut cx),
+            };
+            if let Err(err) = client.runtime.sender.send(request) {
+                callback_with_unexpected_error(&mut cx, callback, err)?;
+            };
+        }
     };
 
     Ok(cx.undefined())
@@ -783,13 +792,26 @@ fn worker_record_activity_heartbeat(mut cx: FunctionContext) -> JsResult<JsUndef
 fn worker_shutdown(mut cx: FunctionContext) -> JsResult<JsUndefined> {
     let worker = cx.argument::<BoxedWorker>(0)?;
     let callback = cx.argument::<JsFunction>(1)?;
-    match worker.runtime.sender.send(Request::ShutdownWorker {
+    if let Err(err) = worker.runtime.sender.send(Request::ShutdownWorker {
         worker: worker.core_worker.clone(),
         callback: callback.root(&mut cx),
     }) {
-        Err(err) => cx.throw_error(format!("{}", err)),
-        _ => Ok(cx.undefined()),
-    }
+        UNEXPECTED_ERROR
+            .from_error(&mut cx, err)
+            .and_then(|err| cx.throw(err))?;
+    };
+    Ok(cx.undefined())
+}
+
+/// Drop a reference to a Client, once all references are dropped, the Client will be closed.
+fn client_close(mut cx: FunctionContext) -> JsResult<JsUndefined> {
+    let client = cx.argument::<BoxedClient>(0)?;
+    if client.replace(None).is_none() {
+        ILLEGAL_STATE_ERROR
+            .from_error(&mut cx, "Client already closed")
+            .and_then(|err| cx.throw(err))?;
+    };
+    Ok(cx.undefined())
 }
 
 /// Convert Rust SystemTime into a JS array with 2 numbers (seconds, nanos)
@@ -824,6 +846,7 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> {
     cx.export_function("newWorker", worker_new)?;
     cx.export_function("newReplayWorker", replay_worker_new)?;
     cx.export_function("workerShutdown", worker_shutdown)?;
+    cx.export_function("clientClose", client_close)?;
     cx.export_function("runtimeShutdown", runtime_shutdown)?;
     cx.export_function("pollLogs", poll_logs)?;
     cx.export_function(