@temporalio/core-bridge 0.16.0 → 0.17.1

package/sdk-core/src/workflow/workflow_tasks/mod.rs

@@ -21,7 +21,7 @@ use crate::{
 use crossbeam::queue::SegQueue;
 use futures::FutureExt;
 use parking_lot::Mutex;
-use std::{fmt::Debug, ops::DerefMut, time::Instant};
+use std::{fmt::Debug, time::Instant};
 use temporal_sdk_core_protos::coresdk::{
     workflow_activation::{
         create_evict_activation, create_query_activation, wf_activation_job, QueryWorkflow,
@@ -77,9 +77,9 @@ pub(crate) enum OutstandingActivation {
 }
 
 impl OutstandingActivation {
-    fn has_eviction(&self) -> bool {
+    const fn has_eviction(self) -> bool {
         matches!(
-            &self,
+            self,
             OutstandingActivation::Normal {
                 contains_eviction: true
             }
@@ -135,7 +135,7 @@ pub enum ActivationAction {
 }
 
 macro_rules! machine_mut {
-    ($myself:ident, $run_id:ident, $task_token:ident, $clos:expr) => {{
+    ($myself:ident, $run_id:ident, $clos:expr) => {{
         $myself
             .workflow_machines
             .access($run_id, $clos)
@@ -143,7 +143,6 @@ macro_rules! machine_mut {
             .map_err(|source| WorkflowUpdateError {
                 source,
                 run_id: $run_id.to_owned(),
-                task_token: Some($task_token.clone()),
             })
     }};
 }
@@ -256,6 +255,7 @@ impl WorkflowTaskManager {
         debug!(
             task_token = %&work.task_token,
             history_length = %work.history.events.len(),
+            attempt = %work.attempt,
             "Applying new workflow task from server"
         );
         let task_start_time = Instant::now();
@@ -303,13 +303,13 @@ impl WorkflowTaskManager {
             )
             .expect("Workflow machines must exist, we just created/updated them");
 
-        if !next_activation.jobs.is_empty() {
+        if next_activation.jobs.is_empty() {
+            NewWfTaskOutcome::Autocomplete
+        } else {
             if let Err(wme) = self.insert_outstanding_activation(&next_activation) {
                 return NewWfTaskOutcome::Evict(wme.into());
             }
             NewWfTaskOutcome::IssueActivation(next_activation)
-        } else {
-            NewWfTaskOutcome::Autocomplete
         }
     }
 
@@ -325,19 +325,20 @@ impl WorkflowTaskManager {
             return Ok(None);
         }
 
-        let task_token = if let Some(entry) = self.workflow_machines.get_task(run_id) {
-            entry.info.task_token.clone()
-        } else {
-            if !self.activation_has_eviction(run_id) {
-                // Don't bother warning if this was an eviction, since it's normal to issue
-                // eviction activations without an associated workflow task in that case.
-                warn!(
-                    run_id,
-                    "Attempted to complete activation for nonexistent run"
-                );
-            }
-            return Ok(None);
-        };
+        let (task_token, is_leg_query_task) =
+            if let Some(entry) = self.workflow_machines.get_task(run_id) {
+                (entry.info.task_token.clone(), entry.legacy_query.is_some())
+            } else {
+                if !self.activation_has_eviction(run_id) {
+                    // Don't bother warning if this was an eviction, since it's normal to issue
+                    // eviction activations without an associated workflow task in that case.
+                    warn!(
+                        run_id,
+                        "Attempted to complete activation for run without associated workflow task"
+                    );
+                }
+                return Ok(None);
+            };
 
         // If the only command in the activation is a legacy query response, that means we need
         // to respond differently than a typical activation.
@@ -364,7 +365,6 @@ impl WorkflowTaskManager {
                             return Err(WorkflowUpdateError {
                                 source: WFMachinesError::Fatal("Legacy query activation response included other commands, this is not allowed and constitutes an error in the lang SDK".to_string()),
                                 run_id: run_id.to_string(),
-                                task_token: Some(task_token)
                             });
                         }
                         query_responses.push(qr);
@@ -375,30 +375,32 @@ impl WorkflowTaskManager {
             }
 
             // Send commands from lang into the machines
-            machine_mut!(self, run_id, task_token, |wfm: &mut WorkflowManager| {
+            machine_mut!(self, run_id, |wfm: &mut WorkflowManager| {
                 wfm.push_commands(commands).boxed()
             })?;
             // Check if the workflow run needs another activation and queue it up if there is one
             // by pushing it into the pending activations list
-            let next_activation = machine_mut!(
-                self,
-                run_id,
-                task_token,
-                move |mgr: &mut WorkflowManager| mgr.get_next_activation().boxed()
-            )?;
+            let next_activation = machine_mut!(self, run_id, move |mgr: &mut WorkflowManager| mgr
+                .get_next_activation()
+                .boxed())?;
             if !next_activation.jobs.is_empty() {
                 self.pending_activations.push(next_activation);
                 let _ = self.pending_activations_notifier.send(true);
             }
             // We want to fetch the outgoing commands only after any new activation has been queued,
             // as doing so may have altered the outgoing commands.
-            let server_cmds =
-                machine_mut!(self, run_id, task_token, |wfm: &mut WorkflowManager| {
-                    async move { Ok(wfm.get_server_commands()) }.boxed()
-                })?;
+            let server_cmds = machine_mut!(self, run_id, |wfm: &mut WorkflowManager| {
+                async move { Ok(wfm.get_server_commands()) }.boxed()
+            })?;
+            let is_query_playback = is_leg_query_task && query_responses.is_empty();
             // We only actually want to send commands back to the server if there are no more
-            // pending activations and we are caught up on replay.
-            if !self.pending_activations.has_pending(run_id) && !server_cmds.replaying {
+            // pending activations and we are caught up on replay. We don't want to complete a wft
+            // if we already saw the final event in the workflow, or if we are playing back for the
+            // express purpose of fulfilling a query
+            if !self.pending_activations.has_pending(run_id)
+                && !server_cmds.replaying
+                && !is_query_playback
+            {
                 Some(ServerCommandsWithWorkflowInfo {
                     task_token,
                     action: ActivationAction::WftComplete {
@@ -406,7 +408,9 @@ impl WorkflowTaskManager {
                         query_responses,
                     },
                 })
-            } else if !query_responses.is_empty() {
+            } else if query_responses.is_empty() {
+                None
+            } else {
                 Some(ServerCommandsWithWorkflowInfo {
                     task_token,
                     action: ActivationAction::WftComplete {
@@ -414,8 +418,6 @@ impl WorkflowTaskManager {
                         query_responses,
                     },
                 })
-            } else {
-                None
             }
         };
         Ok(ret)
@@ -447,13 +449,9 @@ impl WorkflowTaskManager {
             FailedActivationOutcome::ReportLegacyQueryFailure(tt)
         } else {
             // Blow up any cached data associated with the workflow
-            let should_report =
-                if let Some(attempt) = self.request_eviction(run_id, "Activation failed by lang") {
-                    // Only report to server if the last task wasn't also a failure (avoid spam)
-                    attempt <= 1
-                } else {
-                    true
-                };
+            let should_report = self
+                .request_eviction(run_id, "Activation failed")
+                .map_or(true, |attempt| attempt <= 1);
             if should_report {
                 FailedActivationOutcome::Report(tt)
             } else {
@@ -511,11 +509,7 @@ impl WorkflowTaskManager {
 
                 Ok((wft_info, activation))
             }
-            Err(source) => Err(WorkflowUpdateError {
-                source,
-                run_id,
-                task_token: Some(wft_info.task_token),
-            }),
+            Err(source) => Err(WorkflowUpdateError { source, run_id }),
         }
     }
 
@@ -541,11 +535,10 @@ impl WorkflowTaskManager {
             if !just_evicted {
                 // Check if there was a legacy query which must be fulfilled, and if there is create
                 // a new pending activation for it.
-                if let Some(ref mut ot) = self
+                if let Some(ref mut ot) = &mut *self
                     .workflow_machines
                     .get_task_mut(run_id)
                     .expect("Machine must exist")
-                    .deref_mut()
                 {
                     if let Some(query) = ot.legacy_query.take() {
                         let na = create_query_activation(run_id.to_string(), [query]);
@@ -623,7 +616,7 @@ impl WorkflowTaskManager {
     fn activation_has_eviction(&self, run_id: &str) -> bool {
         self.workflow_machines
             .get_activation(run_id)
-            .map(|oa| oa.has_eviction())
+            .map(OutstandingActivation::has_eviction)
             .unwrap_or_default()
     }
 }

package/sdk-core/test_utils/src/lib.rs

@@ -46,7 +46,7 @@ impl CoreWfStarter {
                 .unwrap(),
             worker_config: WorkerConfigBuilder::default()
                 .task_queue(task_queue)
-                .max_cached_workflows(1000usize)
+                .max_cached_workflows(1000_usize)
                 .build()
                 .unwrap(),
             wft_timeout: None,
@@ -63,7 +63,7 @@ impl CoreWfStarter {
     }
 
     pub async fn shutdown(&mut self) {
-        self.get_core().await.shutdown().await
+        self.get_core().await.shutdown().await;
     }
 
     pub async fn get_core(&mut self) -> Arc<dyn Core> {

package/sdk-core/tests/integ_tests/workflow_tests/activities.rs

@@ -8,7 +8,7 @@ use temporal_sdk_core_protos::{
         workflow_activation::{wf_activation_job, FireTimer, ResolveActivity, WfActivationJob},
         workflow_commands::{ActivityCancellationType, RequestCancelActivity, StartTimer},
         workflow_completion::WfActivationCompletion,
-        ActivityTaskCompletion, IntoCompletion,
+        ActivityHeartbeat, ActivityTaskCompletion, IntoCompletion,
     },
     temporal::api::{
         common::v1::{ActivityType, Payloads},
@@ -647,3 +647,63 @@ async fn async_activity_completion_workflow() {
     );
     core.complete_execution(&task_q, &task.run_id).await;
 }
+
+#[tokio::test]
+async fn activity_cancelled_after_heartbeat_times_out() {
+    let test_name = "activity_cancelled_after_heartbeat_times_out";
+    let (core, task_q) = init_core_and_create_wf(test_name).await;
+    let activity_id = "act-1";
+    let task = core.poll_workflow_activation(&task_q).await.unwrap();
+    // Complete workflow task and schedule activity
+    core.complete_workflow_activation(
+        schedule_activity_cmd(
+            0,
+            &task_q,
+            activity_id,
+            ActivityCancellationType::WaitCancellationCompleted,
+            Duration::from_secs(60),
+            Duration::from_secs(1),
+        )
+        .into_completion(task_q.clone(), task.run_id),
+    )
+    .await
+    .unwrap();
+    // Poll activity and verify that it's been scheduled with correct parameters
+    let task = core.poll_activity_task(&task_q).await.unwrap();
+    assert_matches!(
+        task.variant,
+        Some(act_task::Variant::Start(start_activity)) => {
+            assert_eq!(start_activity.activity_type, "test_activity".to_string())
+        }
+    );
+    // Delay the heartbeat
+    sleep(Duration::from_secs(2)).await;
+    core.record_activity_heartbeat(ActivityHeartbeat {
+        task_token: task.task_token.clone(),
+        task_queue: task_q.to_string(),
+        details: vec![],
+    });
+
+    // Verify activity got cancelled
+    let cancel_task = core.poll_activity_task(&task_q).await.unwrap();
+    assert_eq!(cancel_task.task_token, task.task_token.clone());
+    assert_matches!(cancel_task.variant, Some(act_task::Variant::Cancel(_)));
+
+    // Complete activity with cancelled result
+    core.complete_activity_task(ActivityTaskCompletion {
+        task_token: task.task_token.clone(),
+        task_queue: task_q.to_string(),
+        result: Some(ActivityResult::cancel_from_details(None)),
+    })
+    .await
+    .unwrap();
+
+    // Verify shutdown completes
+    core.shutdown_worker(task_q.as_str()).await;
+    core.shutdown().await;
+    // Cleanup just in case
+    core.server_gateway()
+        .terminate_workflow_execution(test_name.to_string(), None)
+        .await
+        .unwrap();
+}

package/src/conversions.rs

@@ -276,6 +276,21 @@ impl ObjectHandleConversionsExt for Handle<'_, JsObject> {
         ) as u64);
         let max_cached_workflows =
             js_value_getter!(cx, self, "maxCachedWorkflows", JsNumber) as usize;
+
+        let max_heartbeat_throttle_interval = Duration::from_millis(js_value_getter!(
+            cx,
+            self,
+            "maxHeartbeatThrottleIntervalMs",
+            JsNumber
+        ) as u64);
+
+        let default_heartbeat_throttle_interval = Duration::from_millis(js_value_getter!(
+            cx,
+            self,
+            "defaultHeartbeatThrottleIntervalMs",
+            JsNumber
+        ) as u64);
+
         Ok(WorkerConfig {
             no_remote_activities: false, // TODO: make this configurable once Core implements local activities
             max_concurrent_at_polls,
@@ -286,6 +301,8 @@ impl ObjectHandleConversionsExt for Handle<'_, JsObject> {
             nonsticky_to_sticky_poll_ratio,
             sticky_queue_schedule_to_start_timeout,
             task_queue,
+            max_heartbeat_throttle_interval,
+            default_heartbeat_throttle_interval,
         })
     }
 }