@temporalio/core-bridge 1.7.4 → 1.8.1

package/sdk-core/core/src/worker/activities/local_activities.rs

@@ -39,15 +39,11 @@ use tokio_util::sync::CancellationToken;
 
 #[allow(clippy::large_enum_variant)] // Timeouts are relatively rare
 #[derive(Debug)]
-pub(crate) enum DispatchOrTimeoutLA {
+pub(crate) enum NextPendingLAAction {
     /// Send the activity task to lang
     Dispatch(ActivityTask),
-    /// Notify the machines (and maybe lang) that this LA has timed out
-    Timeout {
-        run_id: String,
-        resolution: LocalActivityResolution,
-        task: Option<ActivityTask>,
-    },
+    /// The worker must re-feed this completion back through the machines. Includes timeouts.
+    Autocomplete(LACompleteAction),
 }
 
 #[derive(Debug)]
@@ -87,6 +83,24 @@ impl LocalActivityExecutionResult {
             }),
         })
     }
+
+    fn get_timeout_type(&self) -> Option<TimeoutType> {
+        match self {
+            Self::TimedOut(ActFail {
+                failure:
+                    Some(APIFailure {
+                        failure_info:
+                            Some(failure::FailureInfo::TimeoutFailureInfo(TimeoutFailureInfo {
+                                timeout_type,
+                                ..
+                            })),
+                        ..
+                    }),
+                ..
+            }) => TimeoutType::from_i32(*timeout_type),
+            _ => None,
+        }
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -368,46 +382,30 @@ impl LocalActivityManager {
 
     /// Returns the next pending local-activity related action, or None if shutdown has initiated
     /// and there are no more remaining actions to take.
-    pub(crate) async fn next_pending(&self) -> Option<DispatchOrTimeoutLA> {
+    pub(crate) async fn next_pending(&self) -> Option<NextPendingLAAction> {
         let (new_or_retry, permit) = match self.rcvs.lock().await.next().await? {
             NewOrCancel::Cancel(c) => {
                 return match c {
-                    CancelOrTimeout::Cancel(c) => Some(DispatchOrTimeoutLA::Dispatch(c)),
-                    CancelOrTimeout::Timeout {
-                        run_id,
-                        resolution,
-                        dispatch_cancel,
-                    } => {
-                        let task = if dispatch_cancel {
-                            let tt = self
-                                .dat
-                                .lock()
-                                .la_info
-                                .get(&ExecutingLAId {
-                                    run_id: run_id.clone(),
-                                    seq_num: resolution.seq,
-                                })
-                                .as_ref()
-                                .map(|lai| lai.task_token.clone());
-                            if let Some(task_token) = tt {
-                                self.complete(&task_token, &resolution.result);
-                                Some(ActivityTask {
-                                    task_token: task_token.0,
-                                    variant: Some(activity_task::Variant::Cancel(Cancel {
-                                        reason: ActivityCancelReason::TimedOut as i32,
-                                    })),
-                                })
-                            } else {
-                                None
-                            }
+                    CancelOrTimeout::Cancel(c) => Some(NextPendingLAAction::Dispatch(c)),
+                    CancelOrTimeout::Timeout { run_id, resolution } => {
+                        let tt = self
+                            .dat
+                            .lock()
+                            .la_info
+                            .get(&ExecutingLAId {
+                                run_id,
+                                seq_num: resolution.seq,
+                            })
+                            .as_ref()
+                            .map(|lai| lai.task_token.clone());
+                        if let Some(task_token) = tt {
+                            Some(NextPendingLAAction::Autocomplete(
+                                self.complete(&task_token, resolution.result),
+                            ))
                         } else {
+                            // This timeout is for a no-longer-tracked activity, so, whatever
                             None
-                        };
-                        Some(DispatchOrTimeoutLA::Timeout {
-                            run_id,
-                            resolution,
-                            task,
-                        })
+                        }
                     }
                 };
             }
@@ -443,18 +441,22 @@ impl LocalActivityManager {
         if let Some(s2s) = sa.schedule_to_start_timeout.as_ref() {
             let sat_for = new_la.schedule_time.elapsed().unwrap_or_default();
             if sat_for > *s2s {
-                return Some(DispatchOrTimeoutLA::Timeout {
-                    run_id: new_la.workflow_exec_info.run_id,
-                    resolution: LocalActivityResolution {
-                        seq: sa.seq,
-                        result: LocalActivityExecutionResult::timeout(TimeoutType::ScheduleToStart),
-                        runtime: sat_for,
-                        attempt,
-                        backoff: None,
-                        original_schedule_time: orig_sched_time,
+                return Some(NextPendingLAAction::Autocomplete(
+                    LACompleteAction::Report {
+                        run_id: new_la.workflow_exec_info.run_id,
+                        resolution: LocalActivityResolution {
+                            seq: sa.seq,
+                            result: LocalActivityExecutionResult::timeout(
+                                TimeoutType::ScheduleToStart,
+                            ),
+                            runtime: sat_for,
+                            attempt,
+                            backoff: None,
+                            original_schedule_time: orig_sched_time,
+                        },
+                        task: None,
                     },
-                    task: None,
-                });
+                ));
             }
         }
 
@@ -474,7 +476,7 @@ impl LocalActivityManager {
         );
 
         let (schedule_to_close, start_to_close) = sa.close_timeouts.into_sched_and_start();
-        Some(DispatchOrTimeoutLA::Dispatch(ActivityTask {
+        Some(NextPendingLAAction::Dispatch(ActivityTask {
             task_token: tt.0,
             variant: Some(activity_task::Variant::Start(Start {
                 workflow_namespace: self.namespace.clone(),
@@ -489,8 +491,15 @@ impl LocalActivityManager {
                 current_attempt_scheduled_time: Some(new_la.schedule_time.into()),
                 started_time: Some(SystemTime::now().into()),
                 attempt,
-                schedule_to_close_timeout: schedule_to_close.and_then(|d| d.try_into().ok()),
-                start_to_close_timeout: start_to_close.and_then(|d| d.try_into().ok()),
+                schedule_to_close_timeout: schedule_to_close
+                    .unwrap_or(Duration::ZERO)
+                    .try_into()
+                    .ok(),
+                start_to_close_timeout: start_to_close
+                    .or(schedule_to_close)
+                    .unwrap()
+                    .try_into()
+                    .ok(),
                 heartbeat_timeout: None,
                 retry_policy: Some(sa.retry_policy),
                 is_local: true,
@@ -498,11 +507,11 @@ impl LocalActivityManager {
         }))
     }
 
-    /// Mark a local activity as having completed (pass, fail, or cancelled)
+    /// Mark a local activity as having completed
     pub(crate) fn complete(
         &self,
         task_token: &TaskToken,
-        status: &LocalActivityExecutionResult,
+        status: LocalActivityExecutionResult,
     ) -> LACompleteAction {
         let mut dlock = self.dat.lock();
         if let Some(info) = dlock.outstanding_activity_tasks.remove(task_token) {
@@ -524,37 +533,92 @@ impl LocalActivityManager {
                 }
             }
 
-            match status {
-                LocalActivityExecutionResult::Completed(_)
-                | LocalActivityExecutionResult::TimedOut(_)
-                | LocalActivityExecutionResult::Cancelled { .. } => {
-                    // Timeouts are included in this branch since they are not retried
-                    self.complete_notify.notify_one();
-                    LACompleteAction::Report(info)
-                }
-                LocalActivityExecutionResult::Failed(f) => {
-                    if let Some(backoff_dur) = info.la_info.schedule_cmd.retry_policy.should_retry(
+            enum Outcome {
+                FailurePath { backoff: Option<Duration> },
+                JustReport,
+            }
+            macro_rules! calc_backoff {
+                ($fail: ident) => {
+                    info.la_info.schedule_cmd.retry_policy.should_retry(
                         info.attempt as usize,
-                        f.failure
+                        $fail
+                            .failure
                             .as_ref()
                             .and_then(|f| f.maybe_application_failure()),
-                    ) {
+                    )
+                };
+            }
+
+            let mut is_timeout = false;
+            let outcome = match &status {
+                LocalActivityExecutionResult::Failed(fail) => Outcome::FailurePath {
+                    backoff: calc_backoff!(fail),
+                },
+                LocalActivityExecutionResult::TimedOut(fail)
+                    if matches!(status.get_timeout_type(), Some(TimeoutType::StartToClose)) =>
+                {
+                    // Start to close timeouts are retryable, other timeout types aren't.
+                    is_timeout = true;
+                    Outcome::FailurePath {
+                        backoff: calc_backoff!(fail),
+                    }
+                }
+                LocalActivityExecutionResult::TimedOut(_) => {
+                    is_timeout = true;
+                    Outcome::JustReport
+                }
+                LocalActivityExecutionResult::Completed(_)
+                | LocalActivityExecutionResult::Cancelled { .. } => Outcome::JustReport,
+            };
+
+            let mut resolution = LocalActivityResolution {
+                seq: info.la_info.schedule_cmd.seq,
+                result: status,
+                runtime: info.dispatch_time.elapsed(),
+                attempt: info.attempt,
+                backoff: None,
+                original_schedule_time: info.la_info.schedule_cmd.original_schedule_time,
+            };
+            // We want to generate a cancel task if the reason for failure was a timeout.
+            let task = if is_timeout {
+                Some(ActivityTask {
+                    task_token: task_token.clone().0,
+                    variant: Some(activity_task::Variant::Cancel(Cancel {
+                        reason: ActivityCancelReason::TimedOut as i32,
+                    })),
+                })
+            } else {
+                None
+            };
+
+            match outcome {
+                Outcome::FailurePath { backoff } => {
+                    if let Some(backoff_dur) = backoff {
+                        let fail_or_timeout = if is_timeout { "timed out" } else { "failed" };
                         let will_use_timer =
                             backoff_dur > info.la_info.schedule_cmd.local_retry_threshold;
                         debug!(run_id = %info.la_info.workflow_exec_info.run_id,
                                seq_num = %info.la_info.schedule_cmd.seq,
                                attempt = %info.attempt,
                                will_use_timer,
-                            "Local activity failed, will retry after backing off for {:?}",
-                             backoff_dur
+                            "Local activity {}, will retry after backing off for {:?}",
+                            fail_or_timeout,
+                            backoff_dur
                         );
                         if will_use_timer {
-                            // We want this to be reported, as the workflow will mark this
-                            // failure down, then start a timer for backoff.
-                            return LACompleteAction::LangDoesTimerBackoff(
-                                backoff_dur.try_into().expect("backoff fits into proto"),
-                                info,
-                            );
+                            // This la needs to write a failure marker, and then we will tell lang how
+                            // long of a timer to schedule to back off for. We do this because there are
+                            // no other situations where core generates "internal" commands so it is
+                            // much simpler for lang to reply with the timer / next LA command than to
+                            // do it internally. Plus, this backoff hack we'd like to eliminate
+                            // eventually.
+                            resolution.backoff =
+                                Some(backoff_dur.try_into().expect("backoff fits into proto"));
+                            return LACompleteAction::Report {
+                                run_id: info.la_info.workflow_exec_info.run_id,
+                                resolution,
+                                task,
+                            };
                         }
                         // Immediately create a new task token for the to-be-retried LA
                         let tt = dlock.gen_next_token();
@@ -586,14 +650,32 @@ impl LocalActivityManager {
                                 timeout_bag: maybe_old_lai.and_then(|old| old.timeout_bag),
                             },
                         );
-
-                        LACompleteAction::WillBeRetried
+                        LACompleteAction::WillBeRetried(task)
                     } else {
-                        LACompleteAction::Report(info)
+                        LACompleteAction::Report {
+                            run_id: info.la_info.workflow_exec_info.run_id,
+                            resolution,
+                            task,
+                        }
+                    }
+                }
+                Outcome::JustReport => {
+                    self.complete_notify.notify_one();
+                    LACompleteAction::Report {
+                        run_id: info.la_info.workflow_exec_info.run_id,
+                        resolution,
+                        task,
                     }
                 }
             }
         } else {
+            if !matches!(
+                status,
+                LocalActivityExecutionResult::TimedOut(_)
+                    | LocalActivityExecutionResult::Cancelled { .. }
+            ) {
+                warn!("Tried to complete untracked local activity");
+            }
             LACompleteAction::Untracked
         }
     }
@@ -668,13 +750,18 @@ impl LocalActivityManager {
 #[derive(Debug)]
 #[allow(clippy::large_enum_variant)] // Most will be reported
 pub(crate) enum LACompleteAction {
-    /// Caller should report the status to the workflow
-    Report(LocalInFlightActInfo),
-    /// Lang needs to be told to do the schedule-a-timer-then-rerun hack
-    LangDoesTimerBackoff(prost_types::Duration, LocalInFlightActInfo),
+    /// Caller should report the status to the workflow machines - which may resolve the activity
+    /// or might result in scheduling a backoff timer.
+    Report {
+        run_id: String,
+        resolution: LocalActivityResolution,
+        /// May be set if a task also needs to be dispatched to lang. EX: Cancelling a timed-out
+        /// activity.
+        task: Option<ActivityTask>,
+    },
     /// The activity will be re-enqueued for another attempt (and so status should not be reported
     /// to the workflow)
-    WillBeRetried,
+    WillBeRetried(Option<ActivityTask>),
     /// The activity was unknown
     Untracked,
 }
@@ -695,7 +782,6 @@ enum CancelOrTimeout {
     Timeout {
         run_id: String,
         resolution: LocalActivityResolution,
-        dispatch_cancel: bool,
     },
 }
 
@@ -787,7 +873,6 @@ impl TimeoutBag {
         let timeout_dat = CancelOrTimeout::Timeout {
             run_id: new_la.workflow_exec_info.run_id.clone(),
             resolution,
-            dispatch_cancel: true,
         };
         let start_to_close_dur_and_dat = start_to_close.map(|d| (d, timeout_dat.clone()));
         let fut_dat = schedule_to_close.map(|s2c| (s2c, timeout_dat));
@@ -811,7 +896,7 @@ impl TimeoutBag {
 
     /// Must be called once the associated local activity has been started / dispatched to lang.
     fn mark_started(&mut self) {
-        if let Some((start_to_close, mut dat)) = self.start_to_close_dur_and_dat.take() {
+        if let Some((start_to_close, mut dat)) = self.start_to_close_dur_and_dat.as_ref().cloned() {
             let started_t = Instant::now();
             let cchan = self.cancel_chan.clone();
             self.start_to_close_handle = Some(tokio::spawn(async move {
@@ -848,15 +933,24 @@ mod tests {
     };
     use tokio::task::yield_now;
 
-    impl DispatchOrTimeoutLA {
+    impl NextPendingLAAction {
         fn unwrap(self) -> ActivityTask {
             match self {
-                DispatchOrTimeoutLA::Dispatch(t) => t,
+                NextPendingLAAction::Dispatch(t) => t,
                 _ => {
                     panic!("Non-dispatched action returned")
                 }
             }
         }
+
+        fn is_timeout(&self, with_task: bool) -> bool {
+            matches!(
+                self,
+                Self::Autocomplete(LACompleteAction::Report{task, resolution, ..})
+                if matches!(resolution.result, LocalActivityExecutionResult::TimedOut(_)) &&
+                   task.is_some() == with_task
+            )
+        }
     }
 
     #[tokio::test]
@@ -886,7 +980,7 @@ mod tests {
             let complete_branch = async {
                 lam.complete(
                     &next_tt,
-                    &LocalActivityExecutionResult::Completed(Default::default()),
+                    LocalActivityExecutionResult::Completed(Default::default()),
                 )
             };
             tokio::select! {
@@ -926,7 +1020,7 @@ mod tests {
                 // Spin until the receive lock has been grabbed by the call to pending, to ensure
                 // it's advanced enough
                 while lam.rcvs.try_lock().is_ok() { yield_now().await; }
-                lam.complete(&tt, &LocalActivityExecutionResult::Completed(Default::default()));
+                lam.complete(&tt, LocalActivityExecutionResult::Completed(Default::default()));
             } => (),
         };
     }
@@ -986,10 +1080,10 @@ mod tests {
         let tt = TaskToken(next.task_token);
         let res = lam.complete(
             &tt,
-            &LocalActivityExecutionResult::Failed(Default::default()),
+            LocalActivityExecutionResult::Failed(Default::default()),
         );
-        assert_matches!(res, LACompleteAction::LangDoesTimerBackoff(dur, info)
-            if dur.seconds == 10 && info.attempt == 5
+        assert_matches!(res, LACompleteAction::Report{resolution, ..}
+            if resolution.backoff.as_ref().unwrap().seconds == 10 && resolution.attempt == 5
         )
     }
 
@@ -1021,7 +1115,7 @@ mod tests {
         let tt = TaskToken(next.task_token);
         let res = lam.complete(
             &tt,
-            &LocalActivityExecutionResult::Failed(ActFail {
+            LocalActivityExecutionResult::Failed(ActFail {
                 failure: Some(Failure {
                     failure_info: Some(FailureInfo::ApplicationFailureInfo(
                         ApplicationFailureInfo {
@@ -1034,7 +1128,7 @@ mod tests {
                 }),
             }),
         );
-        assert_matches!(res, LACompleteAction::Report(_));
+        assert_matches!(res, LACompleteAction::Report { .. });
     }
 
     #[tokio::test]
@@ -1068,7 +1162,7 @@ mod tests {
         let tt = TaskToken(next.task_token);
         lam.complete(
             &tt,
-            &LocalActivityExecutionResult::Failed(Default::default()),
+            LocalActivityExecutionResult::Failed(Default::default()),
         );
         // Cancel the activity, which is performing local backoff
         let immediate_res = lam.enqueue([LocalActRequest::Cancel(ExecutingLAId {
@@ -1115,7 +1209,7 @@ mod tests {
         let tt = TaskToken(next.task_token);
         lam.complete(
             &tt,
-            &LocalActivityExecutionResult::Failed(Default::default()),
+            LocalActivityExecutionResult::Failed(Default::default()),
         );
         lam.next_pending().await.unwrap().unwrap();
         assert_eq!(lam.num_in_backoff(), 0);
@@ -1152,10 +1246,7 @@ mod tests {
         // Wait more than the timeout before grabbing the task
         sleep(timeout + Duration::from_millis(10)).await;
 
-        assert_matches!(
-            lam.next_pending().await.unwrap(),
-            DispatchOrTimeoutLA::Timeout { .. }
-        );
+        assert!(dbg!(lam.next_pending().await.unwrap()).is_timeout(false));
         assert_eq!(lam.num_in_backoff(), 0);
         assert_eq!(lam.num_outstanding(), 0);
     }
@@ -1180,6 +1271,7 @@ mod tests {
                 retry_policy: RetryPolicy {
                     initial_interval: Some(prost_dur!(from_millis(10))),
                     backoff_coefficient: 1.0,
+                    maximum_attempts: 1,
                     ..Default::default()
                 },
                 local_retry_threshold: Duration::from_secs(500),
@@ -1195,15 +1287,27 @@ mod tests {
         }
         .into()]);
 
-        lam.next_pending().await.unwrap().unwrap();
+        let next = lam.next_pending().await.unwrap().unwrap();
         assert_eq!(lam.num_in_backoff(), 0);
         assert_eq!(lam.num_outstanding(), 1);
 
+        if let Some(activity_task::Variant::Start(start)) = next.variant {
+            // Validate that timeouts reported to lang matches what server would have provided
+            // if this had been a normal activity with the same timeout configuration.
+            if is_schedule {
+                assert_eq!(start.schedule_to_close_timeout, timeout.try_into().ok());
+                assert_eq!(start.start_to_close_timeout, timeout.try_into().ok());
+            } else {
+                assert_eq!(
+                    start.schedule_to_close_timeout,
+                    Duration::ZERO.try_into().ok()
+                );
+                assert_eq!(start.start_to_close_timeout, timeout.try_into().ok());
+            }
+        };
+
         sleep(timeout + Duration::from_millis(10)).await;
-        assert_matches!(
-            lam.next_pending().await.unwrap(),
-            DispatchOrTimeoutLA::Timeout { .. }
-        );
+        assert!(lam.next_pending().await.unwrap().is_timeout(true));
         assert_eq!(lam.num_outstanding(), 0);
     }
 
@@ -1267,7 +1371,7 @@ mod tests {
                 let tt = TaskToken(next.task_token);
                 lam.complete(
                     &tt,
-                    &LocalActivityExecutionResult::Failed(Default::default()),
+                    LocalActivityExecutionResult::Failed(Default::default()),
                 );
             }
         };