@temporalio/core-bridge 1.11.8 → 1.12.0

package/sdk-core/core/src/worker/activities/local_activities.rs

@@ -3,7 +3,7 @@ use crate::{
     abstractions::{MeteredPermitDealer, OwnedMeteredSemPermit, UsedMeteredSemPermit, dbg_panic},
     protosext::ValidScheduleLA,
     retry_logic::RetryPolicyExt,
-    telemetry::metrics::{activity_type, workflow_type},
+    telemetry::metrics::{activity_type, should_record_failure_metric, workflow_type},
     worker::workflow::HeartbeatTimeoutMsg,
 };
 use futures_util::{
@@ -22,7 +22,7 @@ use temporal_sdk_core_protos::{
     coresdk::{
         LocalActivitySlotInfo,
         activity_result::{Cancellation, Failure as ActFail, Success},
-        activity_task::{ActivityCancelReason, ActivityTask, Cancel, Start, activity_task},
+        activity_task::{ActivityCancelReason, ActivityTask, Start, activity_task},
     },
     temporal::api::{
         common::v1::WorkflowExecution,
@@ -255,6 +255,7 @@ impl LocalActivityManager {
                 MetricsContext::no_op(),
                 None,
                 Arc::new(Default::default()),
+                None,
             ),
             hb_tx,
             MetricsContext::no_op(),
@@ -348,10 +349,10 @@ impl LocalActivityManager {
                 LocalActRequest::Cancel(id) => {
                     debug!(id=?id, "Cancelling local activity");
                     let mut dlock = self.dat.lock();
-                    if let Some(lai) = dlock.la_info.get_mut(&id) {
-                        if let Some(immediate_res) = self.cancel_one_la(id.seq_num, lai) {
-                            immediate_resolutions.push(immediate_res);
-                        }
+                    if let Some(lai) = dlock.la_info.get_mut(&id)
+                        && let Some(immediate_res) = self.cancel_one_la(id.seq_num, lai)
+                    {
+                        immediate_resolutions.push(immediate_res);
                     }
                 }
                 LocalActRequest::CancelAllInRun(run_id) => {
@@ -525,6 +526,7 @@ impl LocalActivityManager {
                     .and_then(|t| t.try_into().ok()),
                 heartbeat_timeout: None,
                 retry_policy: Some(sa.retry_policy),
+                priority: Some(Default::default()),
                 is_local: true,
             })),
         }))
@@ -549,11 +551,11 @@ impl LocalActivityManager {
                 seq_num: info.la_info.schedule_cmd.seq,
             };
             let maybe_old_lai = dlock.la_info.remove(&exec_id);
-            if let Some(ref oldlai) = maybe_old_lai {
-                if let Some(ref bot) = oldlai.backing_off_task {
-                    dbg_panic!("Just-resolved LA should not have backoff task");
-                    bot.abort();
-                }
+            if let Some(ref oldlai) = maybe_old_lai
+                && let Some(ref bot) = oldlai.backing_off_task
+            {
+                dbg_panic!("Just-resolved LA should not have backoff task");
+                bot.abort();
             }
 
             let la_metrics = self.metrics.with_new_attrs([
@@ -582,7 +584,9 @@ impl LocalActivityManager {
             la_metrics.la_exec_latency(runtime);
             let outcome = match &status {
                 LocalActivityExecutionResult::Failed(fail) => {
-                    la_metrics.la_execution_failed();
+                    if should_record_failure_metric(&fail.failure) {
+                        la_metrics.la_execution_failed()
+                    }
                     Outcome::FailurePath {
                         backoff: calc_backoff!(fail),
                     }
@@ -626,12 +630,13 @@ impl LocalActivityManager {
             };
             // We want to generate a cancel task if the reason for failure was a timeout.
             let task = if is_timeout {
-                Some(ActivityTask {
-                    task_token: task_token.clone().0,
-                    variant: Some(activity_task::Variant::Cancel(Cancel {
-                        reason: ActivityCancelReason::TimedOut as i32,
-                    })),
-                })
+                Some(ActivityTask::cancel_from_ids(
+                    task_token.clone().0,
+                    ActivityCancelReason::TimedOut,
+                    ActivityTask::primary_reason_to_cancellation_details(
+                        ActivityCancelReason::TimedOut,
+                    ),
+                ))
             } else {
                 None
             };
@@ -783,12 +788,13 @@ impl LocalActivityManager {
         }
 
         self.cancels_req_tx
-            .send(CancelOrTimeout::Cancel(ActivityTask {
-                task_token: lai.task_token.0.clone(),
-                variant: Some(activity_task::Variant::Cancel(Cancel {
-                    reason: ActivityCancelReason::Cancelled as i32,
-                })),
-            }))
+            .send(CancelOrTimeout::Cancel(ActivityTask::cancel_from_ids(
+                lai.task_token.0.clone(),
+                ActivityCancelReason::Cancelled,
+                ActivityTask::primary_reason_to_cancellation_details(
+                    ActivityCancelReason::Cancelled,
+                ),
+            )))
             .expect("Receive half of LA cancel channel cannot be dropped");
         None
     }
@@ -887,6 +893,7 @@ impl TimeoutBag {
     /// as request to schedule it arrives.
     ///
     /// Returns error in the event the activity is *already* timed out
+    #[allow(clippy::result_large_err)]
     fn new(
         new_la: &NewLocalAct,
         cancel_chan: UnboundedSender<CancelOrTimeout>,
@@ -909,10 +916,10 @@ impl TimeoutBag {
         // Remove any time already elapsed since the scheduling time
         let schedule_to_close = schedule_to_close
             .map(|s2c| s2c.saturating_sub(sched_time.elapsed().unwrap_or_default()));
-        if let Some(ref s2c) = schedule_to_close {
-            if s2c.is_zero() {
-                return Err(resolution);
-            }
+        if let Some(ref s2c) = schedule_to_close
+            && s2c.is_zero()
+        {
+            return Err(resolution);
         }
         let timeout_dat = CancelOrTimeout::Timeout {
             run_id: new_la.workflow_exec_info.run_id.clone(),

package/sdk-core/core/src/worker/activities.rs

@@ -13,7 +13,9 @@ use crate::{
         UsedMeteredSemPermit,
     },
     pollers::{BoxedActPoller, PermittedTqResp, TrackedPermittedTqResp, new_activity_task_poller},
-    telemetry::metrics::{MetricsContext, activity_type, eager, workflow_type},
+    telemetry::metrics::{
+        MetricsContext, activity_type, eager, should_record_failure_metric, workflow_type,
+    },
     worker::{
         activities::activity_heartbeat_manager::ActivityHeartbeatError, client::WorkerClient,
     },
@@ -38,7 +40,7 @@ use temporal_sdk_core_protos::{
     coresdk::{
         ActivityHeartbeat, ActivitySlotInfo,
         activity_result::{self as ar, activity_execution_result as aer},
-        activity_task::{ActivityCancelReason, ActivityTask},
+        activity_task::{ActivityCancelReason, ActivityCancellationDetails, ActivityTask},
     },
     temporal::api::{
         failure::v1::{ApplicationFailureInfo, CanceledFailureInfo, Failure, failure::FailureInfo},
@@ -63,16 +65,19 @@ type OutstandingActMap = Arc<DashMap<TaskToken, RemoteInFlightActInfo>>;
 struct PendingActivityCancel {
     task_token: TaskToken,
     reason: ActivityCancelReason,
-    /// Set true if we should assume the server has already forgotten about this activity
-    consider_not_found: bool,
+    details: ActivityCancellationDetails,
 }
 
 impl PendingActivityCancel {
-    fn new(task_token: TaskToken, reason: ActivityCancelReason) -> Self {
+    fn new(
+        task_token: TaskToken,
+        reason: ActivityCancelReason,
+        details: ActivityCancellationDetails,
+    ) -> Self {
         Self {
             task_token,
             reason,
-            consider_not_found: false,
+            details,
         }
     }
 }
@@ -170,7 +175,7 @@ pub(crate) struct WorkerActivityTasks {
 #[derive(derive_more::From)]
 enum ActivityTaskSource {
     PendingCancel(PendingActivityCancel),
-    PendingStart(Result<(PermittedTqResp<PollActivityTaskQueueResponse>, bool), PollError>),
+    PendingStart(Box<Result<(PermittedTqResp<PollActivityTaskQueueResponse>, bool), PollError>>),
 }
 
 impl WorkerActivityTasks {
@@ -204,7 +209,7 @@ impl WorkerActivityTasks {
         let complete_notify = Arc::new(Notify::new());
         let source_stream = stream::select_with_strategy(
             UnboundedReceiverStream::new(cancels_rx).map(ActivityTaskSource::from),
-            starts_stream.map(ActivityTaskSource::from),
+            starts_stream.map(|a| ActivityTaskSource::from(Box::new(a))),
             |_: &mut ()| PollNext::Left,
         );
 
@@ -349,7 +354,9 @@ impl WorkerActivityTasks {
                             .err()
                     }
                     aer::Status::Failed(ar::Failure { failure }) => {
-                        act_metrics.act_execution_failed();
+                        if should_record_failure_metric(&failure) {
+                            act_metrics.act_execution_failed();
+                        }
                         client
                             .fail_activity_task(task_token.clone(), failure)
                             .await
@@ -455,7 +462,7 @@ impl WorkerActivityTasks {
     }
 
     #[cfg(test)]
-    pub(crate) fn remaining_activity_capacity(&self) -> Option<usize> {
+    pub(crate) fn unused_permits(&self) -> Option<usize> {
         self.eager_activities_semaphore.unused_permits()
     }
 }
@@ -504,13 +511,14 @@ where
                             } else {
                                 details.issued_cancel_to_lang = Some(next_pc.reason);
                                 if next_pc.reason == ActivityCancelReason::NotFound
-                                    || next_pc.consider_not_found
+                                    || next_pc.details.is_not_found
                                 {
                                     details.known_not_found = true;
                                 }
                                 Some(Ok(ActivityTask::cancel_from_ids(
                                     next_pc.task_token.0,
                                     next_pc.reason,
+                                    next_pc.details,
                                 )))
                             }
                         } else {
@@ -562,6 +570,9 @@ where
                                 let _ = cancels_tx.send(PendingActivityCancel::new(
                                     tt,
                                     ActivityCancelReason::WorkerShutdown,
+                                    ActivityTask::primary_reason_to_cancellation_details(
+                                        ActivityCancelReason::WorkerShutdown,
+                                    ),
                                 ));
                             } else {
                                 // Fire off task to keep track of local timeouts. We do this so that
@@ -607,11 +618,15 @@ where
                                                 "Timing out activity due to elapsed local \
                                                  {timeout_type} timer"
                                             );
-                                            let _ = cancel_tx.send(PendingActivityCancel {
-                                                task_token: tt,
-                                                reason: ActivityCancelReason::TimedOut,
-                                                consider_not_found: true,
-                                            });
+                                            let _ = cancel_tx.send(PendingActivityCancel::new(
+                                                tt,
+                                                ActivityCancelReason::TimedOut,
+                                                ActivityCancellationDetails {
+                                                    is_not_found: true,
+                                                    is_timed_out: true,
+                                                    ..Default::default()
+                                                },
+                                            ));
                                         }));
                                     outstanding_info.timeout_resetter = resetter;
                                 }
@@ -635,6 +650,9 @@ where
                             let _ = self.cancels_tx.send(PendingActivityCancel::new(
                                 mapref.key().clone(),
                                 ActivityCancelReason::WorkerShutdown,
+                                ActivityTask::primary_reason_to_cancellation_details(
+                                    ActivityCancelReason::WorkerShutdown,
+                                ),
                             ));
                         }
                     }
@@ -705,9 +723,12 @@ fn worker_shutdown_failure() -> Failure {
 mod tests {
     use super::*;
     use crate::{
-        abstractions::tests::fixed_size_permit_dealer, pollers::new_activity_task_buffer,
-        prost_dur, worker::client::mocks::mock_workflow_client,
+        abstractions::tests::fixed_size_permit_dealer,
+        pollers::{ActivityTaskOptions, LongPollBuffer},
+        prost_dur,
+        worker::client::mocks::mock_workflow_client,
     };
+    use temporal_sdk_core_api::worker::PollerBehavior;
     use temporal_sdk_core_protos::coresdk::activity_result::ActivityExecutionResult;
 
     #[tokio::test]
@@ -740,15 +761,18 @@ mod tests {
         let mock_client = Arc::new(mock_client);
         let sem = fixed_size_permit_dealer(10);
         let shutdown_token = CancellationToken::new();
-        let ap = new_activity_task_buffer(
+        let ap = LongPollBuffer::new_activity_task(
             mock_client.clone(),
             "tq".to_string(),
-            5, // Lots of concurrent pollers, to ensure we don't poll to much when that's the case
+            // Lots of concurrent pollers, to ensure we don't poll to much when that's the case
+            PollerBehavior::SimpleMaximum(5),
             sem.clone(),
-            None,
             shutdown_token.clone(),
             None::<fn(usize)>,
-            Some(2.0),
+            ActivityTaskOptions {
+                max_worker_acts_per_second: Some(2.0),
+                max_tps: None,
+            },
         );
         let atm = WorkerActivityTasks::new(
             sem.clone(),
@@ -829,15 +853,17 @@ mod tests {
         let mock_client = Arc::new(mock_client);
         let sem = fixed_size_permit_dealer(1);
         let shutdown_token = CancellationToken::new();
-        let ap = new_activity_task_buffer(
+        let ap = LongPollBuffer::new_activity_task(
             mock_client.clone(),
             "tq".to_string(),
-            1,
+            PollerBehavior::SimpleMaximum(1),
             sem.clone(),
-            None,
             shutdown_token.clone(),
             None::<fn(usize)>,
-            None,
+            ActivityTaskOptions {
+                max_worker_acts_per_second: None,
+                max_tps: None,
+            },
         );
         let atm = WorkerActivityTasks::new(
             sem.clone(),
@@ -900,15 +926,17 @@ mod tests {
         let mock_client = Arc::new(mock_client);
         let sem = fixed_size_permit_dealer(1);
         let shutdown_token = CancellationToken::new();
-        let ap = new_activity_task_buffer(
+        let ap = LongPollBuffer::new_activity_task(
             mock_client.clone(),
             "tq".to_string(),
-            1,
+            PollerBehavior::SimpleMaximum(1),
             sem.clone(),
-            None,
             shutdown_token.clone(),
             None::<fn(usize)>,
-            None,
+            ActivityTaskOptions {
+                max_worker_acts_per_second: None,
+                max_tps: None,
+            },
         );
         let atm = WorkerActivityTasks::new(
             sem.clone(),

package/sdk-core/core/src/worker/client/mocks.rs

@@ -56,15 +56,15 @@ mockall::mock! {
     pub(crate) ManualWorkerClient {}
     #[allow(unused)]
     impl WorkerClient for ManualWorkerClient {
-        fn poll_workflow_task<'a, 'b>(&'a self, task_queue: TaskQueue)
+        fn poll_workflow_task<'a, 'b>(&'a self, poll_options: PollOptions, wf_options: PollWorkflowOptions)
             -> impl Future<Output = Result<PollWorkflowTaskQueueResponse>> + Send + 'b
             where 'a: 'b, Self: 'b;
 
-        fn poll_activity_task<'a, 'b>(&self, task_queue: String, max_tasks_per_sec: Option<f64>)
+        fn poll_activity_task<'a, 'b>(&self, poll_options: PollOptions, act_options: PollActivityOptions)
             -> impl Future<Output = Result<PollActivityTaskQueueResponse>> + Send + 'b
             where 'a: 'b, Self: 'b;
 
-        fn poll_nexus_task<'a, 'b>(&self, task_queue: String)
+        fn poll_nexus_task<'a, 'b>(&self, poll_options: PollOptions)
             -> impl Future<Output = Result<PollNexusTaskQueueResponse>> + Send + 'b
             where 'a: 'b, Self: 'b;