@temporalio/core-bridge 1.9.2 → 1.10.0

package/sdk-core/core/src/test_help/mod.rs

@@ -37,7 +37,7 @@ use temporal_sdk_core_api::{
 };
 use temporal_sdk_core_protos::{
     coresdk::{
-        workflow_activation::WorkflowActivation,
+        workflow_activation::{workflow_activation_job, WorkflowActivation},
         workflow_commands::workflow_command,
         workflow_completion::{self, workflow_activation_completion, WorkflowActivationCompletion},
     },
@@ -51,15 +51,15 @@ use temporal_sdk_core_protos::{
         },
     },
 };
-use temporal_sdk_core_test_utils::TestWorker;
+use temporal_sdk_core_test_utils::{TestWorker, NAMESPACE};
 use tokio::sync::{mpsc::unbounded_channel, Notify};
 use tokio_stream::wrappers::UnboundedReceiverStream;
 
-pub const TEST_Q: &str = "q";
+pub(crate) const TEST_Q: &str = "q";
 
-pub fn test_worker_cfg() -> WorkerConfigBuilder {
+pub(crate) fn test_worker_cfg() -> WorkerConfigBuilder {
     let mut wcb = WorkerConfigBuilder::default();
-    wcb.namespace("default")
+    wcb.namespace(NAMESPACE)
         .task_queue(TEST_Q)
         .worker_build_id("test_bin_id")
         .ignore_evicts_on_shutdown(true)
@@ -71,7 +71,7 @@ pub fn test_worker_cfg() -> WorkerConfigBuilder {
 /// When constructing responses for mocks, indicates how a given response should be built
 #[derive(derive_more::From)]
 #[allow(clippy::large_enum_variant)] // Test only code, whatever.
-pub enum ResponseType {
+pub(crate) enum ResponseType {
     ToTaskNum(usize),
     /// Returns just the history after the WFT completed of the provided task number - 1, through to
     /// the next WFT started. Simulating the incremental history for just the provided task number
@@ -84,8 +84,9 @@ pub enum ResponseType {
     AllHistory,
     Raw(PollWorkflowTaskQueueResponse),
 }
+
 #[derive(Eq, PartialEq, Hash)]
-pub enum HashableResponseType {
+pub(crate) enum HashableResponseType {
     ToTaskNum(usize),
     OneTask(usize),
     UntilResolved(usize),
@@ -93,8 +94,9 @@ pub enum HashableResponseType {
     AllHistory,
     Raw(TaskToken),
 }
+
 impl ResponseType {
-    pub fn hashable(&self) -> HashableResponseType {
+    fn hashable(&self) -> HashableResponseType {
         match self {
             ResponseType::ToTaskNum(x) => HashableResponseType::ToTaskNum(*x),
             ResponseType::OneTask(x) => HashableResponseType::OneTask(*x),
@@ -107,6 +109,7 @@ impl ResponseType {
         }
     }
 }
+
 impl From<&usize> for ResponseType {
     fn from(u: &usize) -> Self {
         Self::ToTaskNum(*u)
@@ -139,7 +142,10 @@ pub(crate) fn build_fake_worker(
 
 pub(crate) fn build_fake_sdk(mock_cfg: MockPollCfg) -> temporal_sdk::Worker {
     let mut mock = build_mock_pollers(mock_cfg);
-    mock.worker_cfg(|c| c.max_cached_workflows = 1);
+    mock.worker_cfg(|c| {
+        c.max_cached_workflows = 1;
+        c.ignore_evicts_on_shutdown = false;
+    });
     let core = mock_worker(mock);
     let mut worker = temporal_sdk::Worker::new_from_core(Arc::new(core), "replay_q".to_string());
     worker.set_worker_interceptor(FailOnNondeterminismInterceptor {});
@@ -179,37 +185,39 @@ pub(crate) fn mock_sdk_cfg(
     TestWorker::new(Arc::new(core), TEST_Q.to_string())
 }
 
-pub struct FakeWfResponses {
-    pub wf_id: String,
-    pub hist: TestHistoryBuilder,
-    pub response_batches: Vec<ResponseType>,
+pub(crate) struct FakeWfResponses {
+    pub(crate) wf_id: String,
+    pub(crate) hist: TestHistoryBuilder,
+    pub(crate) response_batches: Vec<ResponseType>,
 }
 
 // TODO: Should be all-internal to this module
-pub struct MocksHolder {
+pub(crate) struct MocksHolder {
     client: Arc<dyn WorkerClient>,
     inputs: MockWorkerInputs,
-    pub outstanding_task_map: Option<OutstandingWFTMap>,
+    pub(crate) outstanding_task_map: Option<OutstandingWFTMap>,
 }
 
 impl MocksHolder {
-    pub fn worker_cfg(&mut self, mutator: impl FnOnce(&mut WorkerConfig)) {
+    pub(crate) fn worker_cfg(&mut self, mutator: impl FnOnce(&mut WorkerConfig)) {
         mutator(&mut self.inputs.config);
     }
-    pub fn set_act_poller(&mut self, poller: BoxedPoller<PollActivityTaskQueueResponse>) {
+
+    pub(crate) fn set_act_poller(&mut self, poller: BoxedPoller<PollActivityTaskQueueResponse>) {
         self.inputs.act_poller = Some(poller);
     }
+
     /// Can be used for tests that need to avoid auto-shutdown due to running out of mock responses
-    pub fn make_wft_stream_interminable(&mut self) {
+    pub(crate) fn make_wft_stream_interminable(&mut self) {
         let old_stream = std::mem::replace(&mut self.inputs.wft_stream, stream::pending().boxed());
         self.inputs.wft_stream = old_stream.chain(stream::pending()).boxed();
     }
 }
 
-pub struct MockWorkerInputs {
-    pub wft_stream: BoxStream<'static, Result<ValidPollWFTQResponse, tonic::Status>>,
-    pub act_poller: Option<BoxedPoller<PollActivityTaskQueueResponse>>,
-    pub config: WorkerConfig,
+pub(crate) struct MockWorkerInputs {
+    pub(crate) wft_stream: BoxStream<'static, Result<ValidPollWFTQResponse, tonic::Status>>,
+    pub(crate) act_poller: Option<BoxedPoller<PollActivityTaskQueueResponse>>,
+    pub(crate) config: WorkerConfig,
 }
 
 impl Default for MockWorkerInputs {
@@ -219,7 +227,7 @@ impl Default for MockWorkerInputs {
 }
 
 impl MockWorkerInputs {
-    pub fn new(
+    pub(crate) fn new(
         wft_stream: BoxStream<'static, Result<ValidPollWFTQResponse, tonic::Status>>,
     ) -> Self {
         Self {
@@ -313,7 +321,7 @@ where
     Box::new(mock_poller) as BoxedPoller<T>
 }
 
-pub fn mock_poller<T>() -> MockPoller<T>
+pub(crate) fn mock_poller<T>() -> MockPoller<T>
 where
     T: Send + Sync + 'static,
 {
@@ -323,7 +331,7 @@ where
     mock_poller
 }
 
-pub fn mock_manual_poller<T>() -> MockManualPoller<T>
+pub(crate) fn mock_manual_poller<T>() -> MockManualPoller<T>
 where
     T: Send + Sync + 'static,
 {
@@ -372,25 +380,25 @@ pub(crate) fn single_hist_mock_sg(
 
 #[allow(clippy::type_complexity)]
 pub(crate) struct MockPollCfg {
-    pub hists: Vec<FakeWfResponses>,
-    pub enforce_correct_number_of_polls: bool,
-    pub num_expected_fails: usize,
-    pub num_expected_legacy_query_resps: usize,
-    pub mock_client: MockWorkerClient,
+    pub(crate) hists: Vec<FakeWfResponses>,
+    pub(crate) enforce_correct_number_of_polls: bool,
+    pub(crate) num_expected_fails: usize,
+    pub(crate) num_expected_legacy_query_resps: usize,
+    pub(crate) mock_client: MockWorkerClient,
     /// All calls to fail WFTs must match this predicate
-    pub expect_fail_wft_matcher:
+    pub(crate) expect_fail_wft_matcher:
         Box<dyn Fn(&TaskToken, &WorkflowTaskFailedCause, &Option<Failure>) -> bool + Send>,
-    pub completion_asserts: Option<Box<dyn FnMut(&WorkflowTaskCompletion) + Send>>,
-    pub num_expected_completions: Option<TimesRange>,
+    pub(crate) completion_asserts: Option<Box<dyn FnMut(&WorkflowTaskCompletion) + Send>>,
+    pub(crate) num_expected_completions: Option<TimesRange>,
     /// If being used with the Rust SDK, this is set true. It ensures pollers will not error out
     /// early with no work, since we cannot know the exact number of times polling will happen.
     /// Instead, they will just block forever.
-    pub using_rust_sdk: bool,
-    pub make_poll_stream_interminable: bool,
+    pub(crate) using_rust_sdk: bool,
+    pub(crate) make_poll_stream_interminable: bool,
 }
 
 impl MockPollCfg {
-    pub fn new(
+    pub(crate) fn new(
         hists: Vec<FakeWfResponses>,
         enforce_correct_number_of_polls: bool,
         num_expected_fails: usize,
@@ -410,20 +418,20 @@ impl MockPollCfg {
     }
 
     /// Builds a config which will hand out each WFT in the history builder one by one
-    pub fn from_hist_builder(t: TestHistoryBuilder) -> Self {
+    pub(crate) fn from_hist_builder(t: TestHistoryBuilder) -> Self {
         let full_hist_info = t.get_full_history_info().unwrap();
         let tasks = 1..=full_hist_info.wf_task_count();
         Self::from_resp_batches("fake_wf_id", t, tasks, mock_workflow_client())
     }
 
-    pub fn from_resps(
+    pub(crate) fn from_resps(
         t: TestHistoryBuilder,
         resps: impl IntoIterator<Item = impl Into<ResponseType>>,
     ) -> Self {
         Self::from_resp_batches("fake_wf_id", t, resps, mock_workflow_client())
     }
 
-    pub fn from_resp_batches(
+    pub(crate) fn from_resp_batches(
         wf_id: &str,
         t: TestHistoryBuilder,
         resps: impl IntoIterator<Item = impl Into<ResponseType>>,
@@ -447,7 +455,7 @@ impl MockPollCfg {
         }
     }
 
-    pub fn completion_asserts_from_expectations(
+    pub(crate) fn completion_asserts_from_expectations(
         &mut self,
         builder_fn: impl FnOnce(CompletionAssertsBuilder<'_>),
     ) {
@@ -464,8 +472,9 @@ pub(crate) struct CompletionAssertsBuilder<'a> {
     dest: &'a mut Option<Box<dyn FnMut(&WorkflowTaskCompletion) + Send>>,
     assertions: VecDeque<Box<dyn FnOnce(&WorkflowTaskCompletion) + Send>>,
 }
+
 impl CompletionAssertsBuilder<'_> {
-    pub fn then(
+    pub(crate) fn then(
         &mut self,
         assert: impl FnOnce(&WorkflowTaskCompletion) + Send + 'static,
     ) -> &mut Self {
@@ -473,6 +482,7 @@ impl CompletionAssertsBuilder<'_> {
         self
     }
 }
+
 impl Drop for CompletionAssertsBuilder<'_> {
     fn drop(&mut self) {
         let mut asserts = std::mem::take(&mut self.assertions);
@@ -485,27 +495,32 @@ impl Drop for CompletionAssertsBuilder<'_> {
 }
 
 #[derive(Default, Clone)]
-pub struct OutstandingWFTMap {
+pub(crate) struct OutstandingWFTMap {
     map: Arc<RwLock<BiMap<String, TaskToken>>>,
     waker: Arc<Notify>,
     all_work_delivered: Arc<AtomicBool>,
 }
+
 impl OutstandingWFTMap {
     fn has_run(&self, run_id: &str) -> bool {
         self.map.read().contains_left(run_id)
     }
+
     fn put_token(&self, run_id: String, token: TaskToken) {
         self.map.write().insert(run_id, token);
     }
+
     fn release_token(&self, token: &TaskToken) {
         self.map.write().remove_by_right(token);
         self.waker.notify_one();
     }
-    pub fn release_run(&self, run_id: &str) {
+
+    pub(crate) fn release_run(&self, run_id: &str) {
         self.map.write().remove_by_left(run_id);
         self.waker.notify_waiters();
     }
-    pub fn all_work_delivered(&self) -> bool {
+
+    pub(crate) fn all_work_delivered(&self) -> bool {
         self.all_work_delivered.load(Ordering::Acquire)
     }
 }
@@ -690,10 +705,11 @@ pub(crate) fn build_mock_pollers(mut cfg: MockPollCfg) -> MocksHolder {
     mh
 }
 
-pub struct QueueResponse<T> {
-    pub resp: T,
-    pub delay_until: Option<BoxFuture<'static, ()>>,
+pub(crate) struct QueueResponse<T> {
+    pub(crate) resp: T,
+    pub(crate) delay_until: Option<BoxFuture<'static, ()>>,
 }
+
 impl<T> From<T> for QueueResponse<T> {
     fn from(resp: T) -> Self {
         QueueResponse {
@@ -732,7 +748,7 @@ where
     }
 }
 
-pub fn hist_to_poll_resp(
+pub(crate) fn hist_to_poll_resp(
     t: &TestHistoryBuilder,
     wf_id: impl Into<String>,
     response_type: ResponseType,
@@ -828,18 +844,23 @@ pub(crate) async fn poll_and_reply_clears_outstanding_evicts<'a>(
             }
 
             let mut res = worker.poll_workflow_activation().await.unwrap();
-            let contains_eviction = res.eviction_index();
+            if res.jobs.iter().any(|j| {
+                matches!(
+                    j.variant,
+                    Some(workflow_activation_job::Variant::RemoveFromCache(_))
+                )
+            }) && res.jobs.len() > 1
+            {
+                panic!("Saw an activation with an eviction & other work! {res:?}");
+            }
+            let is_eviction = res.is_only_eviction();
 
             let mut do_release = false;
-            if let Some(eviction_job_ix) = contains_eviction {
-                // If the job list has an eviction, make sure it was the last item in the list
-                // then remove it, since in the tests we don't explicitly specify evict assertions
-                assert_eq!(
-                    eviction_job_ix,
-                    res.jobs.len() - 1,
-                    "Eviction job was not last job in job list"
-                );
-                res.jobs.remove(eviction_job_ix);
+
+            if is_eviction {
+                // If the job is an eviction, clear it, since in the tests we don't explicitly
+                // specify evict assertions
+                res.jobs.clear();
                 do_release = true;
             }
 
@@ -870,7 +891,7 @@ pub(crate) async fn poll_and_reply_clears_outstanding_evicts<'a>(
             }
             // Restart assertions from the beginning if it was an eviction (and workflow execution
             // isn't over)
-            if contains_eviction.is_some() && !ends_execution {
+            if is_eviction && !ends_execution {
                 continue 'outer;
             }
 

package/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs

@@ -45,11 +45,11 @@ enum HeartbeatAction {
 }
 
 #[derive(Debug)]
-pub struct ValidActivityHeartbeat {
-    pub task_token: TaskToken,
-    pub details: Vec<Payload>,
-    pub throttle_interval: Duration,
-    pub timeout_resetter: Option<Arc<Notify>>,
+struct ValidActivityHeartbeat {
+    task_token: TaskToken,
+    details: Vec<Payload>,
+    throttle_interval: Duration,
+    timeout_resetter: Option<Arc<Notify>>,
 }
 
 #[derive(Debug)]
@@ -65,7 +65,7 @@ enum HeartbeatExecutorAction {
 
 /// Errors thrown when heartbeating
 #[derive(thiserror::Error, Debug)]
-pub enum ActivityHeartbeatError {
+pub(crate) enum ActivityHeartbeatError {
     /// Heartbeat referenced an activity that we don't think exists. It may have completed already.
     #[error("Heartbeat has been sent for activity that either completed or never started on this worker.")]
     UnknownActivity,

package/sdk-core/core/src/worker/activities/local_activities.rs

@@ -1,5 +1,5 @@
 use crate::{
-    abstractions::{dbg_panic, MeteredSemaphore, OwnedMeteredSemPermit, UsedMeteredSemPermit},
+    abstractions::{dbg_panic, MeteredPermitDealer, OwnedMeteredSemPermit, UsedMeteredSemPermit},
     protosext::ValidScheduleLA,
     retry_logic::RetryPolicyExt,
     worker::workflow::HeartbeatTimeoutMsg,
@@ -12,9 +12,11 @@ use std::{
     collections::{hash_map::Entry, HashMap},
     fmt::{Debug, Formatter},
     pin::Pin,
+    sync::Arc,
     task::{Context, Poll},
     time::{Duration, Instant, SystemTime},
 };
+use temporal_sdk_core_api::worker::{LocalActivitySlotInfo, LocalActivitySlotKind, SlotSupplier};
 use temporal_sdk_core_protos::{
     coresdk::{
         activity_result::{Cancellation, Failure as ActFail, Success},
@@ -47,18 +49,14 @@ pub(crate) enum NextPendingLAAction {
 }
 
 #[derive(Debug)]
-pub(crate) struct LocalInFlightActInfo {
-    pub la_info: NewLocalAct,
-    pub dispatch_time: Instant,
-    pub attempt: u32,
-    _permit: UsedMeteredSemPermit,
+struct LocalInFlightActInfo {
+    la_info: NewLocalAct,
+    dispatch_time: Instant,
+    attempt: u32,
+    _permit: UsedMeteredSemPermit<LocalActivitySlotKind>,
 }
 
 #[derive(Debug, Clone)]
-#[cfg_attr(
-    feature = "save_wf_inputs",
-    derive(serde::Serialize, serde::Deserialize)
-)]
 pub(crate) enum LocalActivityExecutionResult {
     Completed(Success),
     Failed(ActFail),
@@ -97,33 +95,30 @@ impl LocalActivityExecutionResult {
                         ..
                     }),
                 ..
-            }) => TimeoutType::from_i32(*timeout_type),
+            }) => TimeoutType::try_from(*timeout_type).ok(),
             _ => None,
         }
     }
 }
 
 #[derive(Debug, Clone)]
-#[cfg_attr(
-    feature = "save_wf_inputs",
-    derive(serde::Serialize, serde::Deserialize)
-)]
 pub(crate) struct LocalActivityResolution {
-    pub seq: u32,
-    pub result: LocalActivityExecutionResult,
-    pub runtime: Duration,
-    pub attempt: u32,
-    pub backoff: Option<prost_types::Duration>,
-    pub original_schedule_time: Option<SystemTime>,
+    pub(crate) seq: u32,
+    pub(crate) result: LocalActivityExecutionResult,
+    pub(crate) runtime: Duration,
+    pub(crate) attempt: u32,
+    pub(crate) backoff: Option<prost_types::Duration>,
+    pub(crate) original_schedule_time: Option<SystemTime>,
 }
 
 #[derive(Clone)]
 pub(crate) struct NewLocalAct {
-    pub schedule_cmd: ValidScheduleLA,
-    pub workflow_type: String,
-    pub workflow_exec_info: WorkflowExecution,
-    pub schedule_time: SystemTime,
+    pub(crate) schedule_cmd: ValidScheduleLA,
+    pub(crate) workflow_type: String,
+    pub(crate) workflow_exec_info: WorkflowExecution,
+    pub(crate) schedule_time: SystemTime,
 }
+
 impl Debug for NewLocalAct {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(
@@ -154,8 +149,8 @@ pub(crate) enum LocalActRequest {
 
 #[derive(Debug, Clone, Eq, PartialEq, Hash)]
 pub(crate) struct ExecutingLAId {
-    pub run_id: String,
-    pub seq_num: u32,
+    pub(crate) run_id: String,
+    pub(crate) seq_num: u32,
 }
 
 pub(crate) struct LocalActivityManager {
@@ -210,7 +205,7 @@ impl LAMData {
 
 impl LocalActivityManager {
     pub(crate) fn new(
-        max_concurrent: usize,
+        slot_supplier: Arc<dyn SlotSupplier<SlotKind = LocalActivitySlotKind> + Send + Sync>,
         namespace: String,
         heartbeat_timeout_tx: UnboundedSender<HeartbeatTimeoutMsg>,
         metrics_context: MetricsContext,
@@ -218,11 +213,7 @@ impl LocalActivityManager {
         let (act_req_tx, act_req_rx) = unbounded_channel();
         let (cancels_req_tx, cancels_req_rx) = unbounded_channel();
         let shutdown_complete_tok = CancellationToken::new();
-        let semaphore = MeteredSemaphore::new(
-            max_concurrent,
-            metrics_context,
-            MetricsContext::available_task_slots,
-        );
+        let semaphore = MeteredPermitDealer::new(slot_supplier, metrics_context, None);
         Self {
             namespace,
             rcvs: tokio::sync::Mutex::new(RcvChans::new(
@@ -247,13 +238,11 @@ impl LocalActivityManager {
 
     #[cfg(test)]
     fn test(max_concurrent: usize) -> Self {
+        use crate::worker::tuner::FixedSizeSlotSupplier;
+
+        let ss = Arc::new(FixedSizeSlotSupplier::new(max_concurrent));
         let (hb_tx, _hb_rx) = unbounded_channel();
-        Self::new(
-            max_concurrent,
-            "fake_ns".to_string(),
-            hb_tx,
-            MetricsContext::no_op(),
-        )
+        Self::new(ss, "fake_ns".to_string(), hb_tx, MetricsContext::no_op())
     }
 
     #[cfg(test)]
@@ -386,7 +375,19 @@ impl LocalActivityManager {
         let (new_or_retry, permit) = match self.rcvs.lock().await.next().await? {
             NewOrCancel::Cancel(c) => {
                 return match c {
-                    CancelOrTimeout::Cancel(c) => Some(NextPendingLAAction::Dispatch(c)),
+                    CancelOrTimeout::Cancel(c) => {
+                        if self
+                            .dat
+                            .lock()
+                            .outstanding_activity_tasks
+                            .contains_key(c.task_token.as_slice())
+                        {
+                            Some(NextPendingLAAction::Dispatch(c))
+                        } else {
+                            // Don't dispatch cancels for things we've already stopped tracking
+                            None
+                        }
+                    }
                     CancelOrTimeout::Timeout { run_id, resolution } => {
                         let tt = self
                             .dat
@@ -471,7 +472,9 @@ impl LocalActivityManager {
                 la_info: la_info_for_in_flight_map,
                 dispatch_time: Instant::now(),
                 attempt,
-                _permit: permit.into_used(),
+                _permit: permit.into_used(LocalActivitySlotInfo {
+                    activity_type: new_la.workflow_type.as_str(),
+                }),
             },
         );
 
@@ -787,7 +790,7 @@ enum CancelOrTimeout {
 
 #[allow(clippy::large_enum_variant)]
 enum NewOrCancel {
-    New(NewOrRetry, OwnedMeteredSemPermit),
+    New(NewOrRetry, OwnedMeteredSemPermit<LocalActivitySlotKind>),
     Cancel(CancelOrTimeout),
 }
 
@@ -800,7 +803,7 @@ struct RcvChans {
 impl RcvChans {
     fn new(
         new_reqs: UnboundedReceiver<NewOrRetry>,
-        new_sem: MeteredSemaphore,
+        new_sem: MeteredPermitDealer<LocalActivitySlotKind>,
         cancels: UnboundedReceiver<CancelOrTimeout>,
         shutdown_completed: CancellationToken,
     ) -> Self {