@temporalio/core-bridge 0.14.0 → 0.16.4

package/sdk-core/src/core_tests/retry.rs

@@ -45,14 +45,16 @@ async fn long_poll_non_retryable_errors() {
         let mut mock_gateway = MockServerGatewayApis::new();
         mock_gateway
             .expect_poll_workflow_task()
-            .returning(move |_| Err(Status::new(code, "non-retryable failure")))
+            .returning(move |_, _| Err(Status::new(code, "non-retryable failure")))
             .times(1);
         mock_gateway
             .expect_poll_activity_task()
             .returning(move |_| Err(Status::new(code, "non-retryable failure")))
             .times(1);
         let retry_gateway = RetryGateway::new(mock_gateway, Default::default());
-        let result = retry_gateway.poll_workflow_task("tq".to_string()).await;
+        let result = retry_gateway
+            .poll_workflow_task("tq".to_string(), false)
+            .await;
         assert!(result.is_err());
         let result = retry_gateway.poll_activity_task("tq".to_string()).await;
         assert!(result.is_err());
@@ -86,11 +88,11 @@ async fn long_poll_retries_forever() {
     let mut mock_gateway = MockServerGatewayApis::new();
     mock_gateway
         .expect_poll_workflow_task()
-        .returning(move |_| Err(Status::new(Code::Unknown, "retryable failure")))
+        .returning(move |_, _| Err(Status::new(Code::Unknown, "retryable failure")))
         .times(50);
     mock_gateway
         .expect_poll_workflow_task()
-        .returning(|_| Ok(Default::default()))
+        .returning(|_, _| Ok(Default::default()))
         .times(1);
     mock_gateway
         .expect_poll_activity_task()
@@ -103,7 +105,9 @@ async fn long_poll_retries_forever() {
 
     let retry_gateway = RetryGateway::new(mock_gateway, Default::default());
 
-    let result = retry_gateway.poll_workflow_task("tq".to_string()).await;
+    let result = retry_gateway
+        .poll_workflow_task("tq".to_string(), false)
+        .await;
     assert!(result.is_ok());
     let result = retry_gateway.poll_activity_task("tq".to_string()).await;
     assert!(result.is_ok());
@@ -116,11 +120,11 @@ async fn long_poll_retries_deadline_exceeded() {
         let mut mock_gateway = MockServerGatewayApis::new();
         mock_gateway
             .expect_poll_workflow_task()
-            .returning(move |_| Err(Status::new(code, "retryable failure")))
+            .returning(move |_, _| Err(Status::new(code, "retryable failure")))
             .times(5);
         mock_gateway
             .expect_poll_workflow_task()
-            .returning(|_| Ok(Default::default()))
+            .returning(|_, _| Ok(Default::default()))
             .times(1);
         mock_gateway
             .expect_poll_activity_task()
@@ -133,7 +137,9 @@ async fn long_poll_retries_deadline_exceeded() {
 
         let retry_gateway = RetryGateway::new(mock_gateway, Default::default());
 
-        let result = retry_gateway.poll_workflow_task("tq".to_string()).await;
+        let result = retry_gateway
+            .poll_workflow_task("tq".to_string(), false)
+            .await;
         assert!(result.is_ok());
         let result = retry_gateway.poll_activity_task("tq".to_string()).await;
         assert!(result.is_ok());

package/sdk-core/src/core_tests/workers.rs

@@ -179,7 +179,7 @@ async fn after_shutdown_of_worker_get_shutdown_err() {
         let res = core.poll_workflow_activation(TEST_Q).await.unwrap();
         assert_matches!(
             res.jobs[0].variant,
-            Some(wf_activation_job::Variant::RemoveFromCache(true))
+            Some(wf_activation_job::Variant::RemoveFromCache(_))
         );
         core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, run_id.clone()))
             .await

package/sdk-core/src/core_tests/workflow_tasks.rs

@@ -6,7 +6,7 @@ use crate::{
         build_fake_core, build_mock_pollers, build_multihist_mock_sg, canned_histories,
         gen_assert_and_fail, gen_assert_and_reply, hist_to_poll_resp, mock_core, poll_and_reply,
         poll_and_reply_clears_outstanding_evicts, single_hist_mock_sg, FakeWfResponses,
-        MockPollCfg, MocksHolder, ResponseType, TestHistoryBuilder, TEST_Q,
+        MockPollCfg, MocksHolder, ResponseType, TestHistoryBuilder, NO_MORE_WORK_ERROR_MSG, TEST_Q,
     },
     workflow::WorkflowCachingPolicy::{self, AfterEveryReply, NonSticky},
     Core, CoreSDK, WfActivationCompletion,
@@ -30,7 +30,7 @@ use temporal_sdk_core_protos::{
         },
     },
     temporal::api::{
-        enums::v1::EventType,
+        enums::v1::{EventType, WorkflowTaskFailedCause},
         failure::v1::Failure,
         history::v1::{history_event, TimerFiredEventAttributes},
         workflowservice::v1::RespondWorkflowTaskCompletedResponse,
@@ -642,9 +642,9 @@ async fn workflow_update_random_seed_on_workflow_reset() {
                                 UpdateRandomSeed{randomness_seed})),
                         }] => {
                             assert_ne!(randomness_seed_from_start.load(Ordering::SeqCst),
-                                      *randomness_seed)
+                                      *randomness_seed);
                         }
-                    )
+                    );
                 },
                 vec![CompleteWorkflowExecution { result: None }.into()],
             ),
@@ -1391,3 +1391,346 @@ async fn buffering_tasks_doesnt_count_toward_outstanding_max() {
         PollWfError::TonicError(_)
     );
 }
+
+#[tokio::test]
+async fn fail_wft_then_recover() {
+    let t = canned_histories::long_sequential_timers(1);
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        // We need to deliver all of history twice because of eviction
+        [ResponseType::AllHistory, ResponseType::AllHistory],
+        MockServerGatewayApis::new(),
+    );
+    mh.num_expected_fails = Some(1);
+    mh.expect_fail_wft_matcher =
+        Box::new(|_, cause, _| matches!(cause, WorkflowTaskFailedCause::NonDeterministicError));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(TEST_Q, |wc| {
+        wc.max_cached_workflows = 2;
+    });
+    let core = mock_core(mock);
+
+    let act = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    // Start an activity instead of a timer, triggering nondeterminism error
+    core.complete_workflow_activation(WfActivationCompletion::from_cmds(
+        TEST_Q,
+        act.run_id.clone(),
+        vec![ScheduleActivity {
+            activity_id: "fake_activity".to_string(),
+            ..Default::default()
+        }
+        .into()],
+    ))
+    .await
+    .unwrap();
+    // We must handle an eviction now
+    let evict_act = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    assert_eq!(evict_act.run_id, act.run_id);
+    assert_matches!(
+        evict_act.jobs.as_slice(),
+        [WfActivationJob {
+            variant: Some(wf_activation_job::Variant::RemoveFromCache(_)),
+        }]
+    );
+    core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, evict_act.run_id))
+        .await
+        .unwrap();
+
+    // Workflow starting over, this time issue the right command
+    let act = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    core.complete_workflow_activation(WfActivationCompletion::from_cmds(
+        TEST_Q,
+        act.run_id,
+        vec![StartTimer {
+            seq: 1,
+            ..Default::default()
+        }
+        .into()],
+    ))
+    .await
+    .unwrap();
+    let act = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    assert_matches!(
+        act.jobs.as_slice(),
+        [WfActivationJob {
+            variant: Some(wf_activation_job::Variant::FireTimer(_)),
+        },]
+    );
+    core.complete_workflow_activation(WfActivationCompletion::from_cmds(
+        TEST_Q,
+        act.run_id,
+        vec![CompleteWorkflowExecution { result: None }.into()],
+    ))
+    .await
+    .unwrap();
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn poll_response_triggers_wf_error() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    // Add this nonsense event here to make applying the poll response fail
+    t.add_external_signal_completed(100);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::AllHistory],
+        MockServerGatewayApis::new(),
+    );
+    // Since applying the poll response immediately generates an error core will start polling again
+    // Rather than panic on bad expectation we want to return the magic "no more work" error
+    mh.enforce_correct_number_of_polls = false;
+    let mock = build_mock_pollers(mh);
+    let core = mock_core(mock);
+    // Poll for first WFT, which is immediately an eviction
+    let act = core.poll_workflow_activation(TEST_Q).await;
+    assert_matches!(act, Err(PollWfError::TonicError(err))
+                    if err.message() == NO_MORE_WORK_ERROR_MSG);
+}
+
+// Verifies we can handle multiple wft timeouts in a row if lang is being very slow in responding
+#[tokio::test]
+async fn lang_slower_than_wft_timeouts() {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+    t.add_workflow_task_timed_out();
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let tasks = [
+        hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()),
+        hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()),
+        hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()),
+    ];
+    let mut mock = MockServerGatewayApis::new();
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(|_| Err(tonic::Status::not_found("Workflow task not found.")));
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(|_| Ok(Default::default()));
+    let mut mock = MocksHolder::from_gateway_with_responses(mock, tasks, []);
+    mock.worker_cfg(TEST_Q, |wc| {
+        wc.max_cached_workflows = 2;
+    });
+    let core = mock_core(mock);
+
+    let wf_task = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    let poll_until_no_work = core.poll_workflow_activation(TEST_Q).await;
+    assert_matches!(poll_until_no_work, Err(PollWfError::TonicError(err))
+                    if err.message() == NO_MORE_WORK_ERROR_MSG);
+    // This completion runs into a workflow task not found error, since it's completing a stale
+    // task.
+    core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, wf_task.run_id))
+        .await
+        .unwrap();
+    // Now we should get an eviction
+    let wf_task = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WfActivationJob {
+            variant: Some(wf_activation_job::Variant::RemoveFromCache(_)),
+        }]
+    );
+    core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, wf_task.run_id))
+        .await
+        .unwrap();
+    // The last WFT buffered should be applied now
+    let start_again = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    assert_matches!(
+        start_again.jobs[0].variant,
+        Some(wf_activation_job::Variant::StartWorkflow(_))
+    );
+    core.complete_workflow_activation(WfActivationCompletion::from_cmds(
+        TEST_Q,
+        start_again.run_id,
+        vec![CompleteWorkflowExecution { result: None }.into()],
+    ))
+    .await
+    .unwrap();
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn tries_cancel_of_completed_activity() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    let scheduled_event_id = t.add_activity_task_scheduled("1");
+    t.add_we_signaled("sig", vec![]);
+    let started_event_id = t.add_activity_task_started(scheduled_event_id);
+    t.add_activity_task_completed(scheduled_event_id, started_event_id, Default::default());
+    t.add_workflow_task_scheduled_and_started();
+
+    let mock = MockServerGatewayApis::new();
+    let mut mock = single_hist_mock_sg("fake_wf_id", t, &[1, 2], mock, true);
+    mock.worker_cfg(TEST_Q, |cfg| cfg.max_cached_workflows = 1);
+    let core = mock_core(mock);
+
+    let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    core.complete_workflow_activation(WfActivationCompletion::from_cmd(
+        TEST_Q,
+        activation.run_id,
+        ScheduleActivity {
+            seq: 1,
+            activity_id: "1".to_string(),
+            ..Default::default()
+        }
+        .into(),
+    ))
+    .await
+    .unwrap();
+    let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    assert_matches!(
+        activation.jobs.as_slice(),
+        [
+            WfActivationJob {
+                variant: Some(wf_activation_job::Variant::SignalWorkflow(_)),
+            },
+            WfActivationJob {
+                variant: Some(wf_activation_job::Variant::ResolveActivity(_)),
+            }
+        ]
+    );
+    core.complete_workflow_activation(WfActivationCompletion::from_cmds(
+        TEST_Q,
+        activation.run_id,
+        vec![
+            RequestCancelActivity { seq: 1 }.into(),
+            CompleteWorkflowExecution { result: None }.into(),
+        ],
+    ))
+    .await
+    .unwrap();
+
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn failing_wft_doesnt_eat_permit_forever() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+
+    let failures = 5;
+    // One extra response for when we stop failing
+    let resps = (1..=(failures + 1)).map(|_| 1);
+    let mock = MockServerGatewayApis::new();
+    let mut mock = single_hist_mock_sg("fake_wf_id", t, resps, mock, true);
+    mock.worker_cfg(TEST_Q, |cfg| {
+        cfg.max_cached_workflows = 2;
+        cfg.max_outstanding_workflow_tasks = 2;
+    });
+    let core = mock_core(mock);
+
+    // Spin failing the WFT to verify that we don't get stuck
+    for _ in 1..=failures {
+        let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
+        // Issue a nonsense completion that will trigger a WFT failure
+        core.complete_workflow_activation(WfActivationCompletion::from_cmd(
+            TEST_Q,
+            activation.run_id,
+            RequestCancelActivity { seq: 1 }.into(),
+        ))
+        .await
+        .unwrap();
+        let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
+        assert_matches!(
+            activation.jobs.as_slice(),
+            [WfActivationJob {
+                variant: Some(wf_activation_job::Variant::RemoveFromCache(_)),
+            },]
+        );
+        core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, activation.run_id))
+            .await
+            .unwrap();
+        assert_eq!(core.outstanding_wfts(TEST_Q), 0);
+        assert_eq!(core.available_wft_permits(TEST_Q), 2);
+    }
+    let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    core.complete_workflow_activation(WfActivationCompletion::from_cmd(
+        TEST_Q,
+        activation.run_id,
+        CompleteWorkflowExecution { result: None }.into(),
+    ))
+    .await
+    .unwrap();
+
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn cache_miss_doesnt_eat_permit_forever() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_we_signaled("sig", vec![]);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [
+            ResponseType::ToTaskNum(1),
+            ResponseType::OneTask(2),
+            ResponseType::ToTaskNum(1),
+            ResponseType::OneTask(2),
+            ResponseType::ToTaskNum(1),
+            ResponseType::OneTask(2),
+            // Last one to complete successfully
+            ResponseType::ToTaskNum(1),
+        ],
+        MockServerGatewayApis::new(),
+    );
+    mh.num_expected_fails = Some(3);
+    mh.expect_fail_wft_matcher =
+        Box::new(|_, cause, _| matches!(cause, WorkflowTaskFailedCause::ResetStickyTaskQueue));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(TEST_Q, |cfg| {
+        cfg.max_outstanding_workflow_tasks = 2;
+    });
+    let core = mock_core(mock);
+
+    // Spin missing the cache to verify that we don't get stuck
+    for _ in 1..=3 {
+        // Start
+        let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
+        core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, activation.run_id))
+            .await
+            .unwrap();
+        // Evict
+        let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
+        assert_matches!(
+            activation.jobs.as_slice(),
+            [WfActivationJob {
+                variant: Some(wf_activation_job::Variant::RemoveFromCache(_)),
+            },]
+        );
+        core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, activation.run_id))
+            .await
+            .unwrap();
+        assert_eq!(core.outstanding_wfts(TEST_Q), 0);
+        assert_eq!(core.available_wft_permits(TEST_Q), 2);
+        // When we loop back up, the poll will trigger a cache miss, which we should immediately
+        // reply to WFT with failure, and then poll again, which will deliver the from-the-start
+        // history
+    }
+    let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
+    core.complete_workflow_activation(WfActivationCompletion::from_cmd(
+        TEST_Q,
+        activation.run_id,
+        CompleteWorkflowExecution { result: None }.into(),
+    ))
+    .await
+    .unwrap();
+
+    core.shutdown().await;
+}

package/sdk-core/src/errors.rs

@@ -16,6 +16,23 @@ pub(crate) struct WorkflowUpdateError {
     pub task_token: Option<TaskToken>,
 }
 
+impl From<WorkflowMissingError> for WorkflowUpdateError {
+    fn from(wme: WorkflowMissingError) -> Self {
+        Self {
+            source: WFMachinesError::Fatal("Workflow machines missing".to_string()),
+            run_id: wme.run_id,
+            task_token: None,
+        }
+    }
+}
+
+/// The workflow machines were expected to be in the cache but were not
+#[derive(Debug)]
+pub(crate) struct WorkflowMissingError {
+    /// The run id of the erring workflow
+    pub run_id: String,
+}
+
 /// Errors thrown during initialization of [crate::Core]
 #[derive(thiserror::Error, Debug)]
 pub enum CoreInitError {
@@ -33,15 +50,6 @@ pub enum CoreInitError {
 /// Errors thrown by [crate::Core::poll_workflow_activation]
 #[derive(thiserror::Error, Debug)]
 pub enum PollWfError {
-    /// There was an error specific to a workflow instance. The cached workflow should be deleted
-    /// from lang side.
-    #[error("There was an error with the workflow instance with run id ({run_id}): {source:?}")]
-    WorkflowUpdateError {
-        /// Underlying workflow error
-        source: anyhow::Error,
-        /// The run id of the erring workflow
-        run_id: String,
-    },
     /// [crate::Core::shutdown] was called, and there are no more replay tasks to be handled. Lang
     /// must call [crate::Core::complete_workflow_activation] for any remaining tasks, and then may
     /// exit.
@@ -61,20 +69,11 @@ pub enum PollWfError {
     NoWorkerForQueue(String),
 }
 
-impl From<WorkflowUpdateError> for PollWfError {
-    fn from(e: WorkflowUpdateError) -> Self {
-        Self::WorkflowUpdateError {
-            source: e.source.into(),
-            run_id: e.run_id,
-        }
-    }
-}
-
 impl From<WorkerLookupErr> for PollWfError {
     fn from(e: WorkerLookupErr) -> Self {
         match e {
-            WorkerLookupErr::Shutdown(_) => PollWfError::ShutDown,
-            WorkerLookupErr::NoWorker(s) => PollWfError::NoWorkerForQueue(s),
+            WorkerLookupErr::Shutdown(_) => Self::ShutDown,
+            WorkerLookupErr::NoWorker(s) => Self::NoWorkerForQueue(s),
         }
     }
 }
@@ -98,8 +97,8 @@ pub enum PollActivityError {
 impl From<WorkerLookupErr> for PollActivityError {
     fn from(e: WorkerLookupErr) -> Self {
         match e {
-            WorkerLookupErr::Shutdown(_) => PollActivityError::ShutDown,
-            WorkerLookupErr::NoWorker(s) => PollActivityError::NoWorkerForQueue(s),
+            WorkerLookupErr::Shutdown(_) => Self::ShutDown,
+            WorkerLookupErr::NoWorker(s) => Self::NoWorkerForQueue(s),
         }
     }
 }
@@ -116,15 +115,6 @@ pub enum CompleteWfError {
         /// The completion, which may not be included to avoid unnecessary copies.
         completion: Option<WfActivationCompletion>,
     },
-    /// There was an error specific to a workflow instance. The cached workflow should be deleted
-    /// from lang side.
-    #[error("There was an error with the workflow instance with run id ({run_id}): {source:?}")]
-    WorkflowUpdateError {
-        /// Underlying workflow error
-        source: anyhow::Error,
-        /// The run id of the erring workflow
-        run_id: String,
-    },
     /// There is no worker registered for the queue being polled
     #[error("No worker registered for queue: {0}")]
     NoWorkerForQueue(String),
@@ -134,20 +124,12 @@ pub enum CompleteWfError {
     TonicError(#[from] tonic::Status),
 }
 
-impl From<WorkflowUpdateError> for CompleteWfError {
-    fn from(e: WorkflowUpdateError) -> Self {
-        Self::WorkflowUpdateError {
-            source: e.source.into(),
-            run_id: e.run_id,
-        }
-    }
-}
-
 impl From<WorkerLookupErr> for CompleteWfError {
     fn from(e: WorkerLookupErr) -> Self {
         match e {
-            WorkerLookupErr::Shutdown(s) => CompleteWfError::NoWorkerForQueue(s),
-            WorkerLookupErr::NoWorker(s) => CompleteWfError::NoWorkerForQueue(s),
+            WorkerLookupErr::Shutdown(s) | WorkerLookupErr::NoWorker(s) => {
+                Self::NoWorkerForQueue(s)
+            }
         }
     }
 }
@@ -175,8 +157,9 @@ pub enum CompleteActivityError {
 impl From<WorkerLookupErr> for CompleteActivityError {
     fn from(e: WorkerLookupErr) -> Self {
         match e {
-            WorkerLookupErr::Shutdown(s) => CompleteActivityError::NoWorkerForQueue(s),
-            WorkerLookupErr::NoWorker(s) => CompleteActivityError::NoWorkerForQueue(s),
+            WorkerLookupErr::Shutdown(s) | WorkerLookupErr::NoWorker(s) => {
+                Self::NoWorkerForQueue(s)
+            }
         }
     }
 }

package/sdk-core/src/lib.rs

@@ -202,7 +202,7 @@ pub struct CoreInitOptions {
 
 /// Initializes an instance of the core sdk and establishes a connection to the temporal server.
 ///
-/// Note: Also creates a tokio runtime that will be used for all client-server interactions.  
+/// Note: Also creates a tokio runtime that will be used for all client-server interactions.
 ///
 /// # Panics
 /// * Will panic if called from within an async context, as it will construct a runtime and you
@@ -272,7 +272,8 @@ impl Core for CoreSDK {
         }
     }
 
-    #[instrument(level = "debug", skip(self, completion), fields(completion=%&completion))]
+    #[instrument(level = "debug", skip(self, completion),
+      fields(completion=%&completion, run_id=%completion.run_id))]
     async fn complete_workflow_activation(
         &self,
         completion: WfActivationCompletion,
@@ -308,7 +309,7 @@ impl Core for CoreSDK {
 
     fn request_workflow_eviction(&self, task_queue: &str, run_id: &str) {
         if let Ok(w) = self.worker(task_queue) {
-            w.request_wf_eviction(run_id);
+            w.request_wf_eviction(run_id, "Eviction explicitly requested by lang");
         }
     }
 
@@ -366,6 +367,15 @@ impl CoreSDK {
         self.workers.set_worker_for_task_queue(tq, worker).unwrap();
     }
 
+    #[cfg(test)]
+    pub(crate) fn outstanding_wfts(&self, tq: &str) -> usize {
+        self.worker(tq).unwrap().outstanding_workflow_tasks()
+    }
+    #[cfg(test)]
+    pub(crate) fn available_wft_permits(&self, tq: &str) -> usize {
+        self.worker(tq).unwrap().available_wft_permits()
+    }
+
     fn get_sticky_q_name_for_worker(&self, config: &WorkerConfig) -> Option<String> {
         if config.max_cached_workflows > 0 {
             Some(format!(