@temporalio/core-bridge 1.8.5 → 1.9.0-rc.0

package/sdk-core/core/src/worker/client.rs

@@ -1,8 +1,8 @@
 //! Worker-specific client needs
 
 pub(crate) mod mocks;
-
-use temporal_client::{Client, RetryClient, WorkflowService};
+use std::sync::Arc;
+use temporal_client::{Client, RetryClient, SlotManager, WorkflowService};
 use temporal_sdk_core_protos::{
     coresdk::workflow_commands::QueryResult,
     temporal::api::{
@@ -13,6 +13,7 @@ use temporal_sdk_core_protos::{
         },
         enums::v1::{TaskQueueKind, WorkflowTaskFailedCause},
         failure::v1::Failure,
+        protocol::v1::Message as ProtocolMessage,
         query::v1::WorkflowQueryResult,
         sdk::v1::WorkflowTaskCompletedMetadata,
         taskqueue::v1::{StickyExecutionAttributes, TaskQueue, TaskQueueMetadata},
@@ -143,6 +144,8 @@ pub(crate) trait WorkerClient: Sync + Send {
 
     #[allow(clippy::needless_lifetimes)] // Clippy is wrong here
     fn capabilities<'a>(&'a self) -> Option<&'a get_system_info_response::Capabilities>;
+    fn workers(&self) -> Arc<SlotManager>;
+    fn is_mock(&self) -> bool;
 }
 
 #[async_trait::async_trait]
@@ -201,12 +204,12 @@ impl WorkerClient for WorkerClientBag {
         let request = RespondWorkflowTaskCompletedRequest {
             task_token: request.task_token.into(),
             commands: request.commands,
+            messages: request.messages,
             identity: self.identity.clone(),
             sticky_attributes: request.sticky_attributes,
             return_new_workflow_task: request.return_new_workflow_task,
             force_create_new_workflow_task: request.force_create_new_workflow_task,
             worker_version_stamp: self.worker_version_stamp(),
-            messages: vec![],
             binary_checksum: self.binary_checksum(),
             query_results: request
                 .query_responses
@@ -381,6 +384,14 @@ impl WorkerClient for WorkerClientBag {
     fn capabilities(&self) -> Option<&Capabilities> {
         self.client.get_client().inner().capabilities()
     }
+
+    fn workers(&self) -> Arc<SlotManager> {
+        self.client.get_client().inner().workers()
+    }
+
+    fn is_mock(&self) -> bool {
+        false
+    }
 }
 
 /// A version of [RespondWorkflowTaskCompletedRequest] that will finish being filled out by the
@@ -391,6 +402,8 @@ pub(crate) struct WorkflowTaskCompletion {
     pub task_token: TaskToken,
     /// A list of new commands to send to the server, such as starting a timer.
     pub commands: Vec<Command>,
+    /// A list of protocol messages to send to the server.
+    pub messages: Vec<ProtocolMessage>,
     /// If set, indicate that next task should be queued on sticky queue with given attributes.
     pub sticky_attributes: Option<StickyExecutionAttributes>,
     /// Responses to queries in the `queries` field of the workflow task.

package/sdk-core/core/src/worker/mod.rs

@@ -1,5 +1,6 @@
 mod activities;
 pub(crate) mod client;
+mod slot_provider;
 mod workflow;
 
 pub use temporal_sdk_core_api::worker::{WorkerConfig, WorkerConfigBuilder};
@@ -12,13 +13,14 @@ pub(crate) use activities::{
 };
 pub(crate) use workflow::{wft_poller::new_wft_poller, LEGACY_QUERY_ID};
 
-#[cfg(test)]
-pub(crate) use workflow::ManagedWFFunc;
+use temporal_client::WorkerKey;
 
 use crate::{
-    abstractions::MeteredSemaphore,
+    abstractions::{dbg_panic, MeteredSemaphore},
     errors::CompleteWfError,
-    pollers::{new_activity_task_buffer, new_workflow_task_buffer, WorkflowTaskPoller},
+    pollers::{
+        new_activity_task_buffer, new_workflow_task_buffer, BoxedActPoller, WorkflowTaskPoller,
+    },
     protosext::validate_activity_completion,
     telemetry::{
         metrics::{
@@ -35,6 +37,8 @@ use crate::{
     ActivityHeartbeat, CompleteActivityError, PollActivityError, PollWfError, WorkerTrait,
 };
 use activities::WorkerActivityTasks;
+use futures_util::{stream, StreamExt};
+use slot_provider::SlotProvider;
 use std::{
     convert::TryInto,
     future,
@@ -59,9 +63,9 @@ use temporal_sdk_core_protos::{
     TaskToken,
 };
 use tokio::sync::mpsc::unbounded_channel;
+use tokio_stream::wrappers::UnboundedReceiverStream;
 use tokio_util::sync::CancellationToken;
 
-use crate::{abstractions::dbg_panic, pollers::BoxedActPoller};
 #[cfg(test)]
 use {
     crate::{
@@ -69,7 +73,6 @@ use {
         protosext::ValidPollWFTQResponse,
     },
     futures::stream::BoxStream,
-    futures_util::StreamExt,
     temporal_sdk_core_protos::temporal::api::workflowservice::v1::PollActivityTaskQueueResponse,
 };
 
@@ -77,7 +80,8 @@ use {
 pub struct Worker {
     config: WorkerConfig,
     wf_client: Arc<dyn WorkerClient>,
-
+    /// Registration key to enable eager workflow start for this worker
+    worker_key: Option<WorkerKey>,
     /// Manages all workflows and WFT processing
     workflows: Workflows,
     /// Manages activity tasks for this worker/task queue
@@ -98,7 +102,15 @@ pub struct Worker {
 #[async_trait::async_trait]
 impl WorkerTrait for Worker {
     async fn poll_workflow_activation(&self) -> Result<WorkflowActivation, PollWfError> {
-        self.next_workflow_activation().await
+        self.next_workflow_activation().await.map(|mut a| {
+            // Attach this worker's Build ID to the activation if appropriate. This is done here
+            // to avoid cloning the ID for every workflow instance. Can be lowered when
+            // https://github.com/temporalio/sdk-core/issues/567 is done
+            if !a.is_replaying {
+                a.build_id_for_current_task = self.config.worker_build_id.clone();
+            }
+            a
+        })
     }
 
     #[instrument(skip(self))]
@@ -164,7 +176,11 @@ impl WorkerTrait for Worker {
             );
         }
         self.shutdown_token.cancel();
-        // First, we want to stop polling of both activity and workflow tasks
+        // First, disable Eager Workflow Start
+        if let Some(key) = self.worker_key {
+            self.wf_client.workers().unregister(key);
+        }
+        // Second, we want to stop polling of both activity and workflow tasks
         if let Some(atm) = self.at_task_mgr.as_ref() {
             atm.initiate_shutdown();
         }
@@ -194,20 +210,12 @@ impl Worker {
         telem_instance: Option<&TelemetryInstance>,
     ) -> Self {
         info!(task_queue=%config.task_queue, namespace=%config.namespace, "Initializing worker");
-        let metrics = if let Some(ti) = telem_instance {
-            MetricsContext::top_level(config.namespace.clone(), ti)
-                .with_task_q(config.task_queue.clone())
-        } else {
-            MetricsContext::no_op()
-        };
-        metrics.worker_registered();
 
         Self::new_with_pollers(
             config,
             sticky_queue_name,
             client,
             TaskPollers::Real,
-            metrics,
             telem_instance,
         )
     }
@@ -223,9 +231,14 @@ impl Worker {
         sticky_queue_name: Option<String>,
         client: Arc<dyn WorkerClient>,
         task_pollers: TaskPollers,
-        metrics: MetricsContext,
         telem_instance: Option<&TelemetryInstance>,
     ) -> Self {
+        let metrics = if let Some(ti) = telem_instance {
+            MetricsContext::top_level(config.namespace.clone(), config.task_queue.clone(), ti)
+        } else {
+            MetricsContext::no_op()
+        };
+        metrics.worker_registered();
         let shutdown_token = CancellationToken::new();
         let wft_semaphore = Arc::new(MeteredSemaphore::new(
             config.max_outstanding_workflow_tasks,
@@ -237,7 +250,7 @@ impl Worker {
             metrics.with_new_attrs([activity_worker_type()]),
             MetricsContext::available_task_slots,
         ));
-
+        let (external_wft_tx, external_wft_rx) = unbounded_channel();
         let (wft_stream, act_poller) = match task_pollers {
             TaskPollers::Real => {
                 let max_nonsticky_polls = if sticky_queue_name.is_some() {
@@ -299,6 +312,15 @@ impl Worker {
                     sticky_queue_poller,
                 ));
                 let wft_stream = new_wft_poller(wf_task_poll_buffer, metrics.clone());
+                let wft_stream = if !client.is_mock() {
+                    // Some replay tests combine a mock client with real pollers,
+                    // and they don't need to use the external stream
+                    stream::select(wft_stream, UnboundedReceiverStream::new(external_wft_rx))
+                        .left_stream()
+                } else {
+                    wft_stream.right_stream()
+                };
+
                 #[cfg(test)]
                 let wft_stream = wft_stream.left_stream();
                 (wft_stream, act_poll_buffer)
@@ -342,6 +364,7 @@ impl Worker {
                 config.max_heartbeat_throttle_interval,
                 config.default_heartbeat_throttle_interval,
                 config.graceful_shutdown_period,
+                config.local_timeout_buffer_for_activities,
             )
         });
         let poll_on_non_local_activities = at_task_mgr.is_some();
@@ -349,7 +372,15 @@ impl Worker {
             info!("Activity polling is disabled for this worker");
         };
         let la_sink = LAReqSink::new(local_act_mgr.clone(), config.wf_state_inputs.clone());
+        let provider = SlotProvider::new(
+            config.namespace.clone(),
+            config.task_queue.clone(),
+            wft_semaphore.clone(),
+            external_wft_tx,
+        );
+        let worker_key = client.workers().register(Box::new(provider));
         Self {
+            worker_key,
             wf_client: client.clone(),
             workflows: Workflows::new(
                 build_wf_basics(

package/sdk-core/core/src/worker/slot_provider.rs

@@ -0,0 +1,175 @@
+//! This module implements traits defined in the client to dispatch a
+//! WFT to a worker bypassing the server.
+//! This enables latency optimizations such as Eager Workflow Start.
+
+use crate::{
+    abstractions::{MeteredSemaphore, OwnedMeteredSemPermit},
+    protosext::ValidPollWFTQResponse,
+    worker::workflow::wft_poller::validate_wft,
+};
+
+use std::sync::Arc;
+use temporal_client::{Slot as SlotTrait, SlotProvider as SlotProviderTrait};
+use temporal_sdk_core_protos::temporal::api::workflowservice::v1::PollWorkflowTaskQueueResponse;
+use tokio::sync::mpsc::UnboundedSender;
+use tonic::Status;
+
+type WFTStreamSender =
+    UnboundedSender<Result<(ValidPollWFTQResponse, OwnedMeteredSemPermit), Status>>;
+
+pub struct Slot {
+    permit: OwnedMeteredSemPermit,
+    external_wft_tx: WFTStreamSender,
+}
+
+impl Slot {
+    fn new(permit: OwnedMeteredSemPermit, external_wft_tx: WFTStreamSender) -> Self {
+        Self {
+            permit,
+            external_wft_tx,
+        }
+    }
+}
+
+impl SlotTrait for Slot {
+    fn schedule_wft(
+        self: Box<Self>,
+        task: PollWorkflowTaskQueueResponse,
+    ) -> Result<(), anyhow::Error> {
+        let wft = validate_wft(task)?;
+        self.external_wft_tx.send(Ok((wft, self.permit)))?;
+        Ok(())
+    }
+}
+
+#[derive(derive_more::DebugCustom)]
+#[debug(fmt = "SlotProvider {{ namespace:{namespace}, task_queue: {task_queue} }}")]
+pub struct SlotProvider {
+    namespace: String,
+    task_queue: String,
+    wft_semaphore: Arc<MeteredSemaphore>,
+    external_wft_tx: WFTStreamSender,
+}
+
+impl SlotProvider {
+    pub(crate) fn new(
+        namespace: String,
+        task_queue: String,
+        wft_semaphore: Arc<MeteredSemaphore>,
+        external_wft_tx: WFTStreamSender,
+    ) -> Self {
+        Self {
+            namespace,
+            task_queue,
+            wft_semaphore,
+            external_wft_tx,
+        }
+    }
+}
+
+impl SlotProviderTrait for SlotProvider {
+    fn namespace(&self) -> &str {
+        &self.namespace
+    }
+    fn task_queue(&self) -> &str {
+        &self.task_queue
+    }
+    fn try_reserve_wft_slot(&self) -> Option<Box<dyn SlotTrait + Send>> {
+        match self.wft_semaphore.try_acquire_owned().ok() {
+            Some(permit) => Some(Box::new(Slot::new(permit, self.external_wft_tx.clone()))),
+            None => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use temporal_sdk_core_protos::temporal::api::{
+        common::v1::{WorkflowExecution, WorkflowType},
+        history::v1::History,
+        taskqueue::v1::TaskQueue,
+    };
+    use tokio::sync::mpsc::unbounded_channel;
+
+    // make validate_wft() happy
+    fn new_validatable_response() -> PollWorkflowTaskQueueResponse {
+        PollWorkflowTaskQueueResponse {
+            workflow_execution_task_queue: Some(TaskQueue::default()),
+            workflow_execution: Some(WorkflowExecution::default()),
+            workflow_type: Some(WorkflowType::default()),
+            history: Some(History::default()),
+            ..Default::default()
+        }
+    }
+
+    #[tokio::test]
+    async fn slot_propagates_through_channel() {
+        let wft_semaphore = Arc::new(MeteredSemaphore::new(
+            2,
+            crate::MetricsContext::no_op(),
+            |_, _| {},
+        ));
+        let (external_wft_tx, mut external_wft_rx) = unbounded_channel();
+
+        let provider = SlotProvider::new(
+            "my_namespace".to_string(),
+            "my_queue".to_string(),
+            wft_semaphore,
+            external_wft_tx,
+        );
+
+        let slot = provider
+            .try_reserve_wft_slot()
+            .expect("failed to reserver slot");
+        let p = slot.schedule_wft(new_validatable_response());
+        assert!(p.is_ok());
+        assert!(external_wft_rx.recv().await.is_some());
+    }
+
+    #[tokio::test]
+    async fn channel_closes_when_provider_drops() {
+        let (external_wft_tx, mut external_wft_rx) = unbounded_channel();
+        {
+            let external_wft_tx = external_wft_tx;
+            let wft_semaphore = Arc::new(MeteredSemaphore::new(
+                2,
+                crate::MetricsContext::no_op(),
+                |_, _| {},
+            ));
+            let provider = SlotProvider::new(
+                "my_namespace".to_string(),
+                "my_queue".to_string(),
+                wft_semaphore,
+                external_wft_tx,
+            );
+            assert!(provider.try_reserve_wft_slot().is_some());
+        }
+        assert!(external_wft_rx.recv().await.is_none());
+    }
+
+    #[tokio::test]
+    async fn unused_slots_reclaimed() {
+        let wft_semaphore = Arc::new(MeteredSemaphore::new(
+            2,
+            crate::MetricsContext::no_op(),
+            |_, _| {},
+        ));
+        {
+            let wft_semaphore = wft_semaphore.clone();
+            let (external_wft_tx, _) = unbounded_channel();
+            let provider = SlotProvider::new(
+                "my_namespace".to_string(),
+                "my_queue".to_string(),
+                wft_semaphore.clone(),
+                external_wft_tx,
+            );
+            let slot = provider.try_reserve_wft_slot();
+            assert!(slot.is_some());
+            assert_eq!(wft_semaphore.available_permits(), 1);
+            // drop slot without using it
+        }
+        assert_eq!(wft_semaphore.available_permits(), 2);
+    }
+}

package/sdk-core/core/src/worker/workflow/driven_workflow.rs

@@ -1,34 +1,36 @@
-use crate::worker::workflow::{OutgoingJob, WFCommand, WorkflowStartedInfo};
+use crate::{
+    telemetry::VecDisplayer,
+    worker::workflow::{OutgoingJob, WFCommand, WorkflowStartedInfo},
+};
 use prost_types::Timestamp;
+use std::sync::mpsc::{self, Receiver, Sender};
 use temporal_sdk_core_protos::{
     coresdk::workflow_activation::{start_workflow_from_attribs, WorkflowActivationJob},
     temporal::api::history::v1::WorkflowExecutionStartedEventAttributes,
     utilities::TryIntoOrNone,
 };
 
-/// Abstracts away the concept of an actual workflow implementation, handling sending it new
-/// jobs and fetching output from it.
+/// Represents a connection to a lang side workflow that can have activations fed into it and
+/// command responses pulled out.
 pub struct DrivenWorkflow {
     started_attrs: Option<WorkflowStartedInfo>,
-    fetcher: Box<dyn WorkflowFetcher>,
+    incoming_commands: Receiver<Vec<WFCommand>>,
     /// Outgoing activation jobs that need to be sent to the lang sdk
     outgoing_wf_activation_jobs: Vec<OutgoingJob>,
 }
 
-impl<WF> From<Box<WF>> for DrivenWorkflow
-where
-    WF: WorkflowFetcher + 'static,
-{
-    fn from(wf: Box<WF>) -> Self {
-        Self {
-            started_attrs: None,
-            fetcher: wf,
-            outgoing_wf_activation_jobs: Default::default(),
-        }
-    }
-}
-
 impl DrivenWorkflow {
+    pub fn new() -> (Self, Sender<Vec<WFCommand>>) {
+        let (tx, rx) = mpsc::channel();
+        (
+            Self {
+                started_attrs: None,
+                incoming_commands: rx,
+                outgoing_wf_activation_jobs: vec![],
+            },
+            tx,
+        )
+    }
     /// Start the workflow
     pub fn start(
         &mut self,
@@ -72,24 +74,15 @@ impl DrivenWorkflow {
             .map(Into::into)
             .collect()
     }
-}
 
-impl WorkflowFetcher for DrivenWorkflow {
-    fn fetch_workflow_iteration_output(&mut self) -> Vec<WFCommand> {
-        self.fetcher.fetch_workflow_iteration_output()
-    }
-}
-
-/// Implementors of this trait represent a way to fetch output from executing/iterating some
-/// workflow code (or a mocked workflow).
-pub trait WorkflowFetcher: Send {
     /// Obtain any output from the workflow's recent execution(s). Because the lang sdk is
     /// responsible for calling workflow code as a result of receiving tasks from
-    /// [crate::Core::poll_task], we cannot directly iterate it here. Thus implementations of this
-    /// trait are expected to either buffer output or otherwise produce it on demand when this
-    /// function is called.
-    ///
-    /// In the case of the real [WorkflowBridge] implementation, commands are simply pulled from
-    /// a buffer that the language side sinks into when it calls [crate::Core::complete_task]
-    fn fetch_workflow_iteration_output(&mut self) -> Vec<WFCommand>;
+    /// [crate::Core::poll_task], we cannot directly iterate it here. Commands are simply pulled
+    /// from a buffer that the language side sinks into when it calls [crate::Core::complete_task]
+    pub fn fetch_workflow_iteration_output(&mut self) -> Vec<WFCommand> {
+        let in_cmds = self.incoming_commands.try_recv();
+        let in_cmds = in_cmds.unwrap_or_else(|_| vec![WFCommand::NoCommandsFromLang]);
+        debug!(in_cmds = %in_cmds.display(), "wf bridge iteration fetch");
+        in_cmds
+    }
 }

package/sdk-core/core/src/worker/workflow/history_update.rs

@@ -55,8 +55,10 @@ impl Debug for HistoryUpdate {
         if self.is_real() {
             write!(
                 f,
-                "HistoryUpdate(previous_started_event_id: {}, length: {}, first_event_id: {:?})",
+                "HistoryUpdate(previous_started_event_id: {}, started_id: {}, \
+                 length: {}, first_event_id: {:?})",
                 self.previous_wft_started_id,
+                self.wft_started_id,
                 self.events.len(),
                 self.events.first().map(|e| e.event_id)
             )
@@ -164,6 +166,7 @@ impl HistoryPaginator {
             legacy_query: wft.legacy_query,
             query_requests: wft.query_requests,
             update,
+            messages: wft.messages,
         };
         Ok((paginator, prepared))
     }

package/sdk-core/core/src/worker/workflow/machines/activity_state_machine.rs

@@ -804,87 +804,17 @@ fn convert_payloads(
 mod test {
     use super::*;
     use crate::{
-        internal_flags::InternalFlags, replay::TestHistoryBuilder, test_help::canned_histories,
-        worker::workflow::ManagedWFFunc,
+        internal_flags::InternalFlags,
+        replay::TestHistoryBuilder,
+        test_help::{build_fake_sdk, MockPollCfg, ResponseType},
     };
-    use rstest::{fixture, rstest};
     use std::{cell::RefCell, mem::discriminant, rc::Rc};
-    use temporal_sdk::{
-        ActivityOptions, CancellableFuture, WfContext, WorkflowFunction, WorkflowResult,
-    };
+    use temporal_sdk::{ActivityOptions, CancellableFuture, WfContext, WorkflowFunction};
     use temporal_sdk_core_protos::{
         coresdk::workflow_activation::{workflow_activation_job, WorkflowActivationJob},
-        DEFAULT_ACTIVITY_TYPE,
+        DEFAULT_WORKFLOW_TYPE,
     };
-
-    #[fixture]
-    fn activity_happy_hist() -> ManagedWFFunc {
-        let func = WorkflowFunction::new(activity_wf);
-        let t = canned_histories::single_activity("activity-id-1");
-        assert_eq!(2, t.get_full_history_info().unwrap().wf_task_count());
-        ManagedWFFunc::new(t, func, vec![])
-    }
-
-    #[fixture]
-    fn activity_failure_hist() -> ManagedWFFunc {
-        let func = WorkflowFunction::new(activity_wf);
-        let t = canned_histories::single_failed_activity("activity-id-1");
-        assert_eq!(2, t.get_full_history_info().unwrap().wf_task_count());
-        ManagedWFFunc::new(t, func, vec![])
-    }
-
-    async fn activity_wf(command_sink: WfContext) -> WorkflowResult<()> {
-        command_sink
-            .activity(ActivityOptions {
-                activity_id: Some("activity-id-1".to_string()),
-                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
-                ..Default::default()
-            })
-            .await;
-        Ok(().into())
-    }
-
-    #[rstest(
-        wfm,
-        case::success(activity_happy_hist()),
-        case::failure(activity_failure_hist())
-    )]
-    #[tokio::test]
-    async fn single_activity_inc(mut wfm: ManagedWFFunc) {
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 1);
-        assert_eq!(
-            commands[0].command_type,
-            CommandType::ScheduleActivityTask as i32
-        );
-
-        wfm.get_next_activation().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 1);
-        assert_eq!(
-            commands[0].command_type,
-            CommandType::CompleteWorkflowExecution as i32
-        );
-        wfm.shutdown().await.unwrap();
-    }
-
-    #[rstest(
-        wfm,
-        case::success(activity_happy_hist()),
-        case::failure(activity_failure_hist())
-    )]
-    #[tokio::test]
-    async fn single_activity_full(mut wfm: ManagedWFFunc) {
-        wfm.process_all_activations().await.unwrap();
-        let commands = wfm.get_server_commands().commands;
-        assert_eq!(commands.len(), 1);
-        assert_eq!(
-            commands[0].command_type,
-            CommandType::CompleteWorkflowExecution as i32
-        );
-        wfm.shutdown().await.unwrap();
-    }
+    use temporal_sdk_core_test_utils::interceptors::ActivationAssertionsInterceptor;
 
     #[tokio::test]
     async fn immediate_activity_cancelation() {
@@ -900,24 +830,36 @@ mod test {
         t.add_by_type(EventType::WorkflowExecutionStarted);
         t.add_full_wf_task();
         t.add_workflow_execution_completed();
-        let mut wfm = ManagedWFFunc::new(t, func, vec![]);
-
-        let activation = wfm.process_all_activations().await.unwrap();
-        wfm.get_server_commands();
-        assert_matches!(
-            activation.jobs.as_slice(),
-            [WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::ResolveActivity(
-                    ResolveActivity {
-                        result: Some(ActivityResolution {
-                            status: Some(activity_resolution::Status::Cancelled(_))
-                        }),
-                        ..
-                    }
-                )),
-            },]
-        );
-        wfm.shutdown().await.unwrap();
+        let mut worker = build_fake_sdk(MockPollCfg::from_resps(t, [ResponseType::AllHistory]));
+        worker.register_wf(DEFAULT_WORKFLOW_TYPE, func);
+
+        let mut aai = ActivationAssertionsInterceptor::default();
+        aai.then(|a| {
+            assert_matches!(
+                a.jobs.as_slice(),
+                [WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+                }]
+            )
+        });
+        aai.then(|a| {
+            assert_matches!(
+                a.jobs.as_slice(),
+                [WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::ResolveActivity(
+                        ResolveActivity {
+                            result: Some(ActivityResolution {
+                                status: Some(activity_resolution::Status::Cancelled(_))
+                            }),
+                            ..
+                        }
+                    )),
+                },]
+            )
+        });
+
+        worker.set_worker_interceptor(aai);
+        worker.run().await.unwrap();
     }
 
     #[test]