@temporalio/core-bridge 1.0.0 → 1.0.1

package/sdk-core/core/src/test_help/mod.rs

@@ -73,6 +73,8 @@ pub enum ResponseType {
     OneTask(usize),
     /// Waits until the future resolves before responding as `ToTaskNum` with the provided number
     UntilResolved(BoxFuture<'static, ()>, usize),
+    /// Waits until the future resolves before responding with the provided response
+    UntilResolvedRaw(BoxFuture<'static, ()>, PollWorkflowTaskQueueResponse),
     AllHistory,
     Raw(PollWorkflowTaskQueueResponse),
 }
@@ -81,6 +83,7 @@ pub enum HashableResponseType {
     ToTaskNum(usize),
     OneTask(usize),
     UntilResolved(usize),
+    UntilResolvedRaw(TaskToken),
     AllHistory,
     Raw(TaskToken),
 }
@@ -92,6 +95,9 @@ impl ResponseType {
             ResponseType::AllHistory => HashableResponseType::AllHistory,
             ResponseType::Raw(r) => HashableResponseType::Raw(r.task_token.clone().into()),
             ResponseType::UntilResolved(_, x) => HashableResponseType::UntilResolved(*x),
+            ResponseType::UntilResolvedRaw(_, r) => {
+                HashableResponseType::UntilResolvedRaw(r.task_token.clone().into())
+            }
         }
     }
 }
@@ -618,7 +624,11 @@ impl<T> From<T> for QueueResponse<T> {
 }
 impl From<QueueResponse<PollWorkflowTaskQueueResponse>> for ResponseType {
     fn from(qr: QueueResponse<PollWorkflowTaskQueueResponse>) -> Self {
-        ResponseType::Raw(qr.resp)
+        if let Some(du) = qr.delay_until {
+            ResponseType::UntilResolvedRaw(du, qr.resp)
+        } else {
+            ResponseType::Raw(qr.resp)
+        }
     }
 }
 impl<T> Deref for QueueResponse<T> {
@@ -636,13 +646,13 @@ impl<T> DerefMut for QueueResponse<T> {
 
 pub fn hist_to_poll_resp(
     t: &TestHistoryBuilder,
-    wf_id: String,
+    wf_id: impl Into<String>,
     response_type: ResponseType,
     task_queue: impl Into<String>,
 ) -> QueueResponse<PollWorkflowTaskQueueResponse> {
     let run_id = t.get_orig_run_id();
     let wf = WorkflowExecution {
-        workflow_id: wf_id,
+        workflow_id: wf_id.into(),
         run_id: run_id.to_string(),
     };
     let mut delay_until = None;
@@ -660,6 +670,12 @@ pub fn hist_to_poll_resp(
             delay_until = Some(fut);
             t.get_history_info(tn).unwrap()
         }
+        ResponseType::UntilResolvedRaw(fut, r) => {
+            return QueueResponse {
+                resp: r,
+                delay_until: Some(fut),
+            }
+        }
     };
     let mut resp = hist_info.as_poll_wft_response(task_queue);
     resp.workflow_execution = Some(wf);

package/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs

@@ -131,7 +131,7 @@ impl ActivityHeartbeatManager {
     /// Initiates shutdown procedure by stopping lifecycle loop and awaiting for all in-flight
     /// heartbeat requests to be flushed to the server.
     pub(super) async fn shutdown(&self) {
-        let _ = self.shutdown_token.cancel();
+        self.shutdown_token.cancel();
         let mut handle = self.join_handle.lock().await;
         if let Some(h) = handle.take() {
             let handle_r = h.await;
@@ -282,7 +282,7 @@ impl HeartbeatStreamState {
     ) -> Option<HeartbeatExecutorAction> {
         if let Some(state) = self.tt_to_state.remove(&tt) {
             if let Some(cancel_tok) = state.throttled_cancellation_token {
-                let _ = cancel_tok.cancel();
+                cancel_tok.cancel();
             }
             if let Some(last_deets) = state.last_recorded_details {
                 self.tt_needs_flush.insert(tt.clone(), on_complete);

package/sdk-core/core/src/worker/client.rs

@@ -6,7 +6,7 @@ use std::{
     borrow::Borrow,
     ops::{Deref, DerefMut},
 };
-use temporal_client::{WorkflowClientTrait, WorkflowTaskCompletion};
+use temporal_client::{WorkflowClientTrait, WorkflowTaskCompletion, RETRYABLE_ERROR_CODES};
 use temporal_sdk_core_protos::{
     coresdk::workflow_commands::QueryResult,
     temporal::api::{
@@ -15,9 +15,17 @@ use temporal_sdk_core_protos::{
     },
     TaskToken,
 };
+use tonic::Code;
 
 type Result<T, E = tonic::Status> = std::result::Result<T, E>;
 
+/// Returns true if the network error should not be reported to lang. This can happen if we've
+/// exceeded the max number of retries, and we prefer to just warn rather than blowing up lang.
+pub(crate) fn should_swallow_net_error(err: &tonic::Status) -> bool {
+    RETRYABLE_ERROR_CODES.contains(&err.code())
+        || matches!(err.code(), Code::Cancelled | Code::DeadlineExceeded)
+}
+
 /// Contains everything a worker needs to interact with the server
 pub(crate) struct WorkerClientBag {
     client: Box<dyn WorkerClient>,

package/sdk-core/core/src/worker/mod.rs

@@ -18,14 +18,14 @@ use crate::{
         new_activity_task_buffer, new_workflow_task_buffer, BoxedActPoller, Poller,
         WorkflowTaskPoller,
     },
-    protosext::ValidPollWFTQResponse,
+    protosext::{validate_activity_completion, ValidPollWFTQResponse},
     telemetry::metrics::{
         activity_poller, local_activity_worker_type, workflow_poller, workflow_sticky_poller,
         MetricsContext,
     },
     worker::{
         activities::{DispatchOrTimeoutLA, LACompleteAction, LocalActivityManager},
-        client::WorkerClientBag,
+        client::{should_swallow_net_error, WorkerClientBag},
         workflow::{LocalResolution, Workflows},
     },
     ActivityHeartbeat, CompleteActivityError, PollActivityError, PollWfError, WorkerTrait,
@@ -398,6 +398,7 @@ impl Worker {
         task_token: TaskToken,
         status: activity_execution_result::Status,
     ) -> Result<(), CompleteActivityError> {
+        validate_activity_completion(&status)?;
         if task_token.is_local_activity_task() {
             let as_la_res: LocalActivityExecutionResult = status.try_into()?;
             match self.local_act_mgr.complete(&task_token, &as_la_res) {
@@ -408,7 +409,7 @@ impl Worker {
                     // no other situations where core generates "internal" commands so it is much
                     // simpler for lang to reply with the timer / next LA command than to do it
                     // internally. Plus, this backoff hack we'd like to eliminate eventually.
-                    self.complete_local_act(as_la_res, info, Some(backoff))
+                    self.complete_local_act(as_la_res, info, Some(backoff));
                 }
                 LACompleteAction::WillBeRetried => {
                     // Nothing to do here
@@ -421,7 +422,13 @@ impl Worker {
         }
 
         if let Some(atm) = &self.at_task_mgr {
-            atm.complete(task_token, status, &**self.wf_client).await
+            match atm.complete(task_token, status, &**self.wf_client).await {
+                Err(CompleteActivityError::TonicError(e)) if should_swallow_net_error(&e) => {
+                    warn!(error=?e, "Network error while completing activity");
+                    Ok(())
+                }
+                o => o,
+            }
         } else {
             error!(
                 "Tried to complete activity {} on a worker that does not have an activity manager",

package/sdk-core/core/src/worker/workflow/driven_workflow.rs

@@ -1,5 +1,4 @@
 use crate::worker::workflow::{WFCommand, WorkflowStartedInfo};
-use std::collections::VecDeque;
 use temporal_sdk_core_protos::{
     coresdk::workflow_activation::{
         start_workflow_from_attribs, workflow_activation_job, CancelWorkflow, SignalWorkflow,
@@ -15,7 +14,7 @@ pub struct DrivenWorkflow {
     started_attrs: Option<WorkflowStartedInfo>,
     fetcher: Box<dyn WorkflowFetcher>,
     /// Outgoing activation jobs that need to be sent to the lang sdk
-    outgoing_wf_activation_jobs: VecDeque<workflow_activation_job::Variant>,
+    outgoing_wf_activation_jobs: Vec<workflow_activation_job::Variant>,
 }
 
 impl<WF> From<Box<WF>> for DrivenWorkflow
@@ -58,12 +57,12 @@ impl DrivenWorkflow {
 
     /// Enqueue a new job to be sent to the driven workflow
     pub fn send_job(&mut self, job: workflow_activation_job::Variant) {
-        self.outgoing_wf_activation_jobs.push_back(job);
+        self.outgoing_wf_activation_jobs.push(job);
     }
 
-    /// Check if there are pending jobs
-    pub fn has_pending_jobs(&self) -> bool {
-        !self.outgoing_wf_activation_jobs.is_empty()
+    /// Observe pending jobs
+    pub fn peek_pending_jobs(&self) -> &[workflow_activation_job::Variant] {
+        self.outgoing_wf_activation_jobs.as_slice()
     }
 
     /// Drain all pending jobs, so that they may be sent to the driven workflow

package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs

@@ -584,7 +584,7 @@ impl WorkflowMachines {
     }
 
     pub(crate) fn has_pending_jobs(&self) -> bool {
-        self.drive_me.has_pending_jobs()
+        !self.drive_me.peek_pending_jobs().is_empty()
     }
 
     fn set_current_time(&mut self, time: SystemTime) -> SystemTime {

package/sdk-core/core/src/worker/workflow/managed_run.rs

@@ -111,6 +111,7 @@ impl ManagedRun {
         .fold((self, heartbeat_tx), |(mut me, heartbeat_tx), action| {
             let span = action.trace_span;
             let action = action.action;
+            let mut no_wft = false;
             async move {
                 let res = match action {
                     RunActions::NewIncomingWFT(wft) => me
@@ -124,10 +125,15 @@ impl ManagedRun {
                     RunActions::CheckMoreWork {
                         want_to_evict,
                         has_pending_queries,
-                    } => me
-                        .check_more_work(want_to_evict, has_pending_queries)
-                        .await
-                        .map(RunActionOutcome::AfterCheckWork),
+                        has_wft,
+                    } => {
+                        if !has_wft {
+                            no_wft = true;
+                        }
+                        me.check_more_work(want_to_evict, has_pending_queries, has_wft)
+                            .await
+                            .map(RunActionOutcome::AfterCheckWork)
+                    }
                     RunActions::LocalResolution(r) => me
                         .local_resolution(r)
                         .await
@@ -145,7 +151,7 @@ impl ManagedRun {
                 };
                 match res {
                     Ok(outcome) => {
-                        me.send_update_response(outcome);
+                        me.send_update_response(outcome, no_wft);
                     }
                     Err(e) => {
                         error!(error=?e, "Error in run machines");
@@ -308,7 +314,12 @@ impl ManagedRun {
         &mut self,
         want_to_evict: Option<RequestEvictMsg>,
         has_pending_queries: bool,
+        has_wft: bool,
     ) -> Result<Option<ActivationOrAuto>, RunUpdateErr> {
+        if !has_wft {
+            // It doesn't make sense to do work unless we have a WFT
+            return Ok(None);
+        }
         if self.wfm.machines.has_pending_jobs() && !self.am_broken {
             Ok(Some(ActivationOrAuto::LangActivation(
                 self.wfm.get_next_activation().await?,
@@ -427,7 +438,7 @@ impl ManagedRun {
         false
     }
 
-    fn send_update_response(&self, outcome: RunActionOutcome) {
+    fn send_update_response(&self, outcome: RunActionOutcome, no_wft: bool) {
         let mut in_response_to_wft = false;
         let (outgoing_activation, fulfillable_complete) = match outcome {
             RunActionOutcome::AfterNewWFT(a) => {
@@ -439,7 +450,15 @@ impl ManagedRun {
             RunActionOutcome::AfterCompletion(f) => (None, f),
             RunActionOutcome::AfterHeartbeatTimeout(a) => (a, None),
         };
-
+        let mut more_pending_work = self.wfm.machines.has_pending_jobs();
+        // We don't want to consider there to be more local-only work to be done if there is no
+        // workflow task associated with the run right now. This can happen if, ex, we complete
+        // a local activity while waiting for server to send us the next WFT. Activating lang would
+        // be harmful at this stage, as there might be work returned in that next WFT which should
+        // be part of the next activation.
+        if no_wft {
+            more_pending_work = false;
+        }
         self.update_tx
             .send(RunUpdateResponse {
                 kind: RunUpdateResponseKind::Good(GoodRunUpdate {
@@ -447,7 +466,7 @@ impl ManagedRun {
                     outgoing_activation,
                     fulfillable_complete,
                     have_seen_terminal_event: self.wfm.machines.have_seen_terminal_event,
-                    more_pending_work: self.wfm.machines.has_pending_jobs(),
+                    more_pending_work,
                     most_recently_processed_event_number: self.wfm.machines.last_processed_event
                         as usize,
                     in_response_to_wft,

package/sdk-core/core/src/worker/workflow/mod.rs

@@ -24,6 +24,7 @@ use crate::{
     telemetry::VecDisplayer,
     worker::{
         activities::{ActivitiesFromWFTsHandle, PermittedTqResp},
+        client::should_swallow_net_error,
         workflow::{
             managed_run::{ManagedRun, WorkflowManager},
             wft_poller::validate_wft,
@@ -234,7 +235,7 @@ impl Workflows {
                     let reserved_act_permits =
                         self.reserve_activity_slots_for_outgoing_commands(commands.as_mut_slice());
                     debug!(commands=%commands.display(), query_responses=%query_responses.display(),
-                           "Sending responses to server");
+                           force_new_wft, "Sending responses to server");
                     let mut completion = WorkflowTaskCompletion {
                         task_token,
                         commands,
@@ -362,20 +363,26 @@ impl Workflows {
         let mut should_evict = None;
         let res = match completer().await {
             Err(err) => {
-                match err.code() {
-                    // Silence unhandled command errors since the lang SDK cannot do anything about
-                    // them besides poll again, which it will do anyway.
-                    tonic::Code::InvalidArgument if err.message() == "UnhandledCommand" => {
-                        debug!(error = %err, run_id, "Unhandled command response when completing");
-                        should_evict = Some(EvictionReason::UnhandledCommand);
-                        Ok(())
-                    }
-                    tonic::Code::NotFound => {
-                        warn!(error = %err, run_id, "Task not found when completing");
-                        should_evict = Some(EvictionReason::TaskNotFound);
-                        Ok(())
+                if should_swallow_net_error(&err) {
+                    warn!(error= %err, "Network error while completing workflow activation");
+                    should_evict = Some(EvictionReason::Fatal);
+                    Ok(())
+                } else {
+                    match err.code() {
+                        // Silence unhandled command errors since the lang SDK cannot do anything
+                        // about them besides poll again, which it will do anyway.
+                        tonic::Code::InvalidArgument if err.message() == "UnhandledCommand" => {
+                            debug!(error = %err, run_id, "Unhandled command response when completing");
+                            should_evict = Some(EvictionReason::UnhandledCommand);
+                            Ok(())
+                        }
+                        tonic::Code::NotFound => {
+                            warn!(error = %err, run_id, "Task not found when completing");
+                            should_evict = Some(EvictionReason::TaskNotFound);
+                            Ok(())
+                        }
+                        _ => Err(err),
                     }
-                    _ => Err(err),
                 }
             }
             _ => Ok(()),
@@ -486,8 +493,12 @@ impl Workflows {
     ) -> Result<(), tonic::Status> {
         match self.client.respond_legacy_query(tt, res).await {
             Ok(_) => Ok(()),
+            Err(e) if should_swallow_net_error(&e) => {
+                warn!(error= %e, "Network error while responding to legacy query");
+                Ok(())
+            }
             Err(e) if e.code() == tonic::Code::NotFound => {
-                warn!(error=?e,"Query not found when attempting to respond to it");
+                warn!(error=?e, "Query not found when attempting to respond to it");
                 Ok(())
             }
             Err(e) => Err(e),
@@ -497,7 +508,20 @@ impl Workflows {
 
 /// Manages access to a specific workflow run, and contains various bookkeeping information that the
 /// [WFStream] may need to access quickly.
-#[derive(Debug)]
+#[derive(derive_more::DebugCustom)]
+#[debug(
+    fmt = "ManagedRunHandle {{ wft: {:?}, activation: {:?}, buffered_resp: {:?} \
+           have_seen_terminal_event: {}, most_recently_processed_event: {}, more_pending_work: {}, \
+           trying_to_evict: {}, last_action_acked: {} }}",
+    wft,
+    activation,
+    buffered_resp,
+    have_seen_terminal_event,
+    most_recently_processed_event_number,
+    more_pending_work,
+    "trying_to_evict.is_some()",
+    last_action_acked
+)]
 struct ManagedRunHandle {
     /// If set, the WFT this run is currently/will be processing.
     wft: Option<OutstandingTask>,
@@ -567,6 +591,7 @@ impl ManagedRunHandle {
                     .as_ref()
                     .map(|wft| !wft.pending_queries.is_empty())
                     .unwrap_or_default(),
+                has_wft: self.wft.is_some(),
             });
         }
     }
@@ -677,7 +702,8 @@ impl ActivationOrAuto {
     }
 }
 
-#[derive(Debug)]
+#[derive(derive_more::DebugCustom)]
+#[debug(fmt = "PermittedWft {{ {:?} }}", wft)]
 pub(crate) struct PermittedWFT {
     wft: ValidPollWFTQResponse,
     permit: OwnedMeteredSemPermit,
@@ -924,6 +950,7 @@ enum RunActions {
     CheckMoreWork {
         want_to_evict: Option<RequestEvictMsg>,
         has_pending_queries: bool,
+        has_wft: bool,
     },
     LocalResolution(LocalResolution),
     HeartbeatTimeout,

package/sdk-core/core/src/worker/workflow/workflow_stream.rs

@@ -45,7 +45,7 @@ pub(crate) struct WFStream {
     metrics: MetricsContext,
 }
 /// All possible inputs to the [WFStream]
-#[derive(derive_more::From)]
+#[derive(derive_more::From, Debug)]
 enum WFStreamInput {
     NewWft(PermittedWFT),
     Local(LocalInput),
@@ -64,6 +64,8 @@ impl From<RunUpdateResponse> for WFStreamInput {
     }
 }
 /// A non-poller-received input to the [WFStream]
+#[derive(derive_more::DebugCustom)]
+#[debug(fmt = "LocalInput {{ {:?} }}", input)]
 pub(super) struct LocalInput {
     pub input: LocalInputs,
     pub span: Span,
@@ -167,7 +169,7 @@ impl WFStream {
         };
         all_inputs
             .map(move |action| {
-                let span = span!(Level::DEBUG, "new_stream_input");
+                let span = span!(Level::DEBUG, "new_stream_input", action=?action);
                 let _span_g = span.enter();
 
                 let maybe_activation = match action {
@@ -209,7 +211,7 @@ impl WFStream {
                         }
                     }
                     WFStreamInput::PollerDead => {
-                        warn!("WFT poller died, shutting down");
+                        debug!("WFT poller died, shutting down");
                         state.shutdown_token.cancel();
                         None
                     }
@@ -259,17 +261,15 @@ impl WFStream {
         debug!(resp=%resp, "Processing run update response from machines");
         match resp {
             RunUpdateResponseKind::Good(mut resp) => {
-                if let Some(r) = self.runs.get_mut(&resp.run_id) {
-                    r.have_seen_terminal_event = resp.have_seen_terminal_event;
-                    r.more_pending_work = resp.more_pending_work;
-                    r.last_action_acked = true;
-                    r.most_recently_processed_event_number =
-                        resp.most_recently_processed_event_number;
-                }
                 let run_handle = self
                     .runs
                     .get_mut(&resp.run_id)
                     .expect("Workflow must exist, it just sent us an update response");
+                run_handle.have_seen_terminal_event = resp.have_seen_terminal_event;
+                run_handle.more_pending_work = resp.more_pending_work;
+                run_handle.last_action_acked = true;
+                run_handle.most_recently_processed_event_number =
+                    resp.most_recently_processed_event_number;
 
                 let r = match resp.outgoing_activation {
                     Some(ActivationOrAuto::LangActivation(mut activation)) => {
@@ -304,18 +304,20 @@ impl WFStream {
                     None => {
                         // If the response indicates there is no activation to send yet but there
                         // is more pending work, we should check again.
-                        if resp.more_pending_work {
+                        if run_handle.more_pending_work {
                             run_handle.check_more_activations();
                             None
                         } else if let Some(reason) = run_handle.trying_to_evict.as_ref() {
                             // If a run update came back and had nothing to do, but we're trying to
                             // evict, just do that now as long as there's no other outstanding work.
                             if run_handle.activation.is_none() && !run_handle.more_pending_work {
-                                let evict_act = create_evict_activation(
+                                let mut evict_act = create_evict_activation(
                                     resp.run_id,
                                     reason.message.clone(),
                                     reason.reason,
                                 );
+                                evict_act.history_length =
+                                    run_handle.most_recently_processed_event_number as u32;
                                 Some(ActivationOrAuto::LangActivation(evict_act))
                             } else {
                                 None

package/sdk-core/sdk/src/lib.rs

@@ -205,10 +205,10 @@ impl Worker {
 
     /// Register an Activity function to invoke when the Worker is asked to run an activity of
     /// `activity_type`
-    pub fn register_activity<A, R>(
+    pub fn register_activity<A, R, O>(
         &mut self,
         activity_type: impl Into<String>,
-        act_function: impl IntoActivityFunc<A, R>,
+        act_function: impl IntoActivityFunc<A, R, O>,
     ) {
         self.activity_half.activity_fns.insert(
             activity_type.into(),
@@ -485,7 +485,10 @@ impl ActivityHalf {
                 tokio::spawn(async move {
                     let output = (act_fn.act_func)(ctx, arg).await;
                     let result = match output {
-                        Ok(res) => ActivityExecutionResult::ok(res),
+                        Ok(ActExitValue::Normal(p)) => ActivityExecutionResult::ok(p),
+                        Ok(ActExitValue::WillCompleteAsync) => {
+                            ActivityExecutionResult::will_complete_async()
+                        }
                         Err(err) => match err.downcast::<ActivityCancelledError>() {
                             Ok(ce) => ActivityExecutionResult::cancel_from_details(ce.details),
                             Err(other_err) => ActivityExecutionResult::fail(other_err.into()),
@@ -497,7 +500,7 @@ impl ActivityHalf {
                             result: Some(result),
                         })
                         .await?;
-                    Result::<_, anyhow::Error>::Ok(())
+                    Ok::<_, anyhow::Error>(())
                 });
             }
             Some(activity_task::Variant::Cancel(_)) => {
@@ -716,9 +719,22 @@ impl<T: Debug> WfExitValue<T> {
     }
 }
 
+/// Activity functions may return these values when exiting
+#[derive(Debug, derive_more::From)]
+pub enum ActExitValue<T: Debug> {
+    /// Completion requires an asynchronous callback
+    #[from(ignore)]
+    WillCompleteAsync,
+    /// Finish with a result
+    Normal(T),
+}
+
 type BoxActFn = Arc<
-    dyn Fn(ActContext, Payload) -> BoxFuture<'static, Result<Payload, anyhow::Error>> + Send + Sync,
+    dyn Fn(ActContext, Payload) -> BoxFuture<'static, Result<ActExitValue<Payload>, anyhow::Error>>
+        + Send
+        + Sync,
 >;
+
 /// Container for user-defined activity functions
 #[derive(Clone)]
 pub struct ActivityFunction {
@@ -738,24 +754,35 @@ impl Display for ActivityCancelledError {
 }
 
 /// Closures / functions which can be turned into activity functions implement this trait
-pub trait IntoActivityFunc<Args, Res> {
+pub trait IntoActivityFunc<Args, Res, Out> {
     /// Consume the closure or fn pointer and turned it into a boxed activity function
     fn into_activity_fn(self) -> BoxActFn;
 }
 
-impl<A, Rf, R, F> IntoActivityFunc<A, Rf> for F
+impl<A, Rf, R, O, F> IntoActivityFunc<A, Rf, O> for F
 where
     F: (Fn(ActContext, A) -> Rf) + Sync + Send + 'static,
     A: FromJsonPayloadExt + Send,
     Rf: Future<Output = Result<R, anyhow::Error>> + Send + 'static,
-    R: AsJsonPayloadExt,
+    R: Into<ActExitValue<O>>,
+    O: AsJsonPayloadExt + Debug,
 {
     fn into_activity_fn(self) -> BoxActFn {
         let wrapper = move |ctx: ActContext, input: Payload| {
             // Some minor gymnastics are required to avoid needing to clone the function
             match A::from_json_payload(&input) {
                 Ok(deser) => (self)(ctx, deser)
-                    .map(|r| r.map(|r| r.as_json_payload())?)
+                    .map(|r| {
+                        r.and_then(|r| {
+                            let exit_val: ActExitValue<O> = r.into();
+                            Ok(match exit_val {
+                                ActExitValue::WillCompleteAsync => ActExitValue::WillCompleteAsync,
+                                ActExitValue::Normal(x) => {
+                                    ActExitValue::Normal(x.as_json_payload()?)
+                                }
+                            })
+                        })
+                    })
                     .boxed(),
                 Err(e) => async move { Err(e.into()) }.boxed(),
             }

package/sdk-core/sdk-core-protos/src/history_builder.rs

@@ -264,7 +264,7 @@ impl TestHistoryBuilder {
         self.build_and_push_event(EventType::MarkerRecorded, attrs.into());
     }
 
-    fn add_local_activity_marker(
+    pub fn add_local_activity_marker(
         &mut self,
         seq: u32,
         activity_id: &str,

package/sdk-core/test-utils/src/lib.rs

@@ -29,7 +29,7 @@ use temporal_sdk_core_protos::{
     coresdk::{
         workflow_commands::{
             workflow_command, ActivityCancellationType, CompleteWorkflowExecution,
-            ScheduleActivity, StartTimer,
+            ScheduleActivity, ScheduleLocalActivity, StartTimer,
         },
         workflow_completion::WorkflowActivationCompletion,
     },
@@ -290,10 +290,10 @@ impl TestWorker {
         self.inner.register_wf(workflow_type, wf_function)
     }
 
-    pub fn register_activity<A, R>(
+    pub fn register_activity<A, R, O>(
         &mut self,
         activity_type: impl Into<String>,
-        act_function: impl IntoActivityFunc<A, R>,
+        act_function: impl IntoActivityFunc<A, R, O>,
     ) {
         self.inner.register_activity(activity_type, act_function)
     }
@@ -493,6 +493,25 @@ pub fn schedule_activity_cmd(
     .into()
 }
 
+pub fn schedule_local_activity_cmd(
+    seq: u32,
+    activity_id: &str,
+    cancellation_type: ActivityCancellationType,
+    activity_timeout: Duration,
+) -> workflow_command::Variant {
+    ScheduleLocalActivity {
+        seq,
+        activity_id: activity_id.to_string(),
+        activity_type: "test_activity".to_string(),
+        schedule_to_start_timeout: Some(activity_timeout.into()),
+        start_to_close_timeout: Some(activity_timeout.into()),
+        schedule_to_close_timeout: Some(activity_timeout.into()),
+        cancellation_type: cancellation_type as i32,
+        ..Default::default()
+    }
+    .into()
+}
+
 pub fn start_timer_cmd(seq: u32, duration: Duration) -> workflow_command::Variant {
     StartTimer {
         seq,