@temporalio/core-bridge 1.11.1 → 1.11.2

package/sdk-core/tests/integ_tests/metrics_tests.rs

@@ -1,11 +1,16 @@
 use assert_matches::assert_matches;
-use std::{net::SocketAddr, sync::Arc, time::Duration};
+use std::{env, net::SocketAddr, sync::Arc, time::Duration};
 use temporal_client::{WorkflowClientTrait, WorkflowOptions, WorkflowService};
-use temporal_sdk_core::{init_worker, telemetry::start_prometheus_metric_exporter, CoreRuntime};
+use temporal_sdk_core::{
+    init_worker,
+    telemetry::{build_otlp_metric_exporter, start_prometheus_metric_exporter},
+    CoreRuntime,
+};
 use temporal_sdk_core_api::{
     telemetry::{
         metrics::{CoreMeter, MetricAttributes, MetricParameters},
-        PrometheusExporterOptionsBuilder, TelemetryOptions,
+        OtelCollectorOptionsBuilder, PrometheusExporterOptionsBuilder, TelemetryOptions,
+        TelemetryOptionsBuilder,
     },
     worker::WorkerConfigBuilder,
     Worker,
@@ -30,9 +35,10 @@ use temporal_sdk_core_protos::{
     },
 };
 use temporal_sdk_core_test_utils::{
-    get_integ_server_options, get_integ_telem_options, CoreWfStarter, NAMESPACE,
+    get_integ_server_options, get_integ_telem_options, CoreWfStarter, NAMESPACE, OTEL_URL_ENV_VAR,
 };
 use tokio::{join, sync::Barrier, task::AbortHandle};
+use url::Url;
 
 static ANY_PORT: &str = "127.0.0.1:0";
 
@@ -376,6 +382,7 @@ async fn query_of_closed_workflow_doesnt_tick_terminal_metric(
         .complete_workflow_activation(WorkflowActivationCompletion::fail(
             task.run_id,
             "whatever".into(),
+            None,
         ))
         .await
         .unwrap();
@@ -574,3 +581,42 @@ async fn request_fail_codes() {
     assert!(matching_line.contains("status_code=\"INVALID_ARGUMENT\""));
     assert!(matching_line.contains("} 1"));
 }
+
+// OTel collector shutdown hangs in a single-threaded Tokio environment. We used to, in the past
+// have a dedicated runtime just for telemetry which was meant to address problems like this.
+// In reality, users are unlikely to run a single-threaded runtime.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn request_fail_codes_otel() {
+    let exporter = if let Some(url) = env::var(OTEL_URL_ENV_VAR)
+        .ok()
+        .map(|x| x.parse::<Url>().unwrap())
+    {
+        let opts = OtelCollectorOptionsBuilder::default()
+            .url(url)
+            .build()
+            .unwrap();
+        build_otlp_metric_exporter(opts).unwrap()
+    } else {
+        // skip
+        return;
+    };
+    let mut telemopts = TelemetryOptionsBuilder::default();
+    let exporter = Arc::new(exporter);
+    telemopts.metrics(exporter as Arc<dyn CoreMeter>);
+
+    let rt = CoreRuntime::new_assume_tokio(telemopts.build().unwrap()).unwrap();
+    let opts = get_integ_server_options();
+    let mut client = opts
+        .connect(NAMESPACE, rt.telemetry().get_temporal_metric_meter())
+        .await
+        .unwrap();
+
+    for _ in 0..10 {
+        // Describe namespace w/ invalid argument (unset namespace field)
+        WorkflowService::describe_namespace(&mut client, DescribeNamespaceRequest::default())
+            .await
+            .unwrap_err();
+
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+}

package/sdk-core/tests/integ_tests/queries_tests.rs

@@ -12,7 +12,7 @@ use temporal_sdk_core_protos::{
     temporal::api::{failure::v1::Failure, query::v1::WorkflowQuery},
 };
 use temporal_sdk_core_test_utils::{
-    drain_pollers_and_shutdown, init_core_and_create_wf, WorkerTestHelpers,
+    drain_pollers_and_shutdown, init_core_and_create_wf, CoreWfStarter, WorkerTestHelpers,
 };
 use tokio::join;
 
@@ -163,7 +163,7 @@ async fn query_after_execution_complete(#[case] do_evict: bool) {
             if matches!(
                 task.jobs.as_slice(),
                 [WorkflowActivationJob {
-                    variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+                    variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
                 }]
             ) {
                 core.complete_timer(&task.run_id, 1, Duration::from_millis(500))
@@ -215,30 +215,15 @@ async fn query_after_execution_complete(#[case] do_evict: bool) {
 #[tokio::test]
 async fn fail_legacy_query() {
     let query_err = "oh no broken";
-    let mut starter = init_core_and_create_wf("fail_legacy_query").await;
+    let mut starter = CoreWfStarter::new("fail_legacy_query");
     let core = starter.get_worker().await;
+    starter.workflow_options.task_timeout = Some(Duration::from_secs(1));
+    starter.start_wf().await;
     let workflow_id = starter.get_task_queue().to_string();
     let task = core.poll_workflow_activation().await.unwrap();
-    let t1_resp = vec![
-        StartTimer {
-            seq: 1,
-            start_to_fire_timeout: Some(prost_dur!(from_millis(500))),
-        }
-        .into(),
-        StartTimer {
-            seq: 2,
-            start_to_fire_timeout: Some(prost_dur!(from_secs(3))),
-        }
-        .into(),
-    ];
-    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
-        task.run_id.clone(),
-        t1_resp.clone(),
-    ))
-    .await
-    .unwrap();
-    tokio::time::sleep(Duration::from_secs(1)).await;
-    // Query after timer should have fired and there should be new WFT
+    // Queries are *always* legacy on closed workflows, so that's the easiest way to ensure that
+    // path is used.
+    core.complete_execution(&task.run_id).await;
     let query_fut = async {
         starter
             .get_client()
@@ -255,62 +240,26 @@ async fn fail_legacy_query() {
             .await
             .unwrap_err()
     };
-    let workflow_completions_future = async {
-        // Give query a beat to get going
-        tokio::time::sleep(Duration::from_millis(400)).await;
-        // This poll *should* have the `queries` field populated, but doesn't, seemingly due to
-        // a server bug. So, complete the WF task of the first timer firing with empty commands
-        let task = core.poll_workflow_activation().await.unwrap();
-        assert_matches!(
-            task.jobs.as_slice(),
-            [WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::FireTimer(_)),
-            }]
-        );
-        core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
-            task.run_id,
-            vec![],
-        ))
-        .await
-        .unwrap();
+    let query_responder = async {
         let task = core.poll_workflow_activation().await.unwrap();
-        // Poll again, and we end up getting a `query` field query response
         assert_matches!(
             task.jobs.as_slice(),
             [WorkflowActivationJob {
                 variant: Some(workflow_activation_job::Variant::QueryWorkflow(q)),
             }] => q
         );
-        // Fail this task
         core.complete_workflow_activation(WorkflowActivationCompletion::fail(
             task.run_id,
             Failure {
                 message: query_err.to_string(),
                 ..Default::default()
             },
+            None,
         ))
         .await
         .unwrap();
-        // Finish the workflow (handling cache removal)
-        let task = core.poll_workflow_activation().await.unwrap();
-        core.complete_workflow_activation(WorkflowActivationCompletion::empty(task.run_id))
-            .await
-            .unwrap();
-        let task = core.poll_workflow_activation().await.unwrap();
-        core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
-            task.run_id,
-            t1_resp.clone(),
-        ))
-        .await
-        .unwrap();
-        let task = core.poll_workflow_activation().await.unwrap();
-        core.complete_workflow_activation(WorkflowActivationCompletion::empty(task.run_id))
-            .await
-            .unwrap();
-        let task = core.poll_workflow_activation().await.unwrap();
-        core.complete_execution(&task.run_id).await;
     };
-    let (q_resp, _) = join!(query_fut, workflow_completions_future);
+    let (q_resp, _) = join!(query_fut, query_responder);
     // Ensure query response is a failure and has the right message
     assert_eq!(q_resp.message(), query_err);
 }
@@ -483,3 +432,87 @@ async fn queries_handled_before_next_wft() {
     join!(join_all(query_futs), complete_fut);
     drain_pollers_and_shutdown(&core).await;
 }
+
+#[tokio::test]
+async fn query_should_not_be_sent_if_wft_about_to_fail() {
+    let mut starter =
+        init_core_and_create_wf("query_should_not_be_sent_if_wft_about_to_fail").await;
+    let core = starter.get_worker().await;
+    let workflow_id = starter.get_task_queue().to_string();
+    let client = starter.get_client().await;
+    // query straight away
+    let query_fut = client.query_workflow_execution(
+        workflow_id.to_string(),
+        "".to_string(),
+        WorkflowQuery {
+            query_type: "myquery".to_string(),
+            ..Default::default()
+        },
+    );
+    // Poll for the task and respond with a task failure
+    let poll_and_fail_fut = async {
+        let task = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            task.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
+            }]
+        );
+        core.complete_workflow_activation(WorkflowActivationCompletion::fail(
+            task.run_id,
+            Failure {
+                message: "oh no".to_string(),
+                ..Default::default()
+            },
+            None,
+        ))
+        .await
+        .unwrap();
+        let task = core.poll_workflow_activation().await.unwrap();
+        // Should *not* get a query here. If the bug wasn't fixed, this job would have a query.
+        assert_matches!(
+            task.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
+            }]
+        );
+        core.complete_workflow_activation(WorkflowActivationCompletion::empty(task.run_id))
+            .await
+            .unwrap();
+
+        // We can still service the query by trying again
+        let task = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            task.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
+            }]
+        );
+        core.complete_execution(&task.run_id).await;
+        let task = core.poll_workflow_activation().await.unwrap();
+        let qid = assert_matches!(
+            task.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::QueryWorkflow(q)),
+            }] => &q.query_id
+        );
+        core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+            task.run_id,
+            QueryResult {
+                query_id: qid.to_string(),
+                variant: Some(
+                    QuerySuccess {
+                        response: Some("done".into()),
+                    }
+                    .into(),
+                ),
+            }
+            .into(),
+        ))
+        .await
+        .unwrap();
+    };
+    let (qres, _) = join!(query_fut, poll_and_fail_fut);
+    let qres = qres.unwrap().query_result.unwrap();
+    assert_eq!(qres.payloads[0].data, b"done");
+}

package/sdk-core/tests/integ_tests/update_tests.rs

@@ -78,10 +78,9 @@ async fn update_workflow(#[values(FailUpdate::Yes, FailUpdate::No)] will_fail: F
         .unwrap();
     let with_id = HistoryForReplay::new(history, workflow_id.to_string());
     let replay_worker = init_core_replay_preloaded(workflow_id, [with_id]);
-    handle_update(will_fail, CompleteWorkflow::Yes, replay_worker.as_ref()).await;
+    handle_update(will_fail, CompleteWorkflow::Yes, replay_worker.as_ref(), 1).await;
 }
 
-#[rstest::rstest]
 #[tokio::test]
 async fn reapplied_updates_due_to_reset() {
     let mut starter = init_core_and_create_wf("update_workflow").await;
@@ -121,7 +120,9 @@ async fn reapplied_updates_due_to_reset() {
     .into_inner();
 
     // Accept and complete the reapplied update
-    handle_update(FailUpdate::No, CompleteWorkflow::No, core.as_ref()).await;
+    // Index here is 2 because there will be start workflow & update random seed (from the reset)
+    // first.
+    handle_update(FailUpdate::No, CompleteWorkflow::No, core.as_ref(), 2).await;
 
     // Send a second update and complete the workflow
     let post_reset_run_id = send_and_handle_update(
@@ -149,12 +150,19 @@ async fn reapplied_updates_due_to_reset() {
     // We now recapitulate the actions that the worker took on first execution above, pretending
     // that we always followed the post-reset history.
     // First, we handled the post-reset reapplied update and did not complete the workflow.
-    handle_update(FailUpdate::No, CompleteWorkflow::No, replay_worker.as_ref()).await;
+    handle_update(
+        FailUpdate::No,
+        CompleteWorkflow::No,
+        replay_worker.as_ref(),
+        2,
+    )
+    .await;
     // Then the client sent a second update; we handled it and completed the workflow.
     handle_update(
         FailUpdate::No,
         CompleteWorkflow::Yes,
         replay_worker.as_ref(),
+        0,
     )
     .await;
 
@@ -199,7 +207,7 @@ async fn send_and_handle_update(
     };
 
     // Accept update, complete update and complete workflow
-    let processing_task = handle_update(fail_update, complete_workflow, core);
+    let processing_task = handle_update(fail_update, complete_workflow, core, 0);
     let (ur, _) = join!(update_task, processing_task);
 
     let v = ur.outcome.unwrap().value.unwrap();
@@ -218,12 +226,11 @@ async fn handle_update(
     fail_update: FailUpdate,
     complete_workflow: CompleteWorkflow,
     core: &dyn Worker,
+    update_job_index: usize,
 ) {
     let act = core.poll_workflow_activation().await.unwrap();
-    // On replay, the first activation has update & start workflow, but on first execution, it does
-    // not - can happen if update is waiting on some condition.
     let pid = assert_matches!(
-        &act.jobs[0],
+        &act.jobs[update_job_index],
         WorkflowActivationJob {
             variant: Some(workflow_activation_job::Variant::DoUpdate(d)),
         } => &d.protocol_instance_id
@@ -573,7 +580,7 @@ async fn update_speculative_wft() {
         assert_matches!(
             res.jobs.as_slice(),
             [WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+                variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
             }]
         );
         core.complete_workflow_activation(WorkflowActivationCompletion::empty(res.run_id))

package/sdk-core/tests/integ_tests/visibility_tests.rs

@@ -29,7 +29,7 @@ async fn client_list_open_closed_workflow_executions() {
     assert_matches!(
         task.jobs.as_slice(),
         [WorkflowActivationJob {
-            variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+            variant: Some(workflow_activation_job::Variant::InitializeWorkflow(_)),
         }]
     );
 

package/sdk-core/tests/integ_tests/worker_tests.rs

@@ -1,7 +1,20 @@
+use std::cell::Cell;
+use std::sync::Arc;
+
 use assert_matches::assert_matches;
+use temporal_client::WorkflowOptions;
+use temporal_sdk::interceptors::WorkerInterceptor;
 use temporal_sdk_core::{init_worker, CoreRuntime};
 use temporal_sdk_core_api::{errors::WorkerValidationError, worker::WorkerConfigBuilder, Worker};
-use temporal_sdk_core_test_utils::{get_integ_server_options, get_integ_telem_options};
+use temporal_sdk_core_protos::coresdk::workflow_completion::{
+    workflow_activation_completion::Status, Failure, WorkflowActivationCompletion,
+};
+use temporal_sdk_core_protos::temporal::api::failure::v1::Failure as InnerFailure;
+use temporal_sdk_core_test_utils::{
+    drain_pollers_and_shutdown, get_integ_server_options, get_integ_telem_options, CoreWfStarter,
+};
+use tokio::sync::Notify;
+use uuid::Uuid;
 
 #[tokio::test]
 async fn worker_validation_fails_on_nonexistent_namespace() {
@@ -30,3 +43,71 @@ async fn worker_validation_fails_on_nonexistent_namespace() {
         Err(WorkerValidationError::NamespaceDescribeError { .. })
     );
 }
+
+#[tokio::test]
+async fn worker_handles_unknown_workflow_types_gracefully() {
+    let wf_type = "worker_handles_unknown_workflow_types_gracefully";
+    let mut starter = CoreWfStarter::new(wf_type);
+    let mut worker = starter.worker().await;
+
+    let run_id = worker
+        .submit_wf(
+            format!("wce-{}", Uuid::new_v4()),
+            "unregistered".to_string(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+
+    struct GracefulAsserter {
+        notify: Arc<Notify>,
+        run_id: String,
+        unregistered_failure_seen: Cell<bool>,
+    }
+    #[async_trait::async_trait(?Send)]
+    impl WorkerInterceptor for GracefulAsserter {
+        async fn on_workflow_activation_completion(
+            &self,
+            completion: &WorkflowActivationCompletion,
+        ) {
+            if matches!(
+                completion,
+                WorkflowActivationCompletion {
+                    status: Some(Status::Failed(Failure {
+                        failure: Some(InnerFailure { message, .. }),
+                        ..
+                    })),
+                    run_id,
+                } if message == "Workflow type unregistered not found" && *run_id == self.run_id
+            ) {
+                self.unregistered_failure_seen.set(true);
+            }
+            // If we've seen the failure, and the completion is a success for the same run, we're done
+            if matches!(
+                completion,
+                WorkflowActivationCompletion {
+                    status: Some(Status::Successful(..)),
+                    run_id,
+                } if self.unregistered_failure_seen.get() && *run_id == self.run_id
+            ) {
+                // Shutdown the worker
+                self.notify.notify_one();
+            }
+        }
+        fn on_shutdown(&self, _: &temporal_sdk::Worker) {}
+    }
+
+    let inner = worker.inner_mut();
+    let notify = Arc::new(Notify::new());
+    inner.set_worker_interceptor(GracefulAsserter {
+        notify: notify.clone(),
+        run_id,
+        unregistered_failure_seen: Cell::new(false),
+    });
+    tokio::join!(async { inner.run().await.unwrap() }, async move {
+        notify.notified().await;
+        let worker = starter.get_worker().await.clone();
+        drain_pollers_and_shutdown(&worker).await;
+    });
+}

package/sdk-core/tests/integ_tests/workflow_tests/activities.rs

@@ -9,7 +9,7 @@ use std::{
 use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowExecutionResult, WorkflowOptions};
 use temporal_sdk::{
     ActContext, ActExitValue, ActivityError, ActivityOptions, CancellableFuture, WfContext,
-    WorkflowResult,
+    WfExitValue, WorkflowResult,
 };
 use temporal_sdk_core_protos::{
     coresdk::{
@@ -52,7 +52,7 @@ pub(crate) async fn one_activity_wf(ctx: WfContext) -> WorkflowResult<()> {
 }
 
 #[tokio::test]
-async fn one_activity() {
+async fn one_activity_only() {
     let wf_name = "one_activity";
     let mut starter = CoreWfStarter::new(wf_name);
     let mut worker = starter.worker().await;
@@ -129,7 +129,7 @@ async fn activity_workflow() {
                       status: Some(
                         act_res::Status::Completed(activity_result::Success{result: Some(r)})),
                         ..
-                    })}
+                    }), ..}
                 )),
             },
         ] => {
@@ -182,7 +182,7 @@ async fn activity_non_retryable_failure() {
                     ResolveActivity {seq, result: Some(ActivityResolution{
                     status: Some(act_res::Status::Failed(activity_result::Failure{
                         failure: Some(f),
-                    }))})}
+                    }))}),..}
                 )),
             },
         ] => {
@@ -249,7 +249,7 @@ async fn activity_non_retryable_failure_with_error() {
                     ResolveActivity {seq, result: Some(ActivityResolution{
                     status: Some(act_res::Status::Failed(activity_result::Failure{
                         failure: Some(f),
-                    }))})}
+                    }))}),..}
                 )),
             },
         ] => {
@@ -328,7 +328,7 @@ async fn activity_retry() {
             WorkflowActivationJob {
                 variant: Some(workflow_activation_job::Variant::ResolveActivity(
                     ResolveActivity {seq, result: Some(ActivityResolution{
-                    status: Some(act_res::Status::Completed(activity_result::Success{result: Some(r)}))})}
+                    status: Some(act_res::Status::Completed(activity_result::Success{result: Some(r)}))}),..}
                 )),
             },
         ] => {
@@ -526,7 +526,7 @@ async fn started_activity_timeout() {
                                 )
                             ),
                             ..
-                        })
+                        }), ..
                     }
                 )),
             },
@@ -714,7 +714,7 @@ async fn async_activity_completion_workflow() {
                 variant: Some(workflow_activation_job::Variant::ResolveActivity(
                     ResolveActivity {seq, result: Some(ActivityResolution {
                     status: Some(act_res::Status::Completed(activity_result::Success{result: Some(r)})),
-                     ..})}
+                     ..}), ..}
                 )),
             },
         ] => {
@@ -1061,3 +1061,41 @@ async fn activity_can_be_cancelled_by_local_timeout() {
     worker.run_until_done().await.unwrap();
     assert!(WAS_CANCELLED.load(Ordering::Relaxed));
 }
+
+#[tokio::test]
+#[ignore] // Runs forever, used to manually attempt to repro spurious activity completion rpc errs
+async fn long_activity_timeout_repro() {
+    let wf_name = "long_activity_timeout_repro";
+    let mut starter = CoreWfStarter::new(wf_name);
+    starter
+        .worker_config
+        .local_timeout_buffer_for_activities(Duration::from_secs(0));
+    let mut worker = starter.worker().await;
+    worker.register_wf(wf_name.to_owned(), |ctx: WfContext| async move {
+        let mut iter = 1;
+        loop {
+            let res = ctx
+                .activity(ActivityOptions {
+                    activity_type: "echo_activity".to_string(),
+                    start_to_close_timeout: Some(Duration::from_secs(1)),
+                    input: "hi!".as_json_payload().expect("serializes fine"),
+                    retry_policy: Some(RetryPolicy {
+                        maximum_attempts: 1,
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                })
+                .await;
+            assert!(res.completed_ok());
+            ctx.timer(Duration::from_secs(60 * 3)).await;
+            iter += 1;
+            if iter > 5000 {
+                return Ok(WfExitValue::<()>::continue_as_new(Default::default()));
+            }
+        }
+    });
+    worker.register_activity("echo_activity", echo);
+
+    starter.start_with_worker(wf_name, &mut worker).await;
+    worker.run_until_done().await.unwrap();
+}

package/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs

@@ -7,7 +7,7 @@ use std::{
 };
 use temporal_client::WorkflowOptions;
 use temporal_sdk::{
-    interceptors::WorkerInterceptor, ActContext, ActivityError, CancellableFuture,
+    interceptors::WorkerInterceptor, ActContext, ActivityError, ActivityOptions, CancellableFuture,
     LocalActivityOptions, WfContext, WorkflowResult,
 };
 use temporal_sdk_core::replay::HistoryForReplay;
@@ -47,8 +47,13 @@ async fn one_local_activity() {
     worker.register_wf(wf_name.to_owned(), one_local_activity_wf);
     worker.register_activity("echo_activity", echo);
 
-    starter.start_with_worker(wf_name, &mut worker).await;
+    let run_id = starter.start_with_worker(wf_name, &mut worker).await;
     worker.run_until_done().await.unwrap();
+    let tq = starter.get_task_queue().to_string();
+    starter
+        .fetch_history_and_replay(tq, run_id, worker.inner_mut())
+        .await
+        .unwrap();
 }
 
 pub(crate) async fn local_act_concurrent_with_timer_wf(ctx: WfContext) -> WorkflowResult<()> {
@@ -663,3 +668,77 @@ async fn third_weird_la_nondeterminism_repro() {
     });
     worker.run().await.unwrap();
 }
+
+/// This test demonstrates why it's important to send LA resolutions last within a job.
+/// If we were to (during replay) scan ahead, see the marker, and resolve the LA before the
+/// activity cancellation, that would be wrong because, during execution, the LA resolution is
+/// always going to take _longer_ than the instantaneous cancel effect.
+///
+/// This affect applies regardless of how you choose to interleave cancellations and LAs. Ultimately
+/// all cancellations will happen at once (in the order they are submitted) while the LA executions
+/// are queued (because this all happens synchronously in the workflow machines). If you were to
+/// _wait_ on an LA, and then cancel something else, and then run another LA, such that all commands
+/// happened in the same workflow task, it would _still_ be fine to sort LA jobs last _within_ the
+/// 2 activations that would necessarily entail (because, one you wait on the LA result, control
+/// will be yielded and it will take another activation to unblock that LA).
+#[tokio::test]
+async fn la_resolve_same_time_as_other_cancel() {
+    let wf_name = "la_resolve_same_time_as_other_cancel";
+    let mut starter = CoreWfStarter::new(wf_name);
+    // The activity won't get a chance to receive the cancel so make sure we still exit fast
+    starter
+        .worker_config
+        .graceful_shutdown_period(Duration::from_millis(100));
+    let mut worker = starter.worker().await;
+
+    worker.register_wf(wf_name.to_owned(), |ctx: WfContext| async move {
+        let normal_act = ctx.activity(ActivityOptions {
+            activity_type: "delay".to_string(),
+            input: 9000.as_json_payload().expect("serializes fine"),
+            cancellation_type: ActivityCancellationType::TryCancel,
+            start_to_close_timeout: Some(Duration::from_secs(9000)),
+            ..Default::default()
+        });
+        // Make new task
+        ctx.timer(Duration::from_millis(1)).await;
+
+        // Start LA and cancel the activity at the same time
+        let local_act = ctx.local_activity(LocalActivityOptions {
+            activity_type: "delay".to_string(),
+            input: 100.as_json_payload().expect("serializes fine"),
+            ..Default::default()
+        });
+        normal_act.cancel(&ctx);
+        // Race them, starting a timer if LA completes first
+        tokio::select! {
+            biased;
+            _ = normal_act => {},
+            _ = local_act => {
+                ctx.timer(Duration::from_millis(1)).await;
+            },
+        }
+        Ok(().into())
+    });
+    worker.register_activity("delay", |ctx: ActContext, wait_time: u64| async move {
+        tokio::select! {
+            _ = tokio::time::sleep(Duration::from_millis(wait_time)) => {}
+            _ = ctx.cancelled() => {}
+        }
+        Ok(())
+    });
+
+    let run_id = worker
+        .submit_wf(
+            wf_name.to_owned(),
+            wf_name.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+    starter
+        .fetch_history_and_replay(wf_name, run_id, worker.inner_mut())
+        .await
+        .unwrap();
+}