temporalio 0.0.2 → 0.1.0

data/bridge/sdk-core/core/src/core_tests/determinism.rs

@@ -1,4 +1,5 @@
 use crate::{
+    internal_flags::CoreInternalFlags,
     replay::DEFAULT_WORKFLOW_TYPE,
     test_help::{canned_histories, mock_sdk, mock_sdk_cfg, MockPollCfg, ResponseType},
     worker::client::mocks::mock_workflow_client,
@@ -8,8 +9,13 @@ use std::{
     time::Duration,
 };
 use temporal_client::WorkflowOptions;
-use temporal_sdk::{WfContext, WorkflowResult};
-use temporal_sdk_core_protos::temporal::api::enums::v1::WorkflowTaskFailedCause;
+use temporal_sdk::{
+    ActivityOptions, ChildWorkflowOptions, LocalActivityOptions, WfContext, WorkflowResult,
+};
+use temporal_sdk_core_protos::{
+    temporal::api::{enums::v1::WorkflowTaskFailedCause, failure::v1::Failure},
+    DEFAULT_ACTIVITY_TYPE,
+};
 
 static DID_FAIL: AtomicBool = AtomicBool::new(false);
 pub async fn timer_wf_fails_once(ctx: WfContext) -> WorkflowResult<()> {
@@ -105,3 +111,160 @@ async fn test_wf_task_rejected_properly_due_to_nondeterminism(#[case] use_cache:
     // timer and proceed without restarting
     assert_eq!(2, started_count.load(Ordering::Relaxed));
 }
+
+#[rstest::rstest]
+#[tokio::test]
+async fn activity_id_or_type_change_is_nondeterministic(
+    #[values(true, false)] use_cache: bool,
+    #[values(true, false)] id_change: bool,
+    #[values(true, false)] local_act: bool,
+) {
+    let wf_id = "fakeid";
+    let wf_type = DEFAULT_WORKFLOW_TYPE;
+    let mut t = if local_act {
+        canned_histories::single_local_activity("1")
+    } else {
+        canned_histories::single_activity("1")
+    };
+    t.set_flags_first_wft(&[CoreInternalFlags::IdAndTypeDeterminismChecks as u32], &[]);
+    let mock = mock_workflow_client();
+    let mut mh = MockPollCfg::from_resp_batches(
+        wf_id,
+        t,
+        // Two polls are needed, since the first will fail
+        [ResponseType::AllHistory, ResponseType::AllHistory],
+        mock,
+    );
+    // We should see one wft failure which has nondeterminism cause
+    mh.num_expected_fails = 1;
+    mh.expect_fail_wft_matcher = Box::new(move |_, cause, f| {
+        let should_contain = if id_change {
+            "does not match activity id"
+        } else {
+            "does not match activity type"
+        };
+        matches!(cause, WorkflowTaskFailedCause::NonDeterministicError)
+            && matches!(f, Some(Failure {
+                message,
+                ..
+            }) if message.contains(should_contain))
+    });
+    let mut worker = mock_sdk_cfg(mh, |cfg| {
+        if use_cache {
+            cfg.max_cached_workflows = 2;
+        }
+    });
+
+    worker.register_wf(wf_type.to_owned(), move |ctx: WfContext| async move {
+        if local_act {
+            ctx.local_activity(if id_change {
+                LocalActivityOptions {
+                    activity_id: Some("I'm bad and wrong!".to_string()),
+                    activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                    ..Default::default()
+                }
+            } else {
+                LocalActivityOptions {
+                    activity_type: "not the default act type".to_string(),
+                    ..Default::default()
+                }
+            })
+            .await;
+        } else {
+            ctx.activity(if id_change {
+                ActivityOptions {
+                    activity_id: Some("I'm bad and wrong!".to_string()),
+                    activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                    ..Default::default()
+                }
+            } else {
+                ActivityOptions {
+                    activity_type: "not the default act type".to_string(),
+                    ..Default::default()
+                }
+            })
+            .await;
+        }
+        Ok(().into())
+    });
+
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            wf_type.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
+#[rstest::rstest]
+#[tokio::test]
+async fn child_wf_id_or_type_change_is_nondeterministic(
+    #[values(true, false)] use_cache: bool,
+    #[values(true, false)] id_change: bool,
+) {
+    let wf_id = "fakeid";
+    let wf_type = DEFAULT_WORKFLOW_TYPE;
+    let mut t = canned_histories::single_child_workflow("1");
+    t.set_flags_first_wft(&[CoreInternalFlags::IdAndTypeDeterminismChecks as u32], &[]);
+    let mock = mock_workflow_client();
+    let mut mh = MockPollCfg::from_resp_batches(
+        wf_id,
+        t,
+        // Two polls are needed, since the first will fail
+        [ResponseType::AllHistory, ResponseType::AllHistory],
+        mock,
+    );
+    // We should see one wft failure which has nondeterminism cause
+    mh.num_expected_fails = 1;
+    mh.expect_fail_wft_matcher = Box::new(move |_, cause, f| {
+        let should_contain = if id_change {
+            "does not match child workflow id"
+        } else {
+            "does not match child workflow type"
+        };
+        matches!(cause, WorkflowTaskFailedCause::NonDeterministicError)
+            && matches!(f, Some(Failure {
+                message,
+                ..
+            }) if message.contains(should_contain))
+    });
+    let mut worker = mock_sdk_cfg(mh, |cfg| {
+        if use_cache {
+            cfg.max_cached_workflows = 2;
+        }
+    });
+
+    worker.register_wf(wf_type.to_owned(), move |ctx: WfContext| async move {
+        ctx.child_workflow(if id_change {
+            ChildWorkflowOptions {
+                workflow_id: "I'm bad and wrong!".to_string(),
+                workflow_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                ..Default::default()
+            }
+        } else {
+            ChildWorkflowOptions {
+                workflow_id: "1".to_string(),
+                workflow_type: "not the child wf type".to_string(),
+                ..Default::default()
+            }
+        })
+        .start(&ctx)
+        .await;
+        Ok(().into())
+    });
+
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            wf_type.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}

data/bridge/sdk-core/core/src/core_tests/local_activities.rs

@@ -8,6 +8,7 @@ use crate::{
     worker::{client::mocks::mock_workflow_client, LEGACY_QUERY_ID},
 };
 use anyhow::anyhow;
+use crossbeam::queue::SegQueue;
 use futures::{future::join_all, FutureExt};
 use std::{
     collections::HashMap,
@@ -19,7 +20,9 @@ use std::{
     time::{Duration, SystemTime},
 };
 use temporal_client::WorkflowOptions;
-use temporal_sdk::{ActContext, LocalActivityOptions, WfContext, WorkflowResult};
+use temporal_sdk::{
+    ActContext, ActivityCancelledError, LocalActivityOptions, WfContext, WorkflowResult,
+};
 use temporal_sdk_core_api::Worker;
 use temporal_sdk_core_protos::{
     coresdk::{
@@ -31,11 +34,11 @@ use temporal_sdk_core_protos::{
     },
     temporal::api::{
         common::v1::RetryPolicy,
-        enums::v1::{EventType, TimeoutType},
+        enums::v1::{EventType, TimeoutType, WorkflowTaskFailedCause},
         failure::v1::Failure,
-        history::v1::History,
         query::v1::WorkflowQuery,
     },
+    DEFAULT_ACTIVITY_TYPE,
 };
 use temporal_sdk_core_test_utils::{
     schedule_local_activity_cmd, start_timer_cmd, WorkerTestHelpers,
@@ -58,7 +61,7 @@ async fn local_act_two_wfts_before_marker(#[case] replay: bool, #[case] cached:
     let mut t = TestHistoryBuilder::default();
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_full_wf_task();
     t.add_local_activity_result_marker(1, "1", b"echo".into());
     t.add_timer_fired(timer_started_event_id, "1".to_string());
@@ -83,7 +86,7 @@ async fn local_act_two_wfts_before_marker(#[case] replay: bool, #[case] cached:
         DEFAULT_WORKFLOW_TYPE.to_owned(),
         |ctx: WfContext| async move {
             let la = ctx.local_activity(LocalActivityOptions {
-                activity_type: "echo".to_string(),
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
                 input: "hi".as_json_payload().expect("serializes fine"),
                 ..Default::default()
             });
@@ -92,7 +95,7 @@ async fn local_act_two_wfts_before_marker(#[case] replay: bool, #[case] cached:
             Ok(().into())
         },
     );
-    worker.register_activity("echo", echo);
+    worker.register_activity(DEFAULT_ACTIVITY_TYPE, echo);
     worker
         .submit_wf(
             wf_id.to_owned(),
@@ -110,7 +113,7 @@ pub async fn local_act_fanout_wf(ctx: WfContext) -> WorkflowResult<()> {
         .map(|i| {
             ctx.local_activity(LocalActivityOptions {
                 activity_type: "echo".to_string(),
-                input: format!("Hi {}", i)
+                input: format!("Hi {i}")
                     .as_json_payload()
                     .expect("serializes fine"),
                 ..Default::default()
@@ -127,7 +130,7 @@ async fn local_act_many_concurrent() {
     let mut t = TestHistoryBuilder::default();
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_full_wf_task();
     for i in 1..=50 {
         t.add_local_activity_result_marker(i, &i.to_string(), b"echo".into());
@@ -168,12 +171,7 @@ async fn local_act_many_concurrent() {
 async fn local_act_heartbeat(#[case] shutdown_middle: bool) {
     let mut t = TestHistoryBuilder::default();
     let wft_timeout = Duration::from_millis(200);
-    let mut wes_short_wft_timeout = default_wes_attribs();
-    wes_short_wft_timeout.workflow_task_timeout = Some(wft_timeout.try_into().unwrap());
-    t.add(
-        EventType::WorkflowExecutionStarted,
-        wes_short_wft_timeout.into(),
-    );
+    t.add_wfe_started_with_wft_timeout(wft_timeout);
     t.add_full_wf_task();
     // Task created by WFT heartbeat
     t.add_full_wf_task();
@@ -304,7 +302,7 @@ async fn local_act_retry_long_backoff_uses_timer() {
         "1",
         Failure::application_failure("la failed".to_string(), false),
     );
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_timer_fired(timer_started_event_id, "1".to_string());
     t.add_full_wf_task();
     t.add_local_activity_fail_marker(
@@ -312,7 +310,7 @@ async fn local_act_retry_long_backoff_uses_timer() {
         "2",
         Failure::application_failure("la failed".to_string(), false),
     );
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_timer_fired(timer_started_event_id, "2".to_string());
     t.add_full_wf_task();
     t.add_workflow_execution_completed();
@@ -332,7 +330,7 @@ async fn local_act_retry_long_backoff_uses_timer() {
         |ctx: WfContext| async move {
             let la_res = ctx
                 .local_activity(LocalActivityOptions {
-                    activity_type: "echo".to_string(),
+                    activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
                     input: "hi".as_json_payload().expect("serializes fine"),
                     retry_policy: RetryPolicy {
                         initial_interval: Some(prost_dur!(from_millis(65))),
@@ -351,9 +349,12 @@ async fn local_act_retry_long_backoff_uses_timer() {
             Ok(().into())
         },
     );
-    worker.register_activity("echo", move |_ctx: ActContext, _: String| async move {
-        Result::<(), _>::Err(anyhow!("Oh no I failed!"))
-    });
+    worker.register_activity(
+        DEFAULT_ACTIVITY_TYPE,
+        move |_ctx: ActContext, _: String| async move {
+            Result::<(), _>::Err(anyhow!("Oh no I failed!"))
+        },
+    );
     worker
         .submit_wf(
             wf_id.to_owned(),
@@ -413,7 +414,7 @@ async fn local_act_command_immediately_follows_la_marker() {
     t.add_full_wf_task();
     t.add_full_wf_task();
     t.add_local_activity_result_marker(1, "1", "done".into());
-    t.add_get_event_id(EventType::TimerStarted, None);
+    t.add_by_type(EventType::TimerStarted);
     t.add_full_wf_task();
 
     let wf_id = "fakeid";
@@ -452,12 +453,7 @@ async fn local_act_command_immediately_follows_la_marker() {
 async fn query_during_wft_heartbeat_doesnt_accidentally_fail_to_continue_heartbeat() {
     let wfid = "fake_wf_id";
     let mut t = TestHistoryBuilder::default();
-    let mut wes_short_wft_timeout = default_wes_attribs();
-    wes_short_wft_timeout.workflow_task_timeout = Some(prost_dur!(from_millis(200)));
-    t.add(
-        EventType::WorkflowExecutionStarted,
-        wes_short_wft_timeout.into(),
-    );
+    t.add_wfe_started_with_wft_timeout(Duration::from_millis(200));
     t.add_full_wf_task();
     // get query here
     t.add_full_wf_task();
@@ -506,7 +502,7 @@ async fn query_during_wft_heartbeat_doesnt_accidentally_fail_to_continue_heartbe
             task.run_id,
             schedule_local_activity_cmd(
                 1,
-                "act-id",
+                "1",
                 ActivityCancellationType::TryCancel,
                 Duration::from_secs(60),
             ),
@@ -572,15 +568,11 @@ async fn la_resolve_during_legacy_query_does_not_combine(#[case] impossible_quer
     // never happen, but there was an issue where an LA resolving could trigger that.
     let wfid = "fake_wf_id";
     let mut t = TestHistoryBuilder::default();
-    let wes_short_wft_timeout = default_wes_attribs();
-    t.add(
-        EventType::WorkflowExecutionStarted,
-        wes_short_wft_timeout.into(),
-    );
+    t.add(default_wes_attribs());
     // Since we don't send queries with start workflow, need one workflow task of something else
     // b/c we want to get an activation with a job and a nonlegacy query
     t.add_full_wf_task();
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_timer_fired(timer_started_event_id, "1".to_string());
 
     // nonlegacy query got here & LA started here
@@ -622,8 +614,12 @@ async fn la_resolve_during_legacy_query_does_not_combine(#[case] impossible_quer
                     2,
                 ),
             );
-            // Strip history, we need to look like we hit the cache
-            pr.history = Some(History { events: vec![] });
+            // Strip beginning of history so the only events are WFT sched/started, we need to look
+            // like we hit the cache
+            {
+                let h = pr.history.as_mut().unwrap();
+                h.events = h.events.split_off(6);
+            }
             // In the nonsense server response case, we attach a legacy query, otherwise this
             // response looks like a normal response to a forced WFT heartbeat.
             if impossible_query_in_task {
@@ -783,12 +779,19 @@ async fn test_schedule_to_start_timeout() {
     worker.run_until_done().await.unwrap();
 }
 
+#[rstest::rstest]
+#[case::sched_to_start(true)]
+#[case::sched_to_close(false)]
 #[tokio::test]
-async fn test_schedule_to_start_timeout_not_based_on_original_time() {
+async fn test_schedule_to_start_timeout_not_based_on_original_time(
+    #[case] is_sched_to_start: bool,
+) {
     // We used to carry over the schedule time of LAs from the "original" schedule time if these LAs
     // created newly after backing off across a timer. That was a mistake, since schedule-to-start
-    // timeouts should apply to when the new attempt was scheduled. This test verifies we don't
-    // time out on s-t-s timeouts because of that.
+    // timeouts should apply to when the new attempt was scheduled. This test verifies:
+    // * we don't time out on s-t-s timeouts because of that, when the param is true.
+    // * we do properly time out on s-t-c timeouts when the param is false
+
     let mut t = TestHistoryBuilder::default();
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
@@ -805,7 +808,7 @@ async fn test_schedule_to_start_timeout_not_based_on_original_time() {
             deets.backoff = Some(prost_dur!(from_secs(100)));
         },
     );
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_timer_fired(timer_started_event_id, "1".to_string());
     t.add_workflow_task_scheduled_and_started();
 
@@ -814,9 +817,18 @@ async fn test_schedule_to_start_timeout_not_based_on_original_time() {
     let mh = MockPollCfg::from_resp_batches(wf_id, t, [ResponseType::AllHistory], mock);
     let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
 
+    let schedule_to_close_timeout = Some(if is_sched_to_start {
+        // This 60 minute timeout will not have elapsed according to the original
+        // schedule time in the history.
+        Duration::from_secs(60 * 60)
+    } else {
+        // This 10 minute timeout will have already elapsed
+        Duration::from_secs(10 * 60)
+    });
+
     worker.register_wf(
         DEFAULT_WORKFLOW_TYPE.to_owned(),
-        |ctx: WfContext| async move {
+        move |ctx: WfContext| async move {
             let la_res = ctx
                 .local_activity(LocalActivityOptions {
                     activity_type: "echo".to_string(),
@@ -829,11 +841,15 @@ async fn test_schedule_to_start_timeout_not_based_on_original_time() {
                         non_retryable_error_types: vec![],
                     },
                     schedule_to_start_timeout: Some(Duration::from_secs(60)),
-                    schedule_to_close_timeout: Some(Duration::from_secs(60 * 60)),
+                    schedule_to_close_timeout,
                     ..Default::default()
                 })
                 .await;
-            assert!(la_res.completed_ok());
+            if is_sched_to_start {
+                assert!(la_res.completed_ok());
+            } else {
+                assert_eq!(la_res.timed_out(), Some(TimeoutType::ScheduleToClose));
+            }
             Ok(().into())
         },
     );
@@ -854,64 +870,162 @@ async fn test_schedule_to_start_timeout_not_based_on_original_time() {
 }
 
 #[tokio::test]
-async fn test_schedule_to_close_timeout_based_on_original_time() {
-    // We used to carry over the schedule time of LAs from the "original" schedule time if these LAs
-    // created newly after backing off across a timer. That was a mistake, since schedule-to-start
-    // timeouts should apply to when the new attempt was scheduled. This test verifies we don't
-    // time out on s-t-s timeouts because of that.
+async fn wft_failure_cancels_running_las() {
     let mut t = TestHistoryBuilder::default();
-    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_wfe_started_with_wft_timeout(Duration::from_millis(200));
     t.add_full_wf_task();
-    let orig_sched = SystemTime::now().sub(Duration::from_secs(60 * 20));
-    t.add_local_activity_marker(
-        1,
-        "1",
-        None,
-        Some(Failure::application_failure("la failed".to_string(), false)),
-        |deets| {
-            // Really old schedule time, which should _not_ count against schedule_to_start
-            deets.original_schedule_time = Some(orig_sched.into());
-            // Backoff value must be present since we're simulating timer backoff
-            deets.backoff = Some(prost_dur!(from_secs(100)));
-        },
-    );
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_timer_fired(timer_started_event_id, "1".to_string());
     t.add_workflow_task_scheduled_and_started();
 
     let wf_id = "fakeid";
     let mock = mock_workflow_client();
-    let mh = MockPollCfg::from_resp_batches(wf_id, t, [ResponseType::AllHistory], mock);
+    let mut mh = MockPollCfg::from_resp_batches(wf_id, t, [1, 2], mock);
+    mh.num_expected_fails = 1;
     let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
 
     worker.register_wf(
         DEFAULT_WORKFLOW_TYPE.to_owned(),
         |ctx: WfContext| async move {
-            let la_res = ctx
-                .local_activity(LocalActivityOptions {
-                    activity_type: "echo".to_string(),
-                    input: "hi".as_json_payload().expect("serializes fine"),
-                    retry_policy: RetryPolicy {
-                        initial_interval: Some(prost_dur!(from_millis(50))),
-                        backoff_coefficient: 1.2,
-                        maximum_interval: None,
-                        maximum_attempts: 5,
-                        non_retryable_error_types: vec![],
-                    },
-                    // This 10 minute timeout will have already elapsed according to the original
-                    // schedule time in the history.
-                    schedule_to_close_timeout: Some(Duration::from_secs(10 * 60)),
-                    ..Default::default()
-                })
-                .await;
-            assert_eq!(la_res.timed_out(), Some(TimeoutType::ScheduleToClose));
+            let la_handle = ctx.local_activity(LocalActivityOptions {
+                activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
+                input: "hi".as_json_payload().expect("serializes fine"),
+                ..Default::default()
+            });
+            tokio::join!(
+                async {
+                    ctx.timer(Duration::from_secs(1)).await;
+                    panic!("ahhh I'm failing wft")
+                },
+                la_handle
+            );
             Ok(().into())
         },
     );
+    worker.register_activity(
+        DEFAULT_ACTIVITY_TYPE,
+        move |ctx: ActContext, _: String| async move {
+            let res = tokio::time::timeout(Duration::from_millis(500), ctx.cancelled()).await;
+            if res.is_err() {
+                panic!("Activity must be cancelled!!!!");
+            }
+            Result::<(), _>::Err(ActivityCancelledError::default().into())
+        },
+    );
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
+#[tokio::test]
+async fn resolved_las_not_recorded_if_wft_fails_many_times() {
+    // We shouldn't record any LA results if the workflow activation is repeatedly failing. There
+    // was an issue that, because we stop reporting WFT failures after 2 tries, this meant the WFT
+    // was not marked as "completed" and the WFT could accidentally be replied to with LA results.
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+    t.add_workflow_task_failed_with_failure(
+        WorkflowTaskFailedCause::Unspecified,
+        Default::default(),
+    );
+    t.add_workflow_task_scheduled_and_started();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mut mh = MockPollCfg::from_resp_batches(
+        wf_id,
+        t,
+        [1.into(), ResponseType::AllHistory, ResponseType::AllHistory],
+        mock,
+    );
+    mh.num_expected_fails = 2;
+    mh.num_expected_completions = Some(0.into());
+    let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        |ctx: WfContext| async move {
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: "echo".to_string(),
+                input: "hi".as_json_payload().expect("serializes fine"),
+                ..Default::default()
+            })
+            .await;
+            panic!("Oh nooooo")
+        },
+    );
     worker.register_activity(
         "echo",
-        move |_ctx: ActContext, _: String| async move { Ok(()) },
+        move |_: ActContext, _: String| async move { Ok(()) },
+    );
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
+#[tokio::test]
+async fn local_act_records_nonfirst_attempts_ok() {
+    let mut t = TestHistoryBuilder::default();
+    let wft_timeout = Duration::from_millis(200);
+    t.add_wfe_started_with_wft_timeout(wft_timeout);
+    t.add_full_wf_task();
+    t.add_full_wf_task();
+    t.add_full_wf_task();
+    t.add_workflow_task_scheduled_and_started();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mut mh = MockPollCfg::from_resp_batches(wf_id, t, [1, 2, 3], mock);
+    let nonfirst_counts = Arc::new(SegQueue::new());
+    let nfc_c = nonfirst_counts.clone();
+    mh.completion_asserts = Some(Box::new(move |c| {
+        nfc_c.push(
+            c.metering_metadata
+                .nonfirst_local_activity_execution_attempts,
+        );
+    }));
+    let mut worker = mock_sdk_cfg(mh, |wc| {
+        wc.max_cached_workflows = 1;
+        wc.max_outstanding_workflow_tasks = 1;
+    });
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        |ctx: WfContext| async move {
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: "echo".to_string(),
+                input: "hi".as_json_payload().expect("serializes fine"),
+                retry_policy: RetryPolicy {
+                    initial_interval: Some(prost_dur!(from_millis(10))),
+                    backoff_coefficient: 1.0,
+                    maximum_interval: None,
+                    maximum_attempts: 0,
+                    non_retryable_error_types: vec![],
+                },
+                ..Default::default()
+            })
+            .await;
+            Ok(().into())
+        },
     );
+    worker.register_activity("echo", move |_ctx: ActContext, _: String| async move {
+        Result::<(), _>::Err(anyhow!("I fail"))
+    });
     worker
         .submit_wf(
             wf_id.to_owned(),
@@ -922,4 +1036,11 @@ async fn test_schedule_to_close_timeout_based_on_original_time() {
         .await
         .unwrap();
     worker.run_until_done().await.unwrap();
+    // 3 workflow tasks
+    assert_eq!(nonfirst_counts.len(), 3);
+    // First task's non-first count should, of course, be 0
+    assert_eq!(nonfirst_counts.pop().unwrap(), 0);
+    // Next two, some nonzero amount which could vary based on test load
+    assert!(nonfirst_counts.pop().unwrap() > 0);
+    assert!(nonfirst_counts.pop().unwrap() > 0);
 }

data/bridge/sdk-core/core/src/core_tests/queries.rs

@@ -162,7 +162,7 @@ async fn new_queries(#[case] num_queries: usize) {
         pr.queries = HashMap::new();
         for i in 1..=num_queries {
             pr.queries.insert(
-                format!("q{}", i),
+                format!("q{i}"),
                 WorkflowQuery {
                     query_type: "query-type".to_string(),
                     query_args: Some(b"hi".into()),
@@ -206,7 +206,7 @@ async fn new_queries(#[case] num_queries: usize) {
     let mut qresults: Vec<_> = (1..=num_queries)
         .map(|i| {
             QueryResult {
-                query_id: format!("q{}", i),
+                query_id: format!("q{i}"),
                 variant: Some(
                     QuerySuccess {
                         response: Some(query_resp.into()),
@@ -769,11 +769,10 @@ async fn legacy_query_combined_with_timer_fire_repro() {
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
     let scheduled_event_id = t.add_activity_task_scheduled("1");
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_timer_fired(timer_started_event_id, "1".to_string());
     t.add_full_wf_task();
     t.add(
-        EventType::ActivityTaskCancelRequested,
         history_event::Attributes::ActivityTaskCancelRequestedEventAttributes(
             ActivityTaskCancelRequestedEventAttributes {
                 scheduled_event_id,