temporalio 0.0.1 → 0.0.2

data/bridge/sdk-core/core/src/core_tests/local_activities.rs

@@ -3,19 +3,20 @@ use crate::{
     replay::{default_wes_attribs, TestHistoryBuilder, DEFAULT_WORKFLOW_TYPE},
     test_help::{
         hist_to_poll_resp, mock_sdk, mock_sdk_cfg, mock_worker, single_hist_mock_sg, MockPollCfg,
-        ResponseType, TEST_Q,
+        ResponseType,
     },
-    worker::client::mocks::mock_workflow_client,
+    worker::{client::mocks::mock_workflow_client, LEGACY_QUERY_ID},
 };
 use anyhow::anyhow;
 use futures::{future::join_all, FutureExt};
 use std::{
     collections::HashMap,
+    ops::Sub,
     sync::{
         atomic::{AtomicUsize, Ordering},
         Arc,
     },
-    time::Duration,
+    time::{Duration, SystemTime},
 };
 use temporal_client::WorkflowOptions;
 use temporal_sdk::{ActContext, LocalActivityOptions, WfContext, WorkflowResult};
@@ -29,11 +30,16 @@ use temporal_sdk_core_protos::{
         ActivityTaskCompletion, AsJsonPayloadExt,
     },
     temporal::api::{
-        common::v1::RetryPolicy, enums::v1::EventType, failure::v1::Failure,
+        common::v1::RetryPolicy,
+        enums::v1::{EventType, TimeoutType},
+        failure::v1::Failure,
+        history::v1::History,
         query::v1::WorkflowQuery,
     },
 };
-use temporal_sdk_core_test_utils::{schedule_local_activity_cmd, WorkerTestHelpers};
+use temporal_sdk_core_test_utils::{
+    schedule_local_activity_cmd, start_timer_cmd, WorkerTestHelpers,
+};
 use tokio::sync::Barrier;
 
 async fn echo(_ctx: ActContext, e: String) -> anyhow::Result<String> {
@@ -365,7 +371,7 @@ async fn local_act_null_result() {
     let mut t = TestHistoryBuilder::default();
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
-    t.add_local_activity_marker(1, "1", None, None, None);
+    t.add_local_activity_marker(1, "1", None, None, |_| {});
     t.add_workflow_execution_completed();
 
     let wf_id = "fakeid";
@@ -398,6 +404,50 @@ async fn local_act_null_result() {
     worker.run_until_done().await.unwrap();
 }
 
+#[tokio::test]
+async fn local_act_command_immediately_follows_la_marker() {
+    // This repro only works both when cache is off, and there is at least one heartbeat wft
+    // before the marker & next command are recorded.
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_full_wf_task();
+    t.add_local_activity_result_marker(1, "1", "done".into());
+    t.add_get_event_id(EventType::TimerStarted, None);
+    t.add_full_wf_task();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    // Bug only repros when seeing history up to third wft
+    let mh = MockPollCfg::from_resp_batches(wf_id, t, [3], mock);
+    let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 0);
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        |ctx: WfContext| async move {
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: "nullres".to_string(),
+                input: "hi".as_json_payload().expect("serializes fine"),
+                ..Default::default()
+            })
+            .await;
+            ctx.timer(Duration::from_secs(1)).await;
+            Ok(().into())
+        },
+    );
+    worker.register_activity("nullres", |_ctx: ActContext, _: String| async { Ok(()) });
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
 #[tokio::test]
 async fn query_during_wft_heartbeat_doesnt_accidentally_fail_to_continue_heartbeat() {
     let wfid = "fake_wf_id";
@@ -411,11 +461,11 @@ async fn query_during_wft_heartbeat_doesnt_accidentally_fail_to_continue_heartbe
     t.add_full_wf_task();
     // get query here
     t.add_full_wf_task();
-    t.add_local_activity_marker(1, "1", None, None, None);
+    t.add_local_activity_result_marker(1, "1", "done".into());
     t.add_workflow_execution_completed();
 
     let query_with_hist_task = {
-        let mut pr = hist_to_poll_resp(&t, wfid, ResponseType::ToTaskNum(1), TEST_Q);
+        let mut pr = hist_to_poll_resp(&t, wfid, ResponseType::ToTaskNum(1));
         pr.queries = HashMap::new();
         pr.queries.insert(
             "the-query".to_string(),
@@ -441,7 +491,6 @@ async fn query_during_wft_heartbeat_doesnt_accidentally_fail_to_continue_heartbe
                 .boxed(),
                 3,
             ),
-            TEST_Q,
         ),
     ];
     let mock = mock_workflow_client();
@@ -513,3 +562,364 @@ async fn query_during_wft_heartbeat_doesnt_accidentally_fail_to_continue_heartbe
 
     tokio::join!(wf_fut, act_fut);
 }
+
+#[rstest::rstest]
+#[case::impossible_query_in_task(true)]
+#[case::real_history(false)]
+#[tokio::test]
+async fn la_resolve_during_legacy_query_does_not_combine(#[case] impossible_query_in_task: bool) {
+    // Ensures we do not send an activation with a legacy query and any other work, which should
+    // never happen, but there was an issue where an LA resolving could trigger that.
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    let wes_short_wft_timeout = default_wes_attribs();
+    t.add(
+        EventType::WorkflowExecutionStarted,
+        wes_short_wft_timeout.into(),
+    );
+    // Since we don't send queries with start workflow, need one workflow task of something else
+    // b/c we want to get an activation with a job and a nonlegacy query
+    t.add_full_wf_task();
+    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+
+    // nonlegacy query got here & LA started here
+    t.add_full_wf_task();
+    // legacy query got here, at the same time that the LA is resolved
+    t.add_local_activity_result_marker(1, "1", "whatever".into());
+    t.add_workflow_execution_completed();
+
+    let barr = Arc::new(Barrier::new(2));
+    let barr_c = barr.clone();
+
+    let tasks = [
+        hist_to_poll_resp(&t, wfid.to_owned(), ResponseType::ToTaskNum(1)),
+        {
+            let mut pr = hist_to_poll_resp(&t, wfid.to_owned(), ResponseType::OneTask(2));
+            pr.queries = HashMap::new();
+            pr.queries.insert(
+                "q1".to_string(),
+                WorkflowQuery {
+                    query_type: "query-type".to_string(),
+                    query_args: Some(b"hi".into()),
+                    header: None,
+                },
+            );
+            pr
+        },
+        {
+            let mut pr = hist_to_poll_resp(
+                &t,
+                wfid.to_owned(),
+                ResponseType::UntilResolved(
+                    async move {
+                        barr_c.wait().await;
+                        // This sleep is the only not-incredibly-invasive way to ensure the LA
+                        // resolves & updates machines before we process this task
+                        tokio::time::sleep(Duration::from_secs(1)).await;
+                    }
+                    .boxed(),
+                    2,
+                ),
+            );
+            // Strip history, we need to look like we hit the cache
+            pr.history = Some(History { events: vec![] });
+            // In the nonsense server response case, we attach a legacy query, otherwise this
+            // response looks like a normal response to a forced WFT heartbeat.
+            if impossible_query_in_task {
+                pr.query = Some(WorkflowQuery {
+                    query_type: "query-type".to_string(),
+                    query_args: Some(b"hi".into()),
+                    header: None,
+                });
+            }
+            pr
+        },
+    ];
+    let mut mock = mock_workflow_client();
+    if impossible_query_in_task {
+        mock.expect_respond_legacy_query()
+            .times(1)
+            .returning(move |_, _| Ok(Default::default()));
+    }
+    let mut mock = single_hist_mock_sg(wfid, t, tasks, mock, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let wf_fut = async {
+        let task = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            task.jobs.as_slice(),
+            &[WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
+            },]
+        );
+        core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+            task.run_id,
+            start_timer_cmd(1, Duration::from_secs(1)),
+        ))
+        .await
+        .unwrap();
+        let task = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            task.jobs.as_slice(),
+            &[
+                WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::FireTimer(_)),
+                },
+                WorkflowActivationJob {
+                    variant: Some(workflow_activation_job::Variant::QueryWorkflow(_)),
+                }
+            ]
+        );
+        core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+            task.run_id,
+            vec![
+                schedule_local_activity_cmd(
+                    1,
+                    "act-id",
+                    ActivityCancellationType::TryCancel,
+                    Duration::from_secs(60),
+                ),
+                QueryResult {
+                    query_id: "q1".to_string(),
+                    variant: Some(
+                        QuerySuccess {
+                            response: Some("whatev".into()),
+                        }
+                        .into(),
+                    ),
+                }
+                .into(),
+            ],
+        ))
+        .await
+        .unwrap();
+        barr.wait().await;
+        let task = core.poll_workflow_activation().await.unwrap();
+        // The next task needs to be resolve, since the LA is completed immediately
+        assert_matches!(
+            task.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::ResolveActivity(_)),
+            }]
+        );
+        // Complete workflow
+        core.complete_execution(&task.run_id).await;
+        if impossible_query_in_task {
+            // finish last query
+            let task = core.poll_workflow_activation().await.unwrap();
+            core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+                task.run_id,
+                vec![QueryResult {
+                    query_id: LEGACY_QUERY_ID.to_string(),
+                    variant: Some(
+                        QuerySuccess {
+                            response: Some("whatev".into()),
+                        }
+                        .into(),
+                    ),
+                }
+                .into()],
+            ))
+            .await
+            .unwrap();
+        }
+    };
+    let act_fut = async {
+        let act_task = core.poll_activity_task().await.unwrap();
+        core.complete_activity_task(ActivityTaskCompletion {
+            task_token: act_task.task_token,
+            result: Some(ActivityExecutionResult::ok(vec![1].into())),
+        })
+        .await
+        .unwrap();
+    };
+
+    tokio::join!(wf_fut, act_fut);
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn test_schedule_to_start_timeout() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mh = MockPollCfg::from_resp_batches(wf_id, t, [ResponseType::ToTaskNum(1)], mock);
+    let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        |ctx: WfContext| async move {
+            let la_res = ctx
+                .local_activity(LocalActivityOptions {
+                    activity_type: "echo".to_string(),
+                    input: "hi".as_json_payload().expect("serializes fine"),
+                    // Impossibly small timeout so we timeout in the queue
+                    schedule_to_start_timeout: prost_dur!(from_nanos(1)),
+                    ..Default::default()
+                })
+                .await;
+            assert_eq!(la_res.timed_out(), Some(TimeoutType::ScheduleToStart));
+            Ok(().into())
+        },
+    );
+    worker.register_activity(
+        "echo",
+        move |_ctx: ActContext, _: String| async move { Ok(()) },
+    );
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
+#[tokio::test]
+async fn test_schedule_to_start_timeout_not_based_on_original_time() {
+    // We used to carry over the schedule time of LAs from the "original" schedule time if these LAs
+    // created newly after backing off across a timer. That was a mistake, since schedule-to-start
+    // timeouts should apply to when the new attempt was scheduled. This test verifies we don't
+    // time out on s-t-s timeouts because of that.
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    let orig_sched = SystemTime::now().sub(Duration::from_secs(60 * 20));
+    t.add_local_activity_marker(
+        1,
+        "1",
+        None,
+        Some(Failure::application_failure("la failed".to_string(), false)),
+        |deets| {
+            // Really old schedule time, which should _not_ count against schedule_to_start
+            deets.original_schedule_time = Some(orig_sched.into());
+            // Backoff value must be present since we're simulating timer backoff
+            deets.backoff = Some(prost_dur!(from_secs(100)));
+        },
+    );
+    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_workflow_task_scheduled_and_started();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mh = MockPollCfg::from_resp_batches(wf_id, t, [ResponseType::AllHistory], mock);
+    let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        |ctx: WfContext| async move {
+            let la_res = ctx
+                .local_activity(LocalActivityOptions {
+                    activity_type: "echo".to_string(),
+                    input: "hi".as_json_payload().expect("serializes fine"),
+                    retry_policy: RetryPolicy {
+                        initial_interval: Some(prost_dur!(from_millis(50))),
+                        backoff_coefficient: 1.2,
+                        maximum_interval: None,
+                        maximum_attempts: 5,
+                        non_retryable_error_types: vec![],
+                    },
+                    schedule_to_start_timeout: Some(Duration::from_secs(60)),
+                    schedule_to_close_timeout: Some(Duration::from_secs(60 * 60)),
+                    ..Default::default()
+                })
+                .await;
+            assert!(la_res.completed_ok());
+            Ok(().into())
+        },
+    );
+    worker.register_activity(
+        "echo",
+        move |_ctx: ActContext, _: String| async move { Ok(()) },
+    );
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
+#[tokio::test]
+async fn test_schedule_to_close_timeout_based_on_original_time() {
+    // We used to carry over the schedule time of LAs from the "original" schedule time if these LAs
+    // created newly after backing off across a timer. That was a mistake, since schedule-to-start
+    // timeouts should apply to when the new attempt was scheduled. This test verifies we don't
+    // time out on s-t-s timeouts because of that.
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    let orig_sched = SystemTime::now().sub(Duration::from_secs(60 * 20));
+    t.add_local_activity_marker(
+        1,
+        "1",
+        None,
+        Some(Failure::application_failure("la failed".to_string(), false)),
+        |deets| {
+            // Really old schedule time, which should _not_ count against schedule_to_start
+            deets.original_schedule_time = Some(orig_sched.into());
+            // Backoff value must be present since we're simulating timer backoff
+            deets.backoff = Some(prost_dur!(from_secs(100)));
+        },
+    );
+    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_workflow_task_scheduled_and_started();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mh = MockPollCfg::from_resp_batches(wf_id, t, [ResponseType::AllHistory], mock);
+    let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        |ctx: WfContext| async move {
+            let la_res = ctx
+                .local_activity(LocalActivityOptions {
+                    activity_type: "echo".to_string(),
+                    input: "hi".as_json_payload().expect("serializes fine"),
+                    retry_policy: RetryPolicy {
+                        initial_interval: Some(prost_dur!(from_millis(50))),
+                        backoff_coefficient: 1.2,
+                        maximum_interval: None,
+                        maximum_attempts: 5,
+                        non_retryable_error_types: vec![],
+                    },
+                    // This 10 minute timeout will have already elapsed according to the original
+                    // schedule time in the history.
+                    schedule_to_close_timeout: Some(Duration::from_secs(10 * 60)),
+                    ..Default::default()
+                })
+                .await;
+            assert_eq!(la_res.timed_out(), Some(TimeoutType::ScheduleToClose));
+            Ok(().into())
+        },
+    );
+    worker.register_activity(
+        "echo",
+        move |_ctx: ActContext, _: String| async move { Ok(()) },
+    );
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}