@temporalio/core-bridge 0.21.0 → 0.23.0

package/sdk-core/core/src/core_tests/queries.rs

@@ -11,8 +11,13 @@ use std::{
 use temporal_sdk_core_api::Worker as WorkerTrait;
 use temporal_sdk_core_protos::{
     coresdk::{
-        workflow_activation::{workflow_activation_job, WorkflowActivationJob},
-        workflow_commands::{CompleteWorkflowExecution, QueryResult, QuerySuccess},
+        workflow_activation::{
+            remove_from_cache::EvictionReason, workflow_activation_job, WorkflowActivationJob,
+        },
+        workflow_commands::{
+            ActivityCancellationType, CompleteWorkflowExecution, ContinueAsNewWorkflowExecution,
+            QueryResult, QuerySuccess, RequestCancelActivity,
+        },
         workflow_completion::WorkflowActivationCompletion,
     },
     temporal::api::{
@@ -26,7 +31,7 @@ use temporal_sdk_core_protos::{
         },
     },
 };
-use temporal_sdk_core_test_utils::start_timer_cmd;
+use temporal_sdk_core_test_utils::{schedule_activity_cmd, start_timer_cmd};
 
 #[rstest::rstest]
 #[case::with_history(true)]
@@ -81,6 +86,10 @@ async fn legacy_query(#[case] include_history: bool) {
     };
     let clear_eviction = || async {
         let t = worker.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            t.jobs[0].variant,
+            Some(workflow_activation_job::Variant::RemoveFromCache(_))
+        );
         worker
             .complete_workflow_activation(WorkflowActivationCompletion::empty(t.run_id))
             .await
@@ -156,7 +165,12 @@ async fn new_queries(#[case] num_queries: usize) {
     let tasks = VecDeque::from(vec![
         hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()),
         {
-            let mut pr = hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string());
+            let mut pr = hist_to_poll_resp(
+                &t,
+                wfid.to_owned(),
+                ResponseType::OneTask(2),
+                TEST_Q.to_string(),
+            );
             pr.queries = HashMap::new();
             for i in 1..=num_queries {
                 pr.queries.insert(
@@ -383,20 +397,50 @@ async fn legacy_query_after_complete(#[values(false, true)] full_history: bool)
     core.shutdown().await;
 }
 
+enum QueryHists {
+    Empty,
+    Full,
+    Partial,
+}
+#[rstest::rstest]
 #[tokio::test]
-async fn query_cache_miss_causes_page_fetch_dont_reply_wft_too_early() {
+async fn query_cache_miss_causes_page_fetch_dont_reply_wft_too_early(
+    #[values(QueryHists::Empty, QueryHists::Full, QueryHists::Partial)] hist_type: QueryHists,
+) {
     let wfid = "fake_wf_id";
     let query_resp = "response";
     let t = canned_histories::single_timer("1");
     let full_hist = t.get_full_history_info().unwrap();
     let tasks = VecDeque::from(vec![{
-        // Create a partial task
-        let mut pr = hist_to_poll_resp(
-            &t,
-            wfid.to_owned(),
-            ResponseType::OneTask(2),
-            TEST_Q.to_string(),
-        );
+        let mut pr = match hist_type {
+            QueryHists::Empty => {
+                // Create a no-history poll response. This happens to be easiest to do by just ripping
+                // out the history after making a normal one.
+                let mut pr = hist_to_poll_resp(
+                    &t,
+                    wfid.to_owned(),
+                    ResponseType::AllHistory,
+                    TEST_Q.to_string(),
+                );
+                pr.history = Some(Default::default());
+                pr
+            }
+            QueryHists::Full => hist_to_poll_resp(
+                &t,
+                wfid.to_owned(),
+                ResponseType::AllHistory,
+                TEST_Q.to_string(),
+            ),
+            QueryHists::Partial => {
+                // Create a partial task
+                hist_to_poll_resp(
+                    &t,
+                    wfid.to_owned(),
+                    ResponseType::OneTask(2),
+                    TEST_Q.to_string(),
+                )
+            }
+        };
         pr.queries = HashMap::new();
         pr.queries.insert(
             "the-query".to_string(),
@@ -409,14 +453,16 @@ async fn query_cache_miss_causes_page_fetch_dont_reply_wft_too_early() {
         pr
     }]);
     let mut mock_client = mock_workflow_client();
-    mock_client
-        .expect_get_workflow_execution_history()
-        .returning(move |_, _, _| {
-            Ok(GetWorkflowExecutionHistoryResponse {
-                history: Some(full_hist.clone().into()),
-                ..Default::default()
-            })
-        });
+    if !matches!(hist_type, QueryHists::Full) {
+        mock_client
+            .expect_get_workflow_execution_history()
+            .returning(move |_, _, _| {
+                Ok(GetWorkflowExecutionHistoryResponse {
+                    history: Some(full_hist.clone().into()),
+                    ..Default::default()
+                })
+            });
+    }
     mock_client
         .expect_complete_workflow_task()
         .times(1)
@@ -487,3 +533,118 @@ async fn query_cache_miss_causes_page_fetch_dont_reply_wft_too_early() {
 
     core.shutdown().await;
 }
+
+#[tokio::test]
+async fn query_replay_with_continue_as_new_doesnt_reply_empty_command() {
+    let wfid = "fake_wf_id";
+    let t = canned_histories::single_timer("1");
+    let query_with_hist_task = {
+        let mut pr = hist_to_poll_resp(
+            &t,
+            wfid.to_owned(),
+            ResponseType::ToTaskNum(1),
+            TEST_Q.to_string(),
+        );
+        pr.queries = HashMap::new();
+        pr.queries.insert(
+            "the-query".to_string(),
+            WorkflowQuery {
+                query_type: "query-type".to_string(),
+                query_args: Some(b"hi".into()),
+                header: None,
+            },
+        );
+        pr
+    };
+    let tasks = VecDeque::from(vec![query_with_hist_task]);
+    let mut mock_client = mock_workflow_client();
+    mock_client
+        .expect_complete_workflow_task()
+        .times(1)
+        .returning(|resp| {
+            // Verify both the complete command and the query response are sent
+            assert_eq!(resp.commands.len(), 1);
+            assert_eq!(resp.query_responses.len(), 1);
+            Ok(RespondWorkflowTaskCompletedResponse::default())
+        });
+
+    let mut mock = MocksHolder::from_client_with_responses(mock_client, tasks, vec![]);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    let core = mock_worker(mock);
+
+    let task = core.poll_workflow_activation().await.unwrap();
+    // Scheduling and immediately canceling an activity produces another activation which is
+    // important in this repro
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        task.run_id,
+        vec![
+            schedule_activity_cmd(
+                0,
+                "whatever",
+                "act-id",
+                ActivityCancellationType::TryCancel,
+                Duration::from_secs(60),
+                Duration::from_secs(60),
+            ),
+            RequestCancelActivity { seq: 0 }.into(),
+            ContinueAsNewWorkflowExecution {
+                ..Default::default()
+            }
+            .into(),
+        ],
+    ))
+    .await
+    .unwrap();
+
+    // Activity unblocked
+    let task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::ResolveActivity(_)),
+        }]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(task.run_id))
+        .await
+        .unwrap();
+
+    let task = core.poll_workflow_activation().await.unwrap();
+    let query = assert_matches!(
+        task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::QueryWorkflow(q)),
+        }] => q
+    );
+    // Throw an evict in there. Repro required a pending eviction during complete.
+    core.request_wf_eviction(&task.run_id, "I said so", EvictionReason::LangRequested);
+
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        task.run_id,
+        QueryResult {
+            query_id: query.query_id.clone(),
+            variant: Some(
+                QuerySuccess {
+                    response: Some("whatever".into()),
+                }
+                .into(),
+            ),
+        }
+        .into(),
+    ))
+    .await
+    .unwrap();
+
+    // Need to complete the eviction to finally send commands and finish
+    let task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(_))
+        }]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(task.run_id))
+        .await
+        .unwrap();
+
+    core.shutdown().await;
+}

package/sdk-core/core/src/core_tests/workers.rs

@@ -20,38 +20,6 @@ use temporal_sdk_core_protos::{
 use temporal_sdk_core_test_utils::start_timer_cmd;
 use tokio::sync::{watch, Barrier};
 
-#[tokio::test]
-async fn multi_workers() {
-    // TODO: Turn this into a test with multiple independent workers
-    // Make histories for 5 different workflows on 5 different task queues
-    // let hists = (0..5).into_iter().map(|i| {
-    //     let wf_id = format!("fake-wf-{}", i);
-    //     let hist = canned_histories::single_timer("1");
-    //     FakeWfResponses {
-    //         wf_id,
-    //         hist,
-    //         response_batches: vec![1.into(), 2.into()],
-    //         task_q: format!("q-{}", i),
-    //     }
-    // });
-    // let mock = build_multihist_mock_sg(hists, false, None);
-    //
-    // let core = &mock_worker(mock);
-    //
-    // for i in 0..5 {
-    //     let tq = format!("q-{}", i);
-    //     let res = core.poll_workflow_activation().await.unwrap();
-    //     assert_matches!(
-    //         res.jobs[0].variant,
-    //         Some(workflow_activation_job::Variant::StartWorkflow(_))
-    //     );
-    //     core.complete_workflow_activation(WorkflowActivationCompletion::empty(res.run_id))
-    //         .await
-    //         .unwrap();
-    // }
-    // core.shutdown().await;
-}
-
 #[tokio::test]
 async fn after_shutdown_of_worker_get_shutdown_err() {
     let t = canned_histories::single_timer("1");
@@ -195,7 +163,7 @@ async fn can_shutdown_local_act_only_worker_when_act_polling() {
 }
 
 #[tokio::test]
-async fn complete_with_task_not_found_during_shutdwn() {
+async fn complete_with_task_not_found_during_shutdown() {
     let t = canned_histories::single_timer("1");
     let mut mock = mock_workflow_client();
     mock.expect_complete_workflow_task()
@@ -236,5 +204,7 @@ async fn complete_with_task_not_found_during_shutdwn() {
         complete_order.borrow_mut().push(1);
     };
     tokio::join!(shutdown_fut, poll_fut, complete_fut);
-    assert_eq!(&complete_order.into_inner(), &[1, 2, 3])
+    // Shutdown will currently complete first before the actual eviction reply since the
+    // workflow task is marked complete as soon as we get not found back from the server.
+    assert_eq!(&complete_order.into_inner(), &[1, 3, 2])
 }

package/sdk-core/core/src/core_tests/workflow_tasks.rs

@@ -71,7 +71,7 @@ fn single_activity_failure_setup(hist_batches: &'static [usize]) -> Worker {
 #[case::incremental(single_timer_setup(&[1, 2]), NonSticky)]
 #[case::replay(single_timer_setup(&[2]), NonSticky)]
 #[case::incremental_evict(single_timer_setup(&[1, 2]), AfterEveryReply)]
-#[case::replay_evict(single_timer_setup(&[2, 2]), AfterEveryReply)]
+#[case::replay_evict(single_timer_setup(&[2]), AfterEveryReply)]
 #[tokio::test]
 async fn single_timer(#[case] worker: Worker, #[case] evict: WorkflowCachingPolicy) {
     poll_and_reply(
@@ -1114,14 +1114,9 @@ async fn complete_after_eviction() {
     let eviction_activation = core.poll_workflow_activation().await.unwrap();
     assert_matches!(
         eviction_activation.jobs.as_slice(),
-        [
-            WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::FireTimer(_)),
-            },
-            WorkflowActivationJob {
-                variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
-            }
-        ]
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::FireTimer(_)),
+        },]
     );
     // Complete the activation containing the eviction, the way we normally would have
     core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
@@ -1130,6 +1125,13 @@ async fn complete_after_eviction() {
     ))
     .await
     .unwrap();
+    let eviction = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        eviction.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
+        }]
+    );
     core.shutdown().await;
 }
 
@@ -1260,38 +1262,6 @@ async fn buffered_work_drained_on_shutdown() {
     });
 }
 
-#[tokio::test]
-async fn buffering_tasks_doesnt_count_toward_outstanding_max() {
-    let wfid = "fake_wf_id";
-    let t = canned_histories::single_timer("1");
-    let mock = mock_workflow_client();
-    let mut tasks = VecDeque::new();
-    // A way bigger task list than allowed outstanding tasks
-    tasks.extend(
-        std::iter::repeat(hist_to_poll_resp(
-            &t,
-            wfid.to_owned(),
-            2.into(),
-            TEST_Q.to_string(),
-        ))
-        .take(20),
-    );
-    let mut mock = MocksHolder::from_client_with_responses(mock, tasks, []);
-    mock.worker_cfg(|wc| {
-        wc.max_cached_workflows = 10;
-        wc.max_outstanding_workflow_tasks = 5;
-    });
-    let core = mock_worker(mock);
-    // Poll for first WFT
-    core.poll_workflow_activation().await.unwrap();
-    // This will error out when the mock runs out of responses. Otherwise it would hang when we
-    // hit the max
-    assert_matches!(
-        core.poll_workflow_activation().await.unwrap_err(),
-        PollWfError::TonicError(_)
-    );
-}
-
 #[tokio::test]
 async fn fail_wft_then_recover() {
     let t = canned_histories::long_sequential_timers(1);
@@ -1737,3 +1707,189 @@ async fn evict_missing_wf_during_poll_doesnt_eat_permit() {
 
     core.shutdown().await;
 }
+
+#[tokio::test]
+async fn poll_faster_than_complete_wont_overflow_cache() {
+    // Make workflow tasks for 5 different runs
+    let tasks: Vec<_> = (1..=5)
+        .map(|i| {
+            hist_to_poll_resp(
+                // New hist each time for new run ids
+                &canned_histories::single_timer("1"),
+                format!("wf-{}", i),
+                ResponseType::ToTaskNum(1),
+                TEST_Q.to_string(),
+            )
+        })
+        .collect();
+    let mut mock = mock_workflow_client();
+    mock.expect_complete_workflow_task()
+        .times(3)
+        .returning(|_| Ok(Default::default()));
+    let mut mock = MocksHolder::from_client_with_responses(mock, tasks, []);
+    mock.worker_cfg(|wc| {
+        wc.max_cached_workflows = 3;
+        wc.max_outstanding_workflow_tasks = 3;
+    });
+    let core = mock_worker(mock);
+    // Poll 4 times, completing once, such that max tasks are never exceeded
+    let p1 = core.poll_workflow_activation().await.unwrap();
+    let p2 = core.poll_workflow_activation().await.unwrap();
+    let p3 = core.poll_workflow_activation().await.unwrap();
+    for (i, p_res) in [&p1, &p2, &p3].into_iter().enumerate() {
+        assert_matches!(
+            &p_res.jobs[0].variant,
+            Some(workflow_activation_job::Variant::StartWorkflow(sw))
+            if sw.workflow_id == format!("wf-{}", i + 1)
+        );
+    }
+    // Complete first task to free a wft slot. Cache size is at 3
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        p1.run_id,
+        start_timer_cmd(1, Duration::from_secs(1)),
+    ))
+    .await
+    .unwrap();
+    // Now we're at cache limit. We will poll for a task, discover it is for a new run, issue
+    // an eviction, and buffer the new run task. However, the run we're trying to evict has pending
+    // activations! Thus, we must complete them first before this poll will unblock, and then it
+    // will unblock with the eviciton.
+    let p4 = async {
+        let p4 = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            &p4.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
+            }]
+        );
+        p4
+    };
+    let p2_pending_completer = async {
+        // Sleep needed because otherwise the complete unblocks waiting for the cache to free a slot
+        // before we have a chance to actually... wait for it.
+        tokio::time::sleep(Duration::from_millis(100)).await;
+        core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+            p2.run_id,
+            start_timer_cmd(1, Duration::from_secs(1)),
+        ))
+        .await
+        .unwrap();
+    };
+    let (p4, _) = tokio::join!(p4, p2_pending_completer);
+    assert_eq!(core.cached_workflows(), 3);
+
+    // This poll should also block until the eviction is actually completed
+    let blocking_poll = async {
+        let res = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            &res.jobs[0].variant,
+            Some(workflow_activation_job::Variant::StartWorkflow(sw))
+            if sw.workflow_id == format!("wf-{}", 4)
+        );
+        res
+    };
+    let complete_evict = async {
+        core.complete_workflow_activation(WorkflowActivationCompletion::empty(p4.run_id))
+            .await
+            .unwrap();
+    };
+
+    let (_p5, _) = tokio::join!(blocking_poll, complete_evict);
+    assert_eq!(core.cached_workflows(), 3);
+    // The next poll will get an buffer a task for a new run, and generate an eviction for p3 but
+    // that eviction cannot be obtained until we complete the existing outstanding task.
+    let p6 = async {
+        let p6 = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            p6.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
+            }]
+        );
+        p6
+    };
+    let completer = async {
+        core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+            p3.run_id,
+            start_timer_cmd(1, Duration::from_secs(1)),
+        ))
+        .await
+        .unwrap();
+    };
+    let (p6, _) = tokio::join!(p6, completer);
+    let complete_evict = async {
+        core.complete_workflow_activation(WorkflowActivationCompletion::empty(p6.run_id))
+            .await
+            .unwrap();
+    };
+    let blocking_poll = async {
+        // This poll will also block until the last eviction goes through, and when it does it'll
+        // produce the final start workflow task
+        let res = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            &res.jobs[0].variant,
+            Some(workflow_activation_job::Variant::StartWorkflow(sw))
+            if sw.workflow_id == "wf-5"
+        );
+    };
+
+    tokio::join!(blocking_poll, complete_evict);
+    // p5 outstanding and final poll outstanding -- hence one permit available
+    assert_eq!(core.available_wft_permits(), 1);
+    assert_eq!(core.cached_workflows(), 3);
+}
+
+#[tokio::test]
+async fn eviction_waits_until_replay_finished() {
+    let wfid = "fake_wf_id";
+    let t = canned_histories::long_sequential_timers(3);
+    let mock = mock_workflow_client();
+    let mock = single_hist_mock_sg(wfid, t, &[3], mock, true);
+    let core = mock_worker(mock);
+
+    let activation = core.poll_workflow_activation().await.unwrap();
+    // Immediately request eviction after getting start workflow
+    core.request_workflow_eviction(&activation.run_id);
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        activation.run_id,
+        start_timer_cmd(1, Duration::from_secs(1)),
+    ))
+    .await
+    .unwrap();
+    let t1_fired = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        t1_fired.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::FireTimer(_)),
+        }]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        t1_fired.run_id,
+        start_timer_cmd(2, Duration::from_secs(1)),
+    ))
+    .await
+    .unwrap();
+    let t2_fired = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        t2_fired.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::FireTimer(_)),
+        }]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        t2_fired.run_id,
+        vec![CompleteWorkflowExecution { result: None }.into()],
+    ))
+    .await
+    .unwrap();
+    // The first two WFTs were replay, and now that we've caught up, the eviction will be sent
+    let eviction = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        eviction.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
+        }]
+    );
+
+    core.shutdown().await;
+}

package/sdk-core/core/src/lib.rs

@@ -34,7 +34,8 @@ pub use pollers::{
     TlsConfig, WorkflowClientTrait,
 };
 pub use telemetry::{
-    fetch_global_buffered_logs, telemetry_init, TelemetryOptions, TelemetryOptionsBuilder,
+    fetch_global_buffered_logs, telemetry_init, Logger, MetricsExporter, OtelCollectorOptions,
+    TelemetryOptions, TelemetryOptionsBuilder, TraceExporter,
 };
 pub use temporal_sdk_core_api as api;
 pub use temporal_sdk_core_protos as protos;

package/sdk-core/core/src/pending_activations.rs

@@ -22,6 +22,7 @@ struct PaInner {
     queue: VecDeque<ActivationKey>,
 }
 
+#[derive(Debug)]
 pub struct PendingActInfo {
     pub needs_eviction: Option<RemoveFromCache>,
     pub run_id: String,
@@ -41,6 +42,7 @@ impl PendingActivations {
             inner.queue.push_back(key);
         };
     }
+
     pub fn notify_needs_eviction(&self, run_id: &str, message: String, reason: EvictionReason) {
         let mut inner = self.inner.write();
 
@@ -106,6 +108,15 @@ impl PendingActivations {
             inner.activations.remove(k);
         }
     }
+
+    /// Returns true if any pending activation contains an eviction
+    pub fn is_some_eviction(&self) -> bool {
+        self.inner
+            .read()
+            .activations
+            .values()
+            .any(|act| act.needs_eviction.is_some())
+    }
 }
 
 #[cfg(test)]

package/sdk-core/core/src/protosext/mod.rs

@@ -359,9 +359,7 @@ impl ValidScheduleLA {
                 ))
             }
         };
-        let retry_policy = v
-            .retry_policy
-            .ok_or_else(|| anyhow!("Retry policy must be defined!"))?;
+        let retry_policy = v.retry_policy.unwrap_or_default();
         let local_retry_threshold = v
             .local_retry_threshold
             .clone()