@temporalio/core-bridge 1.5.2 → 1.6.0

package/sdk-core/core/src/core_tests/local_activities.rs

@@ -11,14 +11,17 @@ use anyhow::anyhow;
 use futures::{future::join_all, FutureExt};
 use std::{
     collections::HashMap,
+    ops::Sub,
     sync::{
         atomic::{AtomicUsize, Ordering},
         Arc,
     },
-    time::Duration,
+    time::{Duration, SystemTime},
 };
 use temporal_client::WorkflowOptions;
-use temporal_sdk::{ActContext, LocalActivityOptions, WfContext, WorkflowResult};
+use temporal_sdk::{
+    ActContext, ActivityCancelledError, LocalActivityOptions, WfContext, WorkflowResult,
+};
 use temporal_sdk_core_api::Worker;
 use temporal_sdk_core_protos::{
     coresdk::{
@@ -29,7 +32,9 @@ use temporal_sdk_core_protos::{
         ActivityTaskCompletion, AsJsonPayloadExt,
     },
     temporal::api::{
-        common::v1::RetryPolicy, enums::v1::EventType, failure::v1::Failure, history::v1::History,
+        common::v1::RetryPolicy,
+        enums::v1::{EventType, TimeoutType, WorkflowTaskFailedCause},
+        failure::v1::Failure,
         query::v1::WorkflowQuery,
     },
 };
@@ -106,7 +111,7 @@ pub async fn local_act_fanout_wf(ctx: WfContext) -> WorkflowResult<()> {
         .map(|i| {
             ctx.local_activity(LocalActivityOptions {
                 activity_type: "echo".to_string(),
-                input: format!("Hi {}", i)
+                input: format!("Hi {i}")
                     .as_json_payload()
                     .expect("serializes fine"),
                 ..Default::default()
@@ -120,6 +125,7 @@ pub async fn local_act_fanout_wf(ctx: WfContext) -> WorkflowResult<()> {
 
 #[tokio::test]
 async fn local_act_many_concurrent() {
+    crate::telemetry::test_telem_console();
     let mut t = TestHistoryBuilder::default();
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
@@ -164,12 +170,7 @@ async fn local_act_many_concurrent() {
 async fn local_act_heartbeat(#[case] shutdown_middle: bool) {
     let mut t = TestHistoryBuilder::default();
     let wft_timeout = Duration::from_millis(200);
-    let mut wes_short_wft_timeout = default_wes_attribs();
-    wes_short_wft_timeout.workflow_task_timeout = Some(wft_timeout.try_into().unwrap());
-    t.add(
-        EventType::WorkflowExecutionStarted,
-        wes_short_wft_timeout.into(),
-    );
+    t.add_wfe_started_with_wft_timeout(wft_timeout);
     t.add_full_wf_task();
     // Task created by WFT heartbeat
     t.add_full_wf_task();
@@ -367,7 +368,7 @@ async fn local_act_null_result() {
     let mut t = TestHistoryBuilder::default();
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
-    t.add_local_activity_marker(1, "1", None, None, None);
+    t.add_local_activity_marker(1, "1", None, None, |_| {});
     t.add_workflow_execution_completed();
 
     let wf_id = "fakeid";
@@ -408,7 +409,7 @@ async fn local_act_command_immediately_follows_la_marker() {
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
     t.add_full_wf_task();
-    t.add_local_activity_marker(1, "1", None, None, None);
+    t.add_local_activity_result_marker(1, "1", "done".into());
     t.add_get_event_id(EventType::TimerStarted, None);
     t.add_full_wf_task();
 
@@ -448,16 +449,11 @@ async fn local_act_command_immediately_follows_la_marker() {
 async fn query_during_wft_heartbeat_doesnt_accidentally_fail_to_continue_heartbeat() {
     let wfid = "fake_wf_id";
     let mut t = TestHistoryBuilder::default();
-    let mut wes_short_wft_timeout = default_wes_attribs();
-    wes_short_wft_timeout.workflow_task_timeout = Some(prost_dur!(from_millis(200)));
-    t.add(
-        EventType::WorkflowExecutionStarted,
-        wes_short_wft_timeout.into(),
-    );
+    t.add_wfe_started_with_wft_timeout(Duration::from_millis(200));
     t.add_full_wf_task();
     // get query here
     t.add_full_wf_task();
-    t.add_local_activity_marker(1, "1", None, None, None);
+    t.add_local_activity_result_marker(1, "1", "done".into());
     t.add_workflow_execution_completed();
 
     let query_with_hist_task = {
@@ -582,7 +578,7 @@ async fn la_resolve_during_legacy_query_does_not_combine(#[case] impossible_quer
     // nonlegacy query got here & LA started here
     t.add_full_wf_task();
     // legacy query got here, at the same time that the LA is resolved
-    t.add_local_activity_marker(1, "1", None, None, None);
+    t.add_local_activity_result_marker(1, "1", "whatever".into());
     t.add_workflow_execution_completed();
 
     let barr = Arc::new(Barrier::new(2));
@@ -618,8 +614,12 @@ async fn la_resolve_during_legacy_query_does_not_combine(#[case] impossible_quer
                     2,
                 ),
             );
-            // Strip history, we need to look like we hit the cache
-            pr.history = Some(History { events: vec![] });
+            // Strip beginning of history so the only events are WFT sched/started, we need to look
+            // like we hit the cache
+            {
+                let h = pr.history.as_mut().unwrap();
+                h.events = h.events.split_off(6);
+            }
             // In the nonsense server response case, we attach a legacy query, otherwise this
             // response looks like a normal response to a forced WFT heartbeat.
             if impossible_query_in_task {
@@ -735,3 +735,244 @@ async fn la_resolve_during_legacy_query_does_not_combine(#[case] impossible_quer
     tokio::join!(wf_fut, act_fut);
     core.shutdown().await;
 }
+
+#[tokio::test]
+async fn test_schedule_to_start_timeout() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mh = MockPollCfg::from_resp_batches(wf_id, t, [ResponseType::ToTaskNum(1)], mock);
+    let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        |ctx: WfContext| async move {
+            let la_res = ctx
+                .local_activity(LocalActivityOptions {
+                    activity_type: "echo".to_string(),
+                    input: "hi".as_json_payload().expect("serializes fine"),
+                    // Impossibly small timeout so we timeout in the queue
+                    schedule_to_start_timeout: prost_dur!(from_nanos(1)),
+                    ..Default::default()
+                })
+                .await;
+            assert_eq!(la_res.timed_out(), Some(TimeoutType::ScheduleToStart));
+            Ok(().into())
+        },
+    );
+    worker.register_activity(
+        "echo",
+        move |_ctx: ActContext, _: String| async move { Ok(()) },
+    );
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
+#[rstest::rstest]
+#[case::sched_to_start(true)]
+#[case::sched_to_close(false)]
+#[tokio::test]
+async fn test_schedule_to_start_timeout_not_based_on_original_time(
+    #[case] is_sched_to_start: bool,
+) {
+    // We used to carry over the schedule time of LAs from the "original" schedule time if these LAs
+    // created newly after backing off across a timer. That was a mistake, since schedule-to-start
+    // timeouts should apply to when the new attempt was scheduled. This test verifies:
+    // * we don't time out on s-t-s timeouts because of that, when the param is true.
+    // * we do properly time out on s-t-c timeouts when the param is false
+
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    let orig_sched = SystemTime::now().sub(Duration::from_secs(60 * 20));
+    t.add_local_activity_marker(
+        1,
+        "1",
+        None,
+        Some(Failure::application_failure("la failed".to_string(), false)),
+        |deets| {
+            // Really old schedule time, which should _not_ count against schedule_to_start
+            deets.original_schedule_time = Some(orig_sched.into());
+            // Backoff value must be present since we're simulating timer backoff
+            deets.backoff = Some(prost_dur!(from_secs(100)));
+        },
+    );
+    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_workflow_task_scheduled_and_started();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mh = MockPollCfg::from_resp_batches(wf_id, t, [ResponseType::AllHistory], mock);
+    let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
+
+    let schedule_to_close_timeout = Some(if is_sched_to_start {
+        // This 60 minute timeout will not have elapsed according to the original
+        // schedule time in the history.
+        Duration::from_secs(60 * 60)
+    } else {
+        // This 10 minute timeout will have already elapsed
+        Duration::from_secs(10 * 60)
+    });
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        move |ctx: WfContext| async move {
+            let la_res = ctx
+                .local_activity(LocalActivityOptions {
+                    activity_type: "echo".to_string(),
+                    input: "hi".as_json_payload().expect("serializes fine"),
+                    retry_policy: RetryPolicy {
+                        initial_interval: Some(prost_dur!(from_millis(50))),
+                        backoff_coefficient: 1.2,
+                        maximum_interval: None,
+                        maximum_attempts: 5,
+                        non_retryable_error_types: vec![],
+                    },
+                    schedule_to_start_timeout: Some(Duration::from_secs(60)),
+                    schedule_to_close_timeout,
+                    ..Default::default()
+                })
+                .await;
+            if is_sched_to_start {
+                assert!(la_res.completed_ok());
+            } else {
+                assert_eq!(la_res.timed_out(), Some(TimeoutType::ScheduleToClose));
+            }
+            Ok(().into())
+        },
+    );
+    worker.register_activity(
+        "echo",
+        move |_ctx: ActContext, _: String| async move { Ok(()) },
+    );
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
+#[tokio::test]
+async fn wft_failure_cancels_running_las() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_wfe_started_with_wft_timeout(Duration::from_millis(200));
+    t.add_full_wf_task();
+    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_workflow_task_scheduled_and_started();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mut mh = MockPollCfg::from_resp_batches(wf_id, t, [1, 2], mock);
+    mh.num_expected_fails = 1;
+    let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        |ctx: WfContext| async move {
+            let la_handle = ctx.local_activity(LocalActivityOptions {
+                activity_type: "echo".to_string(),
+                input: "hi".as_json_payload().expect("serializes fine"),
+                ..Default::default()
+            });
+            tokio::join!(
+                async {
+                    ctx.timer(Duration::from_secs(1)).await;
+                    panic!("ahhh I'm failing wft")
+                },
+                la_handle
+            );
+            Ok(().into())
+        },
+    );
+    worker.register_activity("echo", move |ctx: ActContext, _: String| async move {
+        let res = tokio::time::timeout(Duration::from_millis(500), ctx.cancelled()).await;
+        if res.is_err() {
+            panic!("Activity must be cancelled!!!!");
+        }
+        Result::<(), _>::Err(ActivityCancelledError::default().into())
+    });
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}
+
+#[tokio::test]
+async fn resolved_las_not_recorded_if_wft_fails_many_times() {
+    // We shouldn't record any LA results if the workflow activation is repeatedly failing. There
+    // was an issue that, because we stop reporting WFT failures after 2 tries, this meant the WFT
+    // was not marked as "completed" and the WFT could accidentally be replied to with LA results.
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+    t.add_workflow_task_failed_with_failure(
+        WorkflowTaskFailedCause::Unspecified,
+        Default::default(),
+    );
+    t.add_workflow_task_scheduled_and_started();
+
+    let wf_id = "fakeid";
+    let mock = mock_workflow_client();
+    let mut mh = MockPollCfg::from_resp_batches(
+        wf_id,
+        t,
+        [1.into(), ResponseType::AllHistory, ResponseType::AllHistory],
+        mock,
+    );
+    mh.num_expected_fails = 2;
+    mh.completion_asserts = Some(Box::new(|_| {
+        panic!("should never successfully complete a WFT");
+    }));
+    let mut worker = mock_sdk_cfg(mh, |w| w.max_cached_workflows = 1);
+
+    worker.register_wf(
+        DEFAULT_WORKFLOW_TYPE.to_owned(),
+        |ctx: WfContext| async move {
+            ctx.local_activity(LocalActivityOptions {
+                activity_type: "echo".to_string(),
+                input: "hi".as_json_payload().expect("serializes fine"),
+                ..Default::default()
+            })
+            .await;
+            panic!("Oh nooooo")
+        },
+    );
+    worker.register_activity(
+        "echo",
+        move |_: ActContext, _: String| async move { Ok(()) },
+    );
+    worker
+        .submit_wf(
+            wf_id.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}

package/sdk-core/core/src/core_tests/queries.rs

@@ -162,7 +162,7 @@ async fn new_queries(#[case] num_queries: usize) {
         pr.queries = HashMap::new();
         for i in 1..=num_queries {
             pr.queries.insert(
-                format!("q{}", i),
+                format!("q{i}"),
                 WorkflowQuery {
                     query_type: "query-type".to_string(),
                     query_args: Some(b"hi".into()),
@@ -206,7 +206,7 @@ async fn new_queries(#[case] num_queries: usize) {
     let mut qresults: Vec<_> = (1..=num_queries)
         .map(|i| {
             QueryResult {
-                query_id: format!("q{}", i),
+                query_id: format!("q{i}"),
                 variant: Some(
                     QuerySuccess {
                         response: Some(query_resp.into()),

package/sdk-core/core/src/core_tests/workflow_tasks.rs

@@ -892,7 +892,7 @@ async fn max_wft_respected() {
     let total_wfs = 100;
     let wf_ids: Vec<_> = (0..total_wfs)
         .into_iter()
-        .map(|i| format!("fake-wf-{}", i))
+        .map(|i| format!("fake-wf-{i}"))
         .collect();
     let hists = wf_ids.iter().map(|wf_id| {
         let hist = canned_histories::single_timer("1");
@@ -1022,7 +1022,7 @@ async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static
 async fn lots_of_workflows() {
     let total_wfs = 500;
     let hists = (0..total_wfs).into_iter().map(|i| {
-        let wf_id = format!("fake-wf-{}", i);
+        let wf_id = format!("fake-wf-{i}");
         let hist = canned_histories::single_timer("1");
         FakeWfResponses {
             wf_id,
@@ -1546,9 +1546,8 @@ async fn failing_wft_doesnt_eat_permit_forever() {
         ))
         .await
         .unwrap();
-    assert_eq!(worker.available_wft_permits().await, 2);
-
     worker.shutdown().await;
+    assert_eq!(worker.available_wft_permits().await, 2);
 }
 
 #[tokio::test]
@@ -1574,6 +1573,8 @@ async fn cache_miss_will_fetch_history() {
     let mut mock = build_mock_pollers(mh);
     mock.worker_cfg(|cfg| {
         cfg.max_cached_workflows = 1;
+        // Also verifies tying the WFT permit to the fetch request doesn't get us stuck
+        cfg.max_outstanding_workflow_tasks = 1;
     });
     let worker = mock_worker(mock);
 
@@ -1655,6 +1656,55 @@ async fn tasks_from_completion_are_delivered() {
     let complete_resp = RespondWorkflowTaskCompletedResponse {
         workflow_task: Some(hist_to_poll_resp(&t, wfid.to_owned(), 2.into()).resp),
         activity_tasks: vec![],
+        reset_history_event_id: 0,
+    };
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(move |_| Ok(complete_resp.clone()));
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(|_| Ok(Default::default()));
+    let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
+    let core = mock_worker(mock);
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        },]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        wf_task.run_id,
+        vec![CompleteWorkflowExecution { result: None }.into()],
+    ))
+    .await
+    .unwrap();
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn pagination_works_with_tasks_from_completion() {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_we_signaled("sig", vec![]);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+    let get_exec_resp: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
+
+    let mut mock = mock_workflow_client();
+    let mut needs_pag_resp = hist_to_poll_resp(&t, wfid.to_owned(), ResponseType::OneTask(2)).resp;
+    needs_pag_resp.next_page_token = vec![1];
+    let complete_resp = RespondWorkflowTaskCompletedResponse {
+        workflow_task: Some(needs_pag_resp),
+        ..Default::default()
     };
     mock.expect_complete_workflow_task()
         .times(1)
@@ -1662,6 +1712,9 @@ async fn tasks_from_completion_are_delivered() {
     mock.expect_complete_workflow_task()
         .times(1)
         .returning(|_| Ok(Default::default()));
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(get_exec_resp.clone()))
+        .times(1);
     let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
     mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
     let core = mock_worker(mock);
@@ -1691,7 +1744,7 @@ async fn poll_faster_than_complete_wont_overflow_cache() {
     // Make workflow tasks for 5 different runs
     let tasks: Vec<_> = (1..=5)
         .map(|i| FakeWfResponses {
-            wf_id: format!("wf-{}", i),
+            wf_id: format!("wf-{i}"),
             hist: canned_histories::single_timer("1"),
             response_batches: vec![ResponseType::ToTaskNum(1)],
         })
@@ -2087,3 +2140,194 @@ async fn ignorable_events_are_ok(#[values(true, false)] attribs_unset: bool) {
         Some(workflow_activation_job::Variant::StartWorkflow(_))
     );
 }
+
+#[tokio::test]
+async fn fetching_to_continue_replay_works() {
+    let mut mock_client = mock_workflow_client();
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_full_wf_task(); // ends 7
+    let mut need_fetch_resp =
+        hist_to_poll_resp(&t, "wfid".to_owned(), ResponseType::AllHistory).resp;
+    need_fetch_resp.next_page_token = vec![1];
+
+    t.add_full_wf_task();
+    t.add_we_signaled("hi", vec![]); // Need to make there be two complete WFTs
+    t.add_full_wf_task(); // end 14
+    let mut fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_full_history_info().unwrap().into();
+    // Should only contain events after 7
+    if let Some(ref mut h) = fetch_resp.history {
+        h.events.retain(|e| e.event_id >= 8);
+    }
+    // And indicate that even *more* needs to be fetched after this, so we see a request for the
+    // next page happen.
+    fetch_resp.next_page_token = vec![2];
+
+    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_full_wf_task();
+    let mut final_fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_full_history_info().unwrap().into();
+    // Should have only the final event
+    if let Some(ref mut h) = final_fetch_resp.history {
+        h.events.retain(|e| e.event_id >= 15);
+    }
+
+    let tasks = vec![
+        ResponseType::ToTaskNum(1),
+        ResponseType::Raw(need_fetch_resp),
+    ];
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(fetch_resp.clone()))
+        .times(1);
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(final_fetch_resp.clone()))
+        .times(1);
+    let mut mock = single_hist_mock_sg("wfid", t, tasks, mock_client, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    let core = mock_worker(mock);
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant,
+        Some(workflow_activation_job::Variant::StartWorkflow(_))
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        }]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        act.run_id,
+        start_timer_cmd(1, Duration::from_secs(1)),
+    ))
+    .await
+    .unwrap();
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::FireTimer(_)),
+        }]
+    );
+}
+
+#[tokio::test]
+async fn fetching_error_evicts_wf() {
+    let mut mock_client = mock_workflow_client();
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+    t.add_workflow_task_completed();
+    let mut need_fetch_resp =
+        hist_to_poll_resp(&t, "wfid".to_owned(), ResponseType::AllHistory).resp;
+    need_fetch_resp.next_page_token = vec![1];
+    let tasks = vec![
+        ResponseType::ToTaskNum(1),
+        ResponseType::Raw(need_fetch_resp),
+    ];
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Err(tonic::Status::not_found("Ahh broken")))
+        .times(1);
+    let mut mock = single_hist_mock_sg("wfid", t, tasks, mock_client, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    let core = mock_worker(mock);
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant,
+        Some(workflow_activation_job::Variant::StartWorkflow(_))
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+    let evict_act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        evict_act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(r)),
+        }] => r.message.contains("Fetching history failed")
+    );
+}
+
+/// This test verifies that if we fail to fetch a page during a completion, that we don't get stuck
+/// in the complete waiting for the completion to finish.
+#[tokio::test]
+async fn ensure_fetching_fail_during_complete_sends_task_failure() {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task(); // started 3
+    t.add_we_signaled("sig1", vec![]);
+    t.add_full_wf_task(); // started 7
+    t.add_we_signaled("sig2", vec![]);
+    t.add_full_wf_task(); // started 11
+    t.add_workflow_execution_completed();
+
+    let mut first_poll = hist_to_poll_resp(&t, wfid, ResponseType::ToTaskNum(1)).resp;
+    first_poll.next_page_token = vec![1];
+    first_poll.previous_started_event_id = 3;
+
+    let mut next_page: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
+    next_page.next_page_token = vec![2];
+
+    let mut mock = mock_workflow_client();
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| {
+            error!("Called fetch!");
+            Ok(next_page.clone())
+        })
+        .times(1);
+    let mut really_empty_fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_history_info(1).unwrap().into();
+    really_empty_fetch_resp.history = Some(Default::default());
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| {
+            error!("Called fetch second time!");
+            Ok(really_empty_fetch_resp.clone())
+        })
+        .times(1);
+    mock.expect_fail_workflow_task()
+        .returning(|_, _, _| Ok(Default::default()))
+        .times(1);
+
+    let mut mock = single_hist_mock_sg(wfid, t, [ResponseType::Raw(first_poll)], mock, true);
+    mock.make_wft_stream_interminable();
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
+    let core = mock_worker(mock);
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        },]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+
+    // Expect to see eviction b/c of history fetching error here.
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
+        },]
+    );
+
+    core.shutdown().await;
+}