temporalio 0.0.1 → 0.1.0

data/bridge/sdk-core/core/src/core_tests/workflow_tasks.rs

@@ -7,7 +7,6 @@ use crate::{
         mock_worker, poll_and_reply, poll_and_reply_clears_outstanding_evicts, single_hist_mock_sg,
         test_worker_cfg, FakeWfResponses, MockPollCfg, MocksHolder, ResponseType,
         WorkflowCachingPolicy::{self, AfterEveryReply, NonSticky},
-        TEST_Q,
     },
     worker::client::mocks::{mock_manual_workflow_client, mock_workflow_client},
     Worker,
@@ -17,7 +16,7 @@ use rstest::{fixture, rstest};
 use std::{
     collections::{HashMap, VecDeque},
     sync::{
-        atomic::{AtomicU64, Ordering},
+        atomic::{AtomicBool, AtomicU64, Ordering},
         Arc,
     },
     time::Duration,
@@ -38,20 +37,25 @@ use temporal_sdk_core_protos::{
         },
         workflow_completion::WorkflowActivationCompletion,
     },
-    default_wes_attribs,
+    default_act_sched, default_wes_attribs,
     temporal::api::{
         command::v1::command::Attributes,
         common::v1::{Payload, RetryPolicy},
         enums::v1::{EventType, WorkflowTaskFailedCause},
         failure::v1::Failure,
-        history::v1::{history_event, TimerFiredEventAttributes},
+        history::v1::{
+            history_event, TimerFiredEventAttributes,
+            WorkflowPropertiesModifiedExternallyEventAttributes,
+        },
         workflowservice::v1::{
             GetWorkflowExecutionHistoryResponse, RespondWorkflowTaskCompletedResponse,
         },
     },
-    DEFAULT_WORKFLOW_TYPE,
+    DEFAULT_ACTIVITY_TYPE, DEFAULT_WORKFLOW_TYPE,
+};
+use temporal_sdk_core_test_utils::{
+    fanout_tasks, schedule_activity_cmd, start_timer_cmd, WorkerTestHelpers,
 };
-use temporal_sdk_core_test_utils::{fanout_tasks, start_timer_cmd};
 use tokio::{
     join,
     sync::{Barrier, Semaphore},
@@ -121,7 +125,7 @@ async fn single_activity_completion(worker: Worker) {
                 &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
                 vec![ScheduleActivity {
                     activity_id: "fake_activity".to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -246,7 +250,7 @@ async fn scheduled_activity_cancellation_try_cancel(hist_batches: &'static [usiz
                     seq: activity_seq,
                     activity_id: activity_id.to_string(),
                     cancellation_type: ActivityCancellationType::TryCancel as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -282,7 +286,7 @@ async fn scheduled_activity_timeout(hist_batches: &'static [usize]) {
                 vec![ScheduleActivity {
                     seq: activity_seq,
                     activity_id: activity_id.to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                     .into()],
             ),
@@ -335,7 +339,7 @@ async fn started_activity_timeout(hist_batches: &'static [usize]) {
                 vec![ScheduleActivity {
                     seq: activity_seq,
                     activity_id: activity_seq.to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                     .into()],
             ),
@@ -390,7 +394,7 @@ async fn cancelled_activity_timeout(hist_batches: &'static [usize]) {
                 vec![ScheduleActivity {
                     seq: activity_seq,
                     activity_id: activity_id.to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -458,10 +462,10 @@ async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
     let act_scheduled_event_id = t.add_activity_task_scheduled(activity_id);
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_timer_fired(timer_started_event_id, "1".to_string());
     t.add_full_wf_task();
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     let act_started_event_id = t.add_activity_task_started(act_scheduled_event_id);
     t.add_activity_task_completed(
         act_scheduled_event_id,
@@ -477,7 +481,7 @@ async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [
 
     worker.register_wf(wf_type.to_owned(), |ctx: WfContext| async move {
         let act_fut = ctx.activity(ActivityOptions {
-            activity_type: "echo_activity".to_string(),
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
             start_to_close_timeout: Some(Duration::from_secs(5)),
             cancellation_type: ActivityCancellationType::Abandon,
             ..Default::default()
@@ -543,7 +547,7 @@ async fn verify_activity_cancellation(
                     seq: activity_seq,
                     activity_id: activity_seq.to_string(),
                     cancellation_type: cancel_type as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -611,7 +615,7 @@ async fn verify_activity_cancellation_wait_for_cancellation(activity_id: u32, wo
                     seq: activity_id,
                     activity_id: activity_id.to_string(),
                     cancellation_type: ActivityCancellationType::WaitCancellationCompleted as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -652,7 +656,7 @@ async fn workflow_update_random_seed_on_workflow_reset() {
         timer_1_id.to_string().as_str(),
         new_run_id,
     );
-    let core = build_fake_worker(wfid, t, &[2]);
+    let core = build_fake_worker(wfid, t, [2]);
 
     poll_and_reply(
         &core,
@@ -706,7 +710,7 @@ async fn cancel_timer_before_sent_wf_bridge() {
     t.add_full_wf_task();
     t.add_workflow_execution_completed();
 
-    let core = build_fake_worker(wfid, t, &[1]);
+    let core = build_fake_worker(wfid, t, [1]);
 
     poll_and_reply(
         &core,
@@ -893,7 +897,7 @@ async fn max_wft_respected() {
     let total_wfs = 100;
     let wf_ids: Vec<_> = (0..total_wfs)
         .into_iter()
-        .map(|i| format!("fake-wf-{}", i))
+        .map(|i| format!("fake-wf-{i}"))
         .collect();
     let hists = wf_ids.iter().map(|wf_id| {
         let hist = canned_histories::single_timer("1");
@@ -948,7 +952,7 @@ async fn activity_not_canceled_on_replay_repro(hist_batches: &'static [usize]) {
                         seq: activity_id,
                         activity_id: activity_id.to_string(),
                         cancellation_type: ActivityCancellationType::TryCancel as i32,
-                        ..Default::default()
+                        ..default_act_sched()
                     }
                     .into(),
                     start_timer_cmd(1, Duration::from_secs(1)),
@@ -990,9 +994,9 @@ async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static
                 &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
                 vec![ScheduleActivity {
                     seq: activity_id,
-                    activity_id: activity_id.to_string(),
+                    activity_id: "act-1".to_string(),
                     cancellation_type: ActivityCancellationType::TryCancel as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -1023,7 +1027,7 @@ async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static
 async fn lots_of_workflows() {
     let total_wfs = 500;
     let hists = (0..total_wfs).into_iter().map(|i| {
-        let wf_id = format!("fake-wf-{}", i);
+        let wf_id = format!("fake-wf-{i}");
         let hist = canned_histories::single_timer("1");
         FakeWfResponses {
             wf_id,
@@ -1037,7 +1041,6 @@ async fn lots_of_workflows() {
     let completed_count = Arc::new(Semaphore::new(0));
     let killer = async {
         let _ = completed_count.acquire_many(total_wfs).await.unwrap();
-        dbg!("Shutdown initted");
         worker.initiate_shutdown();
     };
     let poller = fanout_tasks(5, |_| {
@@ -1094,7 +1097,7 @@ async fn wft_timeout_repro(hist_batches: &'static [usize]) {
                     seq: activity_id,
                     activity_id: activity_id.to_string(),
                     cancellation_type: ActivityCancellationType::TryCancel as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -1122,7 +1125,7 @@ async fn complete_after_eviction() {
     let t = canned_histories::single_timer("1");
     let mut mock = mock_workflow_client();
     mock.expect_complete_workflow_task().times(0);
-    let mock = single_hist_mock_sg(wfid, t, &[2], mock, true);
+    let mock = single_hist_mock_sg(wfid, t, [2], mock, true);
     let core = mock_worker(mock);
 
     let activation = core.poll_workflow_activation().await.unwrap();
@@ -1164,7 +1167,7 @@ async fn sends_appropriate_sticky_task_queue_responses() {
         .times(1)
         .returning(|_| Ok(Default::default()));
     mock.expect_complete_workflow_task().times(0);
-    let mut mock = single_hist_mock_sg(wfid, t, &[1], mock, false);
+    let mut mock = single_hist_mock_sg(wfid, t, [1], mock, false);
     mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
     let core = mock_worker(mock);
 
@@ -1182,7 +1185,7 @@ async fn sends_appropriate_sticky_task_queue_responses() {
 async fn new_server_work_while_eviction_outstanding_doesnt_overwrite_activation() {
     let wfid = "fake_wf_id";
     let t = canned_histories::single_timer("1");
-    let mock = single_hist_mock_sg(wfid, t, &[1, 2], mock_workflow_client(), false);
+    let mock = single_hist_mock_sg(wfid, t, [1, 2], mock_workflow_client(), false);
     let taskmap = mock.outstanding_task_map.clone().unwrap();
     let core = mock_worker(mock);
 
@@ -1224,17 +1227,16 @@ async fn buffered_work_drained_on_shutdown() {
     t.add_workflow_task_scheduled_and_started();
     // Need to build the first response before adding the timeout events b/c otherwise the history
     // builder will include them in the first task
-    let resp_1 = hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()).resp;
+    let resp_1 = hist_to_poll_resp(&t, wfid.to_owned(), 1.into()).resp;
     t.add_workflow_task_timed_out();
     t.add_full_wf_task();
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
-    t.add(
-        EventType::TimerFired,
-        history_event::Attributes::TimerFiredEventAttributes(TimerFiredEventAttributes {
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
+    t.add(history_event::Attributes::TimerFiredEventAttributes(
+        TimerFiredEventAttributes {
             started_event_id: timer_started_event_id,
             timer_id: "1".to_string(),
-        }),
-    );
+        },
+    ));
     t.add_full_wf_task();
     t.add_workflow_execution_completed();
 
@@ -1242,10 +1244,7 @@ async fn buffered_work_drained_on_shutdown() {
     // Extend the task list with the now timeout-included version of the task. We add a bunch of
     // them because the poll loop will spin while new tasks are available and it is buffering them
     tasks.extend(
-        std::iter::repeat_with(|| {
-            hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string()).resp
-        })
-        .take(50),
+        std::iter::repeat_with(|| hist_to_poll_resp(&t, wfid.to_owned(), 2.into()).resp).take(50),
     );
     let mut mock = mock_workflow_client();
     mock.expect_complete_workflow_task()
@@ -1269,6 +1268,8 @@ async fn buffered_work_drained_on_shutdown() {
         .unwrap();
     };
     let complete_first = async move {
+        // If the first complete is sent too fast, we may not have had a chance to buffer work.
+        tokio::time::sleep(Duration::from_millis(50)).await;
         core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
             act1.run_id,
             start_timer_cmd(1, Duration::from_secs(1)),
@@ -1277,8 +1278,6 @@ async fn buffered_work_drained_on_shutdown() {
         .unwrap();
     };
     join!(poll_fut, complete_first, async {
-        // If the shutdown is sent too too fast, we might not have got a chance to even buffer work
-        tokio::time::sleep(Duration::from_millis(5)).await;
         core.shutdown().await;
     });
 }
@@ -1308,7 +1307,7 @@ async fn fail_wft_then_recover() {
         act.run_id.clone(),
         vec![ScheduleActivity {
             activity_id: "fake_activity".to_string(),
-            ..Default::default()
+            ..default_act_sched()
         }
         .into()],
     ))
@@ -1360,12 +1359,17 @@ async fn poll_response_triggers_wf_error() {
     t.add_full_wf_task();
     t.add_workflow_execution_completed();
 
-    let mh = MockPollCfg::from_resp_batches(
+    let mut mh = MockPollCfg::from_resp_batches(
         "fake_wf_id",
         t,
         [ResponseType::AllHistory],
         mock_workflow_client(),
     );
+    // Fail wft will be called when auto-failing.
+    mh.num_expected_fails = 1;
+    mh.expect_fail_wft_matcher = Box::new(move |_, cause, _| {
+        matches!(cause, WorkflowTaskFailedCause::NonDeterministicError)
+    });
     let mock = build_mock_pollers(mh);
     let core = mock_worker(mock);
     // Poll for first WFT, which is immediately an eviction
@@ -1448,7 +1452,7 @@ async fn tries_cancel_of_completed_activity() {
     t.add_workflow_task_scheduled_and_started();
 
     let mock = mock_workflow_client();
-    let mut mock = single_hist_mock_sg("fake_wf_id", t, &[1, 2], mock, true);
+    let mut mock = single_hist_mock_sg("fake_wf_id", t, [1, 2], mock, true);
     mock.worker_cfg(|cfg| cfg.max_cached_workflows = 1);
     let core = mock_worker(mock);
 
@@ -1458,7 +1462,7 @@ async fn tries_cancel_of_completed_activity() {
         ScheduleActivity {
             seq: 1,
             activity_id: "1".to_string(),
-            ..Default::default()
+            ..default_act_sched()
         }
         .into(),
     ))
@@ -1551,9 +1555,8 @@ async fn failing_wft_doesnt_eat_permit_forever() {
         ))
         .await
         .unwrap();
-    assert_eq!(worker.available_wft_permits().await, 2);
-
     worker.shutdown().await;
+    assert_eq!(worker.available_wft_permits().await, 2);
 }
 
 #[tokio::test]
@@ -1579,6 +1582,8 @@ async fn cache_miss_will_fetch_history() {
     let mut mock = build_mock_pollers(mh);
     mock.worker_cfg(|cfg| {
         cfg.max_cached_workflows = 1;
+        // Also verifies tying the WFT permit to the fetch request doesn't get us stuck
+        cfg.max_outstanding_workflow_tasks = 1;
     });
     let worker = mock_worker(mock);
 
@@ -1658,10 +1663,9 @@ async fn tasks_from_completion_are_delivered() {
 
     let mut mock = mock_workflow_client();
     let complete_resp = RespondWorkflowTaskCompletedResponse {
-        workflow_task: Some(
-            hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string()).resp,
-        ),
+        workflow_task: Some(hist_to_poll_resp(&t, wfid.to_owned(), 2.into()).resp),
         activity_tasks: vec![],
+        reset_history_event_id: 0,
     };
     mock.expect_complete_workflow_task()
         .times(1)
@@ -1693,12 +1697,63 @@ async fn tasks_from_completion_are_delivered() {
     core.shutdown().await;
 }
 
+#[tokio::test]
+async fn pagination_works_with_tasks_from_completion() {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_we_signaled("sig", vec![]);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+    let get_exec_resp: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
+
+    let mut mock = mock_workflow_client();
+    let mut needs_pag_resp = hist_to_poll_resp(&t, wfid.to_owned(), ResponseType::OneTask(2)).resp;
+    needs_pag_resp.next_page_token = vec![1];
+    let complete_resp = RespondWorkflowTaskCompletedResponse {
+        workflow_task: Some(needs_pag_resp),
+        ..Default::default()
+    };
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(move |_| Ok(complete_resp.clone()));
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(|_| Ok(Default::default()));
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(get_exec_resp.clone()))
+        .times(1);
+    let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
+    let core = mock_worker(mock);
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        },]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        wf_task.run_id,
+        vec![CompleteWorkflowExecution { result: None }.into()],
+    ))
+    .await
+    .unwrap();
+    core.shutdown().await;
+}
+
 #[tokio::test]
 async fn poll_faster_than_complete_wont_overflow_cache() {
     // Make workflow tasks for 5 different runs
     let tasks: Vec<_> = (1..=5)
         .map(|i| FakeWfResponses {
-            wf_id: format!("wf-{}", i),
+            wf_id: format!("wf-{i}"),
             hist: canned_histories::single_timer("1"),
             response_batches: vec![ResponseType::ToTaskNum(1)],
         })
@@ -1829,7 +1884,7 @@ async fn eviction_waits_until_replay_finished() {
     let wfid = "fake_wf_id";
     let t = canned_histories::long_sequential_timers(3);
     let mock = mock_workflow_client();
-    let mock = single_hist_mock_sg(wfid, t, &[3], mock, true);
+    let mock = single_hist_mock_sg(wfid, t, [3], mock, true);
     let core = mock_worker(mock);
 
     let activation = core.poll_workflow_activation().await.unwrap();
@@ -1890,7 +1945,7 @@ async fn autocompletes_wft_no_work() {
     t.add_activity_task_completed(scheduled_event_id, started_event_id, Default::default());
     t.add_full_wf_task();
     let mock = mock_workflow_client();
-    let mut mock = single_hist_mock_sg(wfid, t, &[1, 2, 3, 4], mock, true);
+    let mut mock = single_hist_mock_sg(wfid, t, [1, 2, 3, 4], mock, true);
     mock.worker_cfg(|w| w.max_cached_workflows = 1);
     let core = mock_worker(mock);
 
@@ -1907,7 +1962,7 @@ async fn autocompletes_wft_no_work() {
             seq: 1,
             activity_id: activity_id.to_string(),
             cancellation_type: ActivityCancellationType::Abandon as i32,
-            ..Default::default()
+            ..default_act_sched()
         }
         .into(),
     ))
@@ -1951,8 +2006,7 @@ async fn no_race_acquiring_permits() {
         .expect_poll_workflow_task()
         .returning(move |_, _| {
             let t = canned_histories::single_timer("1");
-            let poll_resp =
-                hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string()).resp;
+            let poll_resp = hist_to_poll_resp(&t, wfid.to_owned(), 2.into()).resp;
             async move {
                 task_barr.wait().await;
                 Ok(poll_resp.clone())
@@ -2021,23 +2075,14 @@ async fn continue_as_new_preserves_some_values() {
     let mut mock_client = mock_workflow_client();
     let hist = {
         let mut t = TestHistoryBuilder::default();
-        t.add(
-            EventType::WorkflowExecutionStarted,
-            wes_attrs.clone().into(),
-        );
+        t.add(wes_attrs.clone());
         t.add_full_wf_task();
         t
     };
     mock_client
         .expect_poll_workflow_task()
         .returning(move |_, _| {
-            Ok(hist_to_poll_resp(
-                &hist,
-                wfid.to_owned(),
-                ResponseType::AllHistory,
-                TEST_Q.to_string(),
-            )
-            .resp)
+            Ok(hist_to_poll_resp(&hist, wfid.to_owned(), ResponseType::AllHistory).resp)
         });
     mock_client
         .expect_complete_workflow_task()
@@ -2068,3 +2113,321 @@ async fn continue_as_new_preserves_some_values() {
         .await
         .unwrap();
 }
+
+#[rstest]
+#[tokio::test]
+async fn ignorable_events_are_ok(#[values(true, false)] attribs_unset: bool) {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    let id = t.add(WorkflowPropertiesModifiedExternallyEventAttributes::default());
+    t.modify_event(id, |e| {
+        e.worker_may_ignore = true;
+        // Ignorable events are ignored if we can't interpret the proto of either the event attribs
+        // or proto - otherwise (this is the _may_ part of may ignore), we'll still try to process
+        // it. That processing may ultimately still choose to do nothing, if we want to _explicitly_
+        // ignore it.
+        if attribs_unset {
+            e.attributes = None;
+        } else {
+            e.event_type = EventType::Unspecified as i32;
+        }
+    });
+    t.add_workflow_task_scheduled_and_started();
+
+    let mock = mock_workflow_client();
+    let mock = single_hist_mock_sg("wheee", t, [ResponseType::AllHistory], mock, true);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant,
+        Some(workflow_activation_job::Variant::StartWorkflow(_))
+    );
+}
+
+#[tokio::test]
+async fn fetching_to_continue_replay_works() {
+    let mut mock_client = mock_workflow_client();
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_full_wf_task(); // ends 7
+    let mut need_fetch_resp =
+        hist_to_poll_resp(&t, "wfid".to_owned(), ResponseType::AllHistory).resp;
+    need_fetch_resp.next_page_token = vec![1];
+
+    t.add_full_wf_task();
+    t.add_we_signaled("hi", vec![]); // Need to make there be two complete WFTs
+    t.add_full_wf_task(); // end 14
+    let mut fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_full_history_info().unwrap().into();
+    // Should only contain events after 7
+    if let Some(ref mut h) = fetch_resp.history {
+        h.events.retain(|e| e.event_id >= 8);
+    }
+    // And indicate that even *more* needs to be fetched after this, so we see a request for the
+    // next page happen.
+    fetch_resp.next_page_token = vec![2];
+
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_full_wf_task();
+    let mut final_fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_full_history_info().unwrap().into();
+    // Should have only the final event
+    if let Some(ref mut h) = final_fetch_resp.history {
+        h.events.retain(|e| e.event_id >= 15);
+    }
+
+    let tasks = vec![
+        ResponseType::ToTaskNum(1),
+        ResponseType::Raw(need_fetch_resp),
+    ];
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(fetch_resp.clone()))
+        .times(1);
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(final_fetch_resp.clone()))
+        .times(1);
+    let mut mock = single_hist_mock_sg("wfid", t, tasks, mock_client, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    let core = mock_worker(mock);
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant,
+        Some(workflow_activation_job::Variant::StartWorkflow(_))
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        }]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        act.run_id,
+        start_timer_cmd(1, Duration::from_secs(1)),
+    ))
+    .await
+    .unwrap();
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::FireTimer(_)),
+        }]
+    );
+}
+
+#[tokio::test]
+async fn fetching_error_evicts_wf() {
+    let mut mock_client = mock_workflow_client();
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+    t.add_workflow_task_completed();
+    let mut need_fetch_resp =
+        hist_to_poll_resp(&t, "wfid".to_owned(), ResponseType::AllHistory).resp;
+    need_fetch_resp.next_page_token = vec![1];
+    let tasks = vec![
+        ResponseType::ToTaskNum(1),
+        ResponseType::Raw(need_fetch_resp),
+    ];
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Err(tonic::Status::not_found("Ahh broken")))
+        .times(1);
+    let mut mock = single_hist_mock_sg("wfid", t, tasks, mock_client, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    let core = mock_worker(mock);
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant,
+        Some(workflow_activation_job::Variant::StartWorkflow(_))
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+    let evict_act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        evict_act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(r)),
+        }] => r.message.contains("Fetching history failed")
+    );
+}
+
+/// This test verifies that if we fail to fetch a page during a completion, that we don't get stuck
+/// in the complete waiting for the completion to finish.
+#[tokio::test]
+async fn ensure_fetching_fail_during_complete_sends_task_failure() {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task(); // started 3
+    t.add_we_signaled("sig1", vec![]);
+    t.add_full_wf_task(); // started 7
+    t.add_we_signaled("sig2", vec![]);
+    t.add_full_wf_task(); // started 11
+    t.add_workflow_execution_completed();
+
+    let mut first_poll = hist_to_poll_resp(&t, wfid, ResponseType::ToTaskNum(1)).resp;
+    first_poll.next_page_token = vec![1];
+    first_poll.previous_started_event_id = 3;
+
+    let mut next_page: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
+    next_page.next_page_token = vec![2];
+
+    let mut mock = mock_workflow_client();
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| {
+            error!("Called fetch!");
+            Ok(next_page.clone())
+        })
+        .times(1);
+    let mut really_empty_fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_history_info(1).unwrap().into();
+    really_empty_fetch_resp.history = Some(Default::default());
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| {
+            error!("Called fetch second time!");
+            Ok(really_empty_fetch_resp.clone())
+        })
+        .times(1);
+    mock.expect_fail_workflow_task()
+        .returning(|_, _, _| Ok(Default::default()))
+        .times(1);
+
+    let mut mock = single_hist_mock_sg(wfid, t, [ResponseType::Raw(first_poll)], mock, true);
+    mock.make_wft_stream_interminable();
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
+    let core = mock_worker(mock);
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        },]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+
+    // Expect to see eviction b/c of history fetching error here.
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
+        },]
+    );
+
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn lang_internal_flags() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.set_flags_first_wft(&[], &[1]);
+    t.add_we_signaled("sig1", vec![]);
+    t.add_full_wf_task();
+    t.set_flags_last_wft(&[], &[2]);
+    t.add_we_signaled("sig2", vec![]);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::ToTaskNum(2), ResponseType::AllHistory],
+        mock_workflow_client(),
+    );
+    mh.completion_asserts = Some(Box::new(|c| {
+        assert_matches!(c.sdk_metadata.lang_used_flags.as_slice(), &[2]);
+    }));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(act.available_internal_flags.as_slice(), [1]);
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    let mut completion = WorkflowActivationCompletion::empty(act.run_id);
+    completion.add_internal_flags(2);
+    core.complete_workflow_activation(completion).await.unwrap();
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(act.available_internal_flags.as_slice(), [1, 2]);
+    core.complete_execution(&act.run_id).await;
+    core.shutdown().await;
+}
+
+// Verify we send flags to server when they're used
+#[tokio::test]
+async fn core_internal_flags() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    let act_scheduled_event_id = t.add_activity_task_scheduled("act-id");
+    let act_started_event_id = t.add_activity_task_started(act_scheduled_event_id);
+    t.add_activity_task_completed(
+        act_scheduled_event_id,
+        act_started_event_id,
+        Default::default(),
+    );
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::ToTaskNum(1), ResponseType::ToTaskNum(2)],
+        mock_workflow_client(),
+    );
+    let first_poll = AtomicBool::new(true);
+    mh.completion_asserts = Some(Box::new(move |c| {
+        if !first_poll.load(Ordering::Acquire) {
+            assert_matches!(c.sdk_metadata.core_used_flags.as_slice(), &[1]);
+        }
+        first_poll.store(false, Ordering::Release);
+    }));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        act.run_id,
+        schedule_activity_cmd(
+            1,
+            "whatever",
+            "act-id",
+            ActivityCancellationType::TryCancel,
+            Duration::from_secs(60),
+            Duration::from_secs(60),
+        ),
+    ))
+    .await
+    .unwrap();
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    core.complete_execution(&act.run_id).await;
+    core.shutdown().await;
+}