@temporalio/core-bridge 1.5.2 → 1.7.0

package/sdk-core/core/src/core_tests/workflow_tasks.rs

@@ -16,11 +16,12 @@ use rstest::{fixture, rstest};
 use std::{
     collections::{HashMap, VecDeque},
     sync::{
-        atomic::{AtomicU64, Ordering},
+        atomic::{AtomicBool, AtomicU64, Ordering},
         Arc,
     },
     time::Duration,
 };
+use temporal_client::WorkflowOptions;
 use temporal_sdk::{ActivityOptions, CancellableFuture, WfContext};
 use temporal_sdk_core_api::{errors::PollWfError, Worker as WorkerTrait};
 use temporal_sdk_core_protos::{
@@ -33,24 +34,29 @@ use temporal_sdk_core_protos::{
         workflow_commands::{
             ActivityCancellationType, CancelTimer, CompleteWorkflowExecution,
             ContinueAsNewWorkflowExecution, FailWorkflowExecution, RequestCancelActivity,
-            ScheduleActivity,
+            ScheduleActivity, SetPatchMarker,
         },
         workflow_completion::WorkflowActivationCompletion,
     },
-    default_wes_attribs,
+    default_act_sched, default_wes_attribs,
     temporal::api::{
         command::v1::command::Attributes,
         common::v1::{Payload, RetryPolicy},
         enums::v1::{EventType, WorkflowTaskFailedCause},
         failure::v1::Failure,
-        history::v1::{history_event, TimerFiredEventAttributes},
+        history::v1::{
+            history_event, TimerFiredEventAttributes,
+            WorkflowPropertiesModifiedExternallyEventAttributes,
+        },
         workflowservice::v1::{
             GetWorkflowExecutionHistoryResponse, RespondWorkflowTaskCompletedResponse,
         },
     },
-    DEFAULT_WORKFLOW_TYPE,
+    DEFAULT_ACTIVITY_TYPE, DEFAULT_WORKFLOW_TYPE,
+};
+use temporal_sdk_core_test_utils::{
+    fanout_tasks, schedule_activity_cmd, start_timer_cmd, WorkerTestHelpers,
 };
-use temporal_sdk_core_test_utils::{fanout_tasks, start_timer_cmd};
 use tokio::{
     join,
     sync::{Barrier, Semaphore},
@@ -120,7 +126,7 @@ async fn single_activity_completion(worker: Worker) {
                 &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
                 vec![ScheduleActivity {
                     activity_id: "fake_activity".to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -245,7 +251,7 @@ async fn scheduled_activity_cancellation_try_cancel(hist_batches: &'static [usiz
                     seq: activity_seq,
                     activity_id: activity_id.to_string(),
                     cancellation_type: ActivityCancellationType::TryCancel as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -281,7 +287,7 @@ async fn scheduled_activity_timeout(hist_batches: &'static [usize]) {
                 vec![ScheduleActivity {
                     seq: activity_seq,
                     activity_id: activity_id.to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                     .into()],
             ),
@@ -334,7 +340,7 @@ async fn started_activity_timeout(hist_batches: &'static [usize]) {
                 vec![ScheduleActivity {
                     seq: activity_seq,
                     activity_id: activity_seq.to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                     .into()],
             ),
@@ -389,7 +395,7 @@ async fn cancelled_activity_timeout(hist_batches: &'static [usize]) {
                 vec![ScheduleActivity {
                     seq: activity_seq,
                     activity_id: activity_id.to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -457,10 +463,10 @@ async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
     let act_scheduled_event_id = t.add_activity_task_scheduled(activity_id);
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_timer_fired(timer_started_event_id, "1".to_string());
     t.add_full_wf_task();
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     let act_started_event_id = t.add_activity_task_started(act_scheduled_event_id);
     t.add_activity_task_completed(
         act_scheduled_event_id,
@@ -476,7 +482,7 @@ async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [
 
     worker.register_wf(wf_type.to_owned(), |ctx: WfContext| async move {
         let act_fut = ctx.activity(ActivityOptions {
-            activity_type: "echo_activity".to_string(),
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
             start_to_close_timeout: Some(Duration::from_secs(5)),
             cancellation_type: ActivityCancellationType::Abandon,
             ..Default::default()
@@ -542,7 +548,7 @@ async fn verify_activity_cancellation(
                     seq: activity_seq,
                     activity_id: activity_seq.to_string(),
                     cancellation_type: cancel_type as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -610,7 +616,7 @@ async fn verify_activity_cancellation_wait_for_cancellation(activity_id: u32, wo
                     seq: activity_id,
                     activity_id: activity_id.to_string(),
                     cancellation_type: ActivityCancellationType::WaitCancellationCompleted as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -892,7 +898,7 @@ async fn max_wft_respected() {
     let total_wfs = 100;
     let wf_ids: Vec<_> = (0..total_wfs)
         .into_iter()
-        .map(|i| format!("fake-wf-{}", i))
+        .map(|i| format!("fake-wf-{i}"))
         .collect();
     let hists = wf_ids.iter().map(|wf_id| {
         let hist = canned_histories::single_timer("1");
@@ -947,7 +953,7 @@ async fn activity_not_canceled_on_replay_repro(hist_batches: &'static [usize]) {
                         seq: activity_id,
                         activity_id: activity_id.to_string(),
                         cancellation_type: ActivityCancellationType::TryCancel as i32,
-                        ..Default::default()
+                        ..default_act_sched()
                     }
                     .into(),
                     start_timer_cmd(1, Duration::from_secs(1)),
@@ -989,9 +995,9 @@ async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static
                 &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
                 vec![ScheduleActivity {
                     seq: activity_id,
-                    activity_id: activity_id.to_string(),
+                    activity_id: "act-1".to_string(),
                     cancellation_type: ActivityCancellationType::TryCancel as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -1022,7 +1028,7 @@ async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static
 async fn lots_of_workflows() {
     let total_wfs = 500;
     let hists = (0..total_wfs).into_iter().map(|i| {
-        let wf_id = format!("fake-wf-{}", i);
+        let wf_id = format!("fake-wf-{i}");
         let hist = canned_histories::single_timer("1");
         FakeWfResponses {
             wf_id,
@@ -1092,7 +1098,7 @@ async fn wft_timeout_repro(hist_batches: &'static [usize]) {
                     seq: activity_id,
                     activity_id: activity_id.to_string(),
                     cancellation_type: ActivityCancellationType::TryCancel as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -1225,14 +1231,13 @@ async fn buffered_work_drained_on_shutdown() {
     let resp_1 = hist_to_poll_resp(&t, wfid.to_owned(), 1.into()).resp;
     t.add_workflow_task_timed_out();
     t.add_full_wf_task();
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
-    t.add(
-        EventType::TimerFired,
-        history_event::Attributes::TimerFiredEventAttributes(TimerFiredEventAttributes {
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
+    t.add(history_event::Attributes::TimerFiredEventAttributes(
+        TimerFiredEventAttributes {
             started_event_id: timer_started_event_id,
             timer_id: "1".to_string(),
-        }),
-    );
+        },
+    ));
     t.add_full_wf_task();
     t.add_workflow_execution_completed();
 
@@ -1264,6 +1269,8 @@ async fn buffered_work_drained_on_shutdown() {
         .unwrap();
     };
     let complete_first = async move {
+        // If the first complete is sent too fast, we may not have had a chance to buffer work.
+        tokio::time::sleep(Duration::from_millis(50)).await;
         core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
             act1.run_id,
             start_timer_cmd(1, Duration::from_secs(1)),
@@ -1272,8 +1279,6 @@ async fn buffered_work_drained_on_shutdown() {
         .unwrap();
     };
     join!(poll_fut, complete_first, async {
-        // If the shutdown is sent too too fast, we might not have got a chance to even buffer work
-        tokio::time::sleep(Duration::from_millis(5)).await;
         core.shutdown().await;
     });
 }
@@ -1303,7 +1308,7 @@ async fn fail_wft_then_recover() {
         act.run_id.clone(),
         vec![ScheduleActivity {
             activity_id: "fake_activity".to_string(),
-            ..Default::default()
+            ..default_act_sched()
         }
         .into()],
     ))
@@ -1355,12 +1360,17 @@ async fn poll_response_triggers_wf_error() {
     t.add_full_wf_task();
     t.add_workflow_execution_completed();
 
-    let mh = MockPollCfg::from_resp_batches(
+    let mut mh = MockPollCfg::from_resp_batches(
         "fake_wf_id",
         t,
         [ResponseType::AllHistory],
         mock_workflow_client(),
     );
+    // Fail wft will be called when auto-failing.
+    mh.num_expected_fails = 1;
+    mh.expect_fail_wft_matcher = Box::new(move |_, cause, _| {
+        matches!(cause, WorkflowTaskFailedCause::NonDeterministicError)
+    });
     let mock = build_mock_pollers(mh);
     let core = mock_worker(mock);
     // Poll for first WFT, which is immediately an eviction
@@ -1453,7 +1463,7 @@ async fn tries_cancel_of_completed_activity() {
         ScheduleActivity {
             seq: 1,
             activity_id: "1".to_string(),
-            ..Default::default()
+            ..default_act_sched()
         }
         .into(),
     ))
@@ -1546,9 +1556,8 @@ async fn failing_wft_doesnt_eat_permit_forever() {
         ))
         .await
         .unwrap();
-    assert_eq!(worker.available_wft_permits().await, 2);
-
     worker.shutdown().await;
+    assert_eq!(worker.available_wft_permits().await, 2);
 }
 
 #[tokio::test]
@@ -1574,6 +1583,8 @@ async fn cache_miss_will_fetch_history() {
     let mut mock = build_mock_pollers(mh);
     mock.worker_cfg(|cfg| {
         cfg.max_cached_workflows = 1;
+        // Also verifies tying the WFT permit to the fetch request doesn't get us stuck
+        cfg.max_outstanding_workflow_tasks = 1;
     });
     let worker = mock_worker(mock);
 
@@ -1655,6 +1666,7 @@ async fn tasks_from_completion_are_delivered() {
     let complete_resp = RespondWorkflowTaskCompletedResponse {
         workflow_task: Some(hist_to_poll_resp(&t, wfid.to_owned(), 2.into()).resp),
         activity_tasks: vec![],
+        reset_history_event_id: 0,
     };
     mock.expect_complete_workflow_task()
         .times(1)
@@ -1686,12 +1698,63 @@ async fn tasks_from_completion_are_delivered() {
     core.shutdown().await;
 }
 
+#[tokio::test]
+async fn pagination_works_with_tasks_from_completion() {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_we_signaled("sig", vec![]);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+    let get_exec_resp: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
+
+    let mut mock = mock_workflow_client();
+    let mut needs_pag_resp = hist_to_poll_resp(&t, wfid.to_owned(), ResponseType::OneTask(2)).resp;
+    needs_pag_resp.next_page_token = vec![1];
+    let complete_resp = RespondWorkflowTaskCompletedResponse {
+        workflow_task: Some(needs_pag_resp),
+        ..Default::default()
+    };
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(move |_| Ok(complete_resp.clone()));
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(|_| Ok(Default::default()));
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(get_exec_resp.clone()))
+        .times(1);
+    let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
+    let core = mock_worker(mock);
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        },]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        wf_task.run_id,
+        vec![CompleteWorkflowExecution { result: None }.into()],
+    ))
+    .await
+    .unwrap();
+    core.shutdown().await;
+}
+
 #[tokio::test]
 async fn poll_faster_than_complete_wont_overflow_cache() {
     // Make workflow tasks for 5 different runs
     let tasks: Vec<_> = (1..=5)
         .map(|i| FakeWfResponses {
-            wf_id: format!("wf-{}", i),
+            wf_id: format!("wf-{i}"),
             hist: canned_histories::single_timer("1"),
             response_batches: vec![ResponseType::ToTaskNum(1)],
         })
@@ -1900,7 +1963,7 @@ async fn autocompletes_wft_no_work() {
             seq: 1,
             activity_id: activity_id.to_string(),
             cancellation_type: ActivityCancellationType::Abandon as i32,
-            ..Default::default()
+            ..default_act_sched()
         }
         .into(),
     ))
@@ -2013,10 +2076,7 @@ async fn continue_as_new_preserves_some_values() {
     let mut mock_client = mock_workflow_client();
     let hist = {
         let mut t = TestHistoryBuilder::default();
-        t.add(
-            EventType::WorkflowExecutionStarted,
-            wes_attrs.clone().into(),
-        );
+        t.add(wes_attrs.clone());
         t.add_full_wf_task();
         t
     };
@@ -2060,21 +2120,19 @@ async fn continue_as_new_preserves_some_values() {
 async fn ignorable_events_are_ok(#[values(true, false)] attribs_unset: bool) {
     let mut t = TestHistoryBuilder::default();
     t.add_by_type(EventType::WorkflowExecutionStarted);
-    let id = t.add_get_event_id(
-        EventType::Unspecified,
-        Some(
-            history_event::Attributes::WorkflowPropertiesModifiedExternallyEventAttributes(
-                Default::default(),
-            ),
-        ),
-    );
-    t.modify_event(id, |e| e.worker_may_ignore = true);
-    if attribs_unset {
-        t.modify_event(id, |e| {
-            e.event_type = EventType::WorkflowPropertiesModifiedExternally as i32;
+    let id = t.add(WorkflowPropertiesModifiedExternallyEventAttributes::default());
+    t.modify_event(id, |e| {
+        e.worker_may_ignore = true;
+        // Ignorable events are ignored if we can't interpret the proto of either the event attribs
+        // or proto - otherwise (this is the _may_ part of may ignore), we'll still try to process
+        // it. That processing may ultimately still choose to do nothing, if we want to _explicitly_
+        // ignore it.
+        if attribs_unset {
             e.attributes = None;
-        });
-    }
+        } else {
+            e.event_type = EventType::Unspecified as i32;
+        }
+    });
     t.add_workflow_task_scheduled_and_started();
 
     let mock = mock_workflow_client();
@@ -2087,3 +2145,478 @@ async fn ignorable_events_are_ok(#[values(true, false)] attribs_unset: bool) {
         Some(workflow_activation_job::Variant::StartWorkflow(_))
     );
 }
+
+#[tokio::test]
+async fn fetching_to_continue_replay_works() {
+    let mut mock_client = mock_workflow_client();
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_full_wf_task(); // ends 7
+    let mut need_fetch_resp =
+        hist_to_poll_resp(&t, "wfid".to_owned(), ResponseType::AllHistory).resp;
+    need_fetch_resp.next_page_token = vec![1];
+
+    t.add_full_wf_task();
+    t.add_we_signaled("hi", vec![]); // Need to make there be two complete WFTs
+    t.add_full_wf_task(); // end 14
+    let mut fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_full_history_info().unwrap().into();
+    // Should only contain events after 7
+    if let Some(ref mut h) = fetch_resp.history {
+        h.events.retain(|e| e.event_id >= 8);
+    }
+    // And indicate that even *more* needs to be fetched after this, so we see a request for the
+    // next page happen.
+    fetch_resp.next_page_token = vec![2];
+
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_full_wf_task();
+    let mut final_fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_full_history_info().unwrap().into();
+    // Should have only the final event
+    if let Some(ref mut h) = final_fetch_resp.history {
+        h.events.retain(|e| e.event_id >= 15);
+    }
+
+    let tasks = vec![
+        ResponseType::ToTaskNum(1),
+        ResponseType::Raw(need_fetch_resp),
+    ];
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(fetch_resp.clone()))
+        .times(1);
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(final_fetch_resp.clone()))
+        .times(1);
+    let mut mock = single_hist_mock_sg("wfid", t, tasks, mock_client, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    let core = mock_worker(mock);
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant,
+        Some(workflow_activation_job::Variant::StartWorkflow(_))
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        }]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        act.run_id,
+        start_timer_cmd(1, Duration::from_secs(1)),
+    ))
+    .await
+    .unwrap();
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::FireTimer(_)),
+        }]
+    );
+}
+
+#[tokio::test]
+async fn fetching_error_evicts_wf() {
+    let mut mock_client = mock_workflow_client();
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+    t.add_workflow_task_completed();
+    let mut need_fetch_resp =
+        hist_to_poll_resp(&t, "wfid".to_owned(), ResponseType::AllHistory).resp;
+    need_fetch_resp.next_page_token = vec![1];
+    let tasks = vec![
+        ResponseType::ToTaskNum(1),
+        ResponseType::Raw(need_fetch_resp),
+    ];
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Err(tonic::Status::not_found("Ahh broken")))
+        .times(1);
+    let mut mock = single_hist_mock_sg("wfid", t, tasks, mock_client, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    let core = mock_worker(mock);
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant,
+        Some(workflow_activation_job::Variant::StartWorkflow(_))
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+    let evict_act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        evict_act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(r)),
+        }] => r.message.contains("Fetching history failed")
+    );
+}
+
+/// This test verifies that if we fail to fetch a page during a completion, that we don't get stuck
+/// in the complete waiting for the completion to finish.
+#[tokio::test]
+async fn ensure_fetching_fail_during_complete_sends_task_failure() {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task(); // started 3
+    t.add_we_signaled("sig1", vec![]);
+    t.add_full_wf_task(); // started 7
+    t.add_we_signaled("sig2", vec![]);
+    t.add_full_wf_task(); // started 11
+    t.add_workflow_execution_completed();
+
+    let mut first_poll = hist_to_poll_resp(&t, wfid, ResponseType::ToTaskNum(1)).resp;
+    first_poll.next_page_token = vec![1];
+    first_poll.previous_started_event_id = 3;
+
+    let mut next_page: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
+    next_page.next_page_token = vec![2];
+
+    let mut mock = mock_workflow_client();
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| {
+            error!("Called fetch!");
+            Ok(next_page.clone())
+        })
+        .times(1);
+    let mut really_empty_fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_history_info(1).unwrap().into();
+    really_empty_fetch_resp.history = Some(Default::default());
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| {
+            error!("Called fetch second time!");
+            Ok(really_empty_fetch_resp.clone())
+        })
+        .times(1);
+    mock.expect_fail_workflow_task()
+        .returning(|_, _, _| Ok(Default::default()))
+        .times(1);
+
+    let mut mock = single_hist_mock_sg(wfid, t, [ResponseType::Raw(first_poll)], mock, true);
+    mock.make_wft_stream_interminable();
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
+    let core = mock_worker(mock);
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        },]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+
+    // Expect to see eviction b/c of history fetching error here.
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
+        },]
+    );
+
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn lang_internal_flags() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.set_flags_first_wft(&[], &[1]);
+    t.add_we_signaled("sig1", vec![]);
+    t.add_full_wf_task();
+    t.set_flags_last_wft(&[], &[2]);
+    t.add_we_signaled("sig2", vec![]);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::ToTaskNum(2), ResponseType::AllHistory],
+        mock_workflow_client(),
+    );
+    mh.completion_asserts = Some(Box::new(|c| {
+        assert_matches!(c.sdk_metadata.lang_used_flags.as_slice(), &[2]);
+    }));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(act.available_internal_flags.as_slice(), [1]);
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    let mut completion = WorkflowActivationCompletion::empty(act.run_id);
+    completion.add_internal_flags(2);
+    core.complete_workflow_activation(completion).await.unwrap();
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(act.available_internal_flags.as_slice(), [1, 2]);
+    core.complete_execution(&act.run_id).await;
+    core.shutdown().await;
+}
+
+// Verify we send flags to server when they're used
+#[tokio::test]
+async fn core_internal_flags() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    let act_scheduled_event_id = t.add_activity_task_scheduled("act-id");
+    let act_started_event_id = t.add_activity_task_started(act_scheduled_event_id);
+    t.add_activity_task_completed(
+        act_scheduled_event_id,
+        act_started_event_id,
+        Default::default(),
+    );
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::ToTaskNum(1), ResponseType::ToTaskNum(2)],
+        mock_workflow_client(),
+    );
+    let first_poll = AtomicBool::new(true);
+    mh.completion_asserts = Some(Box::new(move |c| {
+        if !first_poll.load(Ordering::Acquire) {
+            assert_matches!(c.sdk_metadata.core_used_flags.as_slice(), &[1]);
+        }
+        first_poll.store(false, Ordering::Release);
+    }));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        act.run_id,
+        schedule_activity_cmd(
+            1,
+            "whatever",
+            "act-id",
+            ActivityCancellationType::TryCancel,
+            Duration::from_secs(60),
+            Duration::from_secs(60),
+        ),
+    ))
+    .await
+    .unwrap();
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    core.complete_execution(&act.run_id).await;
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn post_terminal_commands_are_discarded() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::ToTaskNum(1), ResponseType::AllHistory],
+        mock_workflow_client(),
+    );
+    mh.completion_asserts = Some(Box::new(|c| {
+        // Only the complete execution command should actually be sent
+        assert_eq!(c.commands.len(), 1);
+    }));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        act.run_id,
+        vec![
+            CompleteWorkflowExecution { result: None }.into(),
+            start_timer_cmd(1, Duration::from_secs(1)),
+        ],
+    ))
+    .await
+    .unwrap();
+
+    // This just ensures applying the complete history w/ the completion command works, though
+    // there's no activation.
+    let act = core.poll_workflow_activation().await;
+    assert_matches!(act.unwrap_err(), PollWfError::ShutDown);
+
+    core.shutdown().await;
+}
+
+// Lang expects to always see jobs in this order:
+//   patches, signals, everything else, queries
+#[tokio::test]
+async fn jobs_are_in_appropriate_order() {
+    let p1 = "patchy-mc-patchface";
+    let p2 = "enchi-the-kitty";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_has_change_marker(p1, false);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_we_signaled("yummy-salmon", vec![]);
+    t.add_full_wf_task();
+    t.add_has_change_marker(p2, false);
+    t.add_workflow_execution_completed();
+
+    let mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::AllHistory],
+        mock_workflow_client(),
+    );
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    // Patch notifications always come first
+    assert_matches!(
+        act.jobs[0].variant.as_ref().unwrap(),
+        workflow_activation_job::Variant::NotifyHasPatch(_)
+    );
+    assert_matches!(
+        act.jobs[1].variant.as_ref().unwrap(),
+        workflow_activation_job::Variant::StartWorkflow(_)
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        act.run_id,
+        vec![
+            SetPatchMarker {
+                patch_id: p1.to_string(),
+                deprecated: false,
+            }
+            .into(),
+            start_timer_cmd(1, Duration::from_secs(1)),
+        ],
+    ))
+    .await
+    .unwrap();
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant.as_ref().unwrap(),
+        workflow_activation_job::Variant::NotifyHasPatch(_)
+    );
+    assert_matches!(
+        act.jobs[1].variant.as_ref().unwrap(),
+        workflow_activation_job::Variant::SignalWorkflow(_)
+    );
+    assert_matches!(
+        act.jobs[2].variant.as_ref().unwrap(),
+        workflow_activation_job::Variant::FireTimer(_)
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn history_length_with_fail_and_timeout(
+    #[values(true, false)] use_cache: bool,
+    #[values(1, 2, 3)] history_responses_case: u8,
+) {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_workflow_task_scheduled_and_started();
+    t.add_workflow_task_failed_with_failure(WorkflowTaskFailedCause::Unspecified, "ahh".into());
+    t.add_workflow_task_scheduled_and_started();
+    t.add_workflow_task_timed_out();
+    t.add_full_wf_task();
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
+    t.add_timer_fired(timer_started_event_id, "2".to_string());
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mock_client = mock_workflow_client();
+    let history_responses = match history_responses_case {
+        1 => vec![ResponseType::AllHistory],
+        2 => vec![
+            ResponseType::ToTaskNum(1),
+            ResponseType::ToTaskNum(2),
+            ResponseType::AllHistory,
+        ],
+        3 => {
+            let mut needs_fetch = hist_to_poll_resp(&t, wfid, ResponseType::ToTaskNum(2)).resp;
+            needs_fetch.next_page_token = vec![1];
+            // Truncate the history a bit in order to force incomplete WFT
+            needs_fetch.history.as_mut().unwrap().events.truncate(6);
+            let needs_fetch_resp = ResponseType::Raw(needs_fetch);
+            let mut empty_fetch_resp: GetWorkflowExecutionHistoryResponse =
+                t.get_history_info(1).unwrap().into();
+            empty_fetch_resp.history.as_mut().unwrap().events = vec![];
+            mock_client
+                .expect_get_workflow_execution_history()
+                .returning(move |_, _, _| Ok(empty_fetch_resp.clone()))
+                .times(1);
+            vec![
+                ResponseType::ToTaskNum(1),
+                needs_fetch_resp,
+                ResponseType::ToTaskNum(2),
+                ResponseType::AllHistory,
+            ]
+        }
+        _ => unreachable!(),
+    };
+
+    let mut mh = MockPollCfg::from_resp_batches(wfid, t, history_responses, mock_client);
+    if history_responses_case == 3 {
+        // Expect the failed pagination fetch
+        mh.num_expected_fails = 1;
+    }
+    let mut worker = mock_sdk_cfg(mh, |wc| {
+        if use_cache {
+            wc.max_cached_workflows = 1;
+        }
+    });
+    worker.register_wf(DEFAULT_WORKFLOW_TYPE, |ctx: WfContext| async move {
+        assert_eq!(ctx.history_length(), 3);
+        ctx.timer(Duration::from_secs(1)).await;
+        assert_eq!(ctx.history_length(), 14);
+        ctx.timer(Duration::from_secs(1)).await;
+        assert_eq!(ctx.history_length(), 19);
+        Ok(().into())
+    });
+    worker
+        .submit_wf(
+            wfid.to_owned(),
+            DEFAULT_WORKFLOW_TYPE.to_owned(),
+            vec![],
+            WorkflowOptions::default(),
+        )
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+}