temporalio 0.0.2 → 0.1.0

data/bridge/sdk-core/core/src/core_tests/workers.rs

@@ -190,9 +190,7 @@ async fn complete_with_task_not_found_during_shutdown() {
         complete_order.borrow_mut().push(1);
     };
     tokio::join!(shutdown_fut, poll_fut, complete_fut);
-    // Shutdown will currently complete first before the actual eviction reply since the
-    // workflow task is marked complete as soon as we get not found back from the server.
-    assert_eq!(&complete_order.into_inner(), &[1, 3, 2])
+    assert_eq!(&complete_order.into_inner(), &[1, 2, 3])
 }
 
 #[tokio::test]

data/bridge/sdk-core/core/src/core_tests/workflow_tasks.rs

@@ -16,7 +16,7 @@ use rstest::{fixture, rstest};
 use std::{
     collections::{HashMap, VecDeque},
     sync::{
-        atomic::{AtomicU64, Ordering},
+        atomic::{AtomicBool, AtomicU64, Ordering},
         Arc,
     },
     time::Duration,
@@ -37,20 +37,25 @@ use temporal_sdk_core_protos::{
         },
         workflow_completion::WorkflowActivationCompletion,
     },
-    default_wes_attribs,
+    default_act_sched, default_wes_attribs,
     temporal::api::{
         command::v1::command::Attributes,
         common::v1::{Payload, RetryPolicy},
         enums::v1::{EventType, WorkflowTaskFailedCause},
         failure::v1::Failure,
-        history::v1::{history_event, TimerFiredEventAttributes},
+        history::v1::{
+            history_event, TimerFiredEventAttributes,
+            WorkflowPropertiesModifiedExternallyEventAttributes,
+        },
         workflowservice::v1::{
             GetWorkflowExecutionHistoryResponse, RespondWorkflowTaskCompletedResponse,
         },
     },
-    DEFAULT_WORKFLOW_TYPE,
+    DEFAULT_ACTIVITY_TYPE, DEFAULT_WORKFLOW_TYPE,
+};
+use temporal_sdk_core_test_utils::{
+    fanout_tasks, schedule_activity_cmd, start_timer_cmd, WorkerTestHelpers,
 };
-use temporal_sdk_core_test_utils::{fanout_tasks, start_timer_cmd};
 use tokio::{
     join,
     sync::{Barrier, Semaphore},
@@ -120,7 +125,7 @@ async fn single_activity_completion(worker: Worker) {
                 &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
                 vec![ScheduleActivity {
                     activity_id: "fake_activity".to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -245,7 +250,7 @@ async fn scheduled_activity_cancellation_try_cancel(hist_batches: &'static [usiz
                     seq: activity_seq,
                     activity_id: activity_id.to_string(),
                     cancellation_type: ActivityCancellationType::TryCancel as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -281,7 +286,7 @@ async fn scheduled_activity_timeout(hist_batches: &'static [usize]) {
                 vec![ScheduleActivity {
                     seq: activity_seq,
                     activity_id: activity_id.to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                     .into()],
             ),
@@ -334,7 +339,7 @@ async fn started_activity_timeout(hist_batches: &'static [usize]) {
                 vec![ScheduleActivity {
                     seq: activity_seq,
                     activity_id: activity_seq.to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                     .into()],
             ),
@@ -389,7 +394,7 @@ async fn cancelled_activity_timeout(hist_batches: &'static [usize]) {
                 vec![ScheduleActivity {
                     seq: activity_seq,
                     activity_id: activity_id.to_string(),
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -457,10 +462,10 @@ async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [
     t.add_by_type(EventType::WorkflowExecutionStarted);
     t.add_full_wf_task();
     let act_scheduled_event_id = t.add_activity_task_scheduled(activity_id);
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     t.add_timer_fired(timer_started_event_id, "1".to_string());
     t.add_full_wf_task();
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
     let act_started_event_id = t.add_activity_task_started(act_scheduled_event_id);
     t.add_activity_task_completed(
         act_scheduled_event_id,
@@ -476,7 +481,7 @@ async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [
 
     worker.register_wf(wf_type.to_owned(), |ctx: WfContext| async move {
         let act_fut = ctx.activity(ActivityOptions {
-            activity_type: "echo_activity".to_string(),
+            activity_type: DEFAULT_ACTIVITY_TYPE.to_string(),
             start_to_close_timeout: Some(Duration::from_secs(5)),
             cancellation_type: ActivityCancellationType::Abandon,
             ..Default::default()
@@ -542,7 +547,7 @@ async fn verify_activity_cancellation(
                     seq: activity_seq,
                     activity_id: activity_seq.to_string(),
                     cancellation_type: cancel_type as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -610,7 +615,7 @@ async fn verify_activity_cancellation_wait_for_cancellation(activity_id: u32, wo
                     seq: activity_id,
                     activity_id: activity_id.to_string(),
                     cancellation_type: ActivityCancellationType::WaitCancellationCompleted as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -892,7 +897,7 @@ async fn max_wft_respected() {
     let total_wfs = 100;
     let wf_ids: Vec<_> = (0..total_wfs)
         .into_iter()
-        .map(|i| format!("fake-wf-{}", i))
+        .map(|i| format!("fake-wf-{i}"))
         .collect();
     let hists = wf_ids.iter().map(|wf_id| {
         let hist = canned_histories::single_timer("1");
@@ -947,7 +952,7 @@ async fn activity_not_canceled_on_replay_repro(hist_batches: &'static [usize]) {
                         seq: activity_id,
                         activity_id: activity_id.to_string(),
                         cancellation_type: ActivityCancellationType::TryCancel as i32,
-                        ..Default::default()
+                        ..default_act_sched()
                     }
                     .into(),
                     start_timer_cmd(1, Duration::from_secs(1)),
@@ -989,9 +994,9 @@ async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static
                 &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
                 vec![ScheduleActivity {
                     seq: activity_id,
-                    activity_id: activity_id.to_string(),
+                    activity_id: "act-1".to_string(),
                     cancellation_type: ActivityCancellationType::TryCancel as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -1022,7 +1027,7 @@ async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static
 async fn lots_of_workflows() {
     let total_wfs = 500;
     let hists = (0..total_wfs).into_iter().map(|i| {
-        let wf_id = format!("fake-wf-{}", i);
+        let wf_id = format!("fake-wf-{i}");
         let hist = canned_histories::single_timer("1");
         FakeWfResponses {
             wf_id,
@@ -1092,7 +1097,7 @@ async fn wft_timeout_repro(hist_batches: &'static [usize]) {
                     seq: activity_id,
                     activity_id: activity_id.to_string(),
                     cancellation_type: ActivityCancellationType::TryCancel as i32,
-                    ..Default::default()
+                    ..default_act_sched()
                 }
                 .into()],
             ),
@@ -1225,14 +1230,13 @@ async fn buffered_work_drained_on_shutdown() {
     let resp_1 = hist_to_poll_resp(&t, wfid.to_owned(), 1.into()).resp;
     t.add_workflow_task_timed_out();
     t.add_full_wf_task();
-    let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
-    t.add(
-        EventType::TimerFired,
-        history_event::Attributes::TimerFiredEventAttributes(TimerFiredEventAttributes {
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
+    t.add(history_event::Attributes::TimerFiredEventAttributes(
+        TimerFiredEventAttributes {
             started_event_id: timer_started_event_id,
             timer_id: "1".to_string(),
-        }),
-    );
+        },
+    ));
     t.add_full_wf_task();
     t.add_workflow_execution_completed();
 
@@ -1264,6 +1268,8 @@ async fn buffered_work_drained_on_shutdown() {
         .unwrap();
     };
     let complete_first = async move {
+        // If the first complete is sent too fast, we may not have had a chance to buffer work.
+        tokio::time::sleep(Duration::from_millis(50)).await;
         core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
             act1.run_id,
             start_timer_cmd(1, Duration::from_secs(1)),
@@ -1272,8 +1278,6 @@ async fn buffered_work_drained_on_shutdown() {
         .unwrap();
     };
     join!(poll_fut, complete_first, async {
-        // If the shutdown is sent too too fast, we might not have got a chance to even buffer work
-        tokio::time::sleep(Duration::from_millis(5)).await;
         core.shutdown().await;
     });
 }
@@ -1303,7 +1307,7 @@ async fn fail_wft_then_recover() {
         act.run_id.clone(),
         vec![ScheduleActivity {
             activity_id: "fake_activity".to_string(),
-            ..Default::default()
+            ..default_act_sched()
         }
         .into()],
     ))
@@ -1355,12 +1359,17 @@ async fn poll_response_triggers_wf_error() {
     t.add_full_wf_task();
     t.add_workflow_execution_completed();
 
-    let mh = MockPollCfg::from_resp_batches(
+    let mut mh = MockPollCfg::from_resp_batches(
         "fake_wf_id",
         t,
         [ResponseType::AllHistory],
         mock_workflow_client(),
     );
+    // Fail wft will be called when auto-failing.
+    mh.num_expected_fails = 1;
+    mh.expect_fail_wft_matcher = Box::new(move |_, cause, _| {
+        matches!(cause, WorkflowTaskFailedCause::NonDeterministicError)
+    });
     let mock = build_mock_pollers(mh);
     let core = mock_worker(mock);
     // Poll for first WFT, which is immediately an eviction
@@ -1453,7 +1462,7 @@ async fn tries_cancel_of_completed_activity() {
         ScheduleActivity {
             seq: 1,
             activity_id: "1".to_string(),
-            ..Default::default()
+            ..default_act_sched()
         }
         .into(),
     ))
@@ -1546,9 +1555,8 @@ async fn failing_wft_doesnt_eat_permit_forever() {
         ))
         .await
         .unwrap();
-    assert_eq!(worker.available_wft_permits().await, 2);
-
     worker.shutdown().await;
+    assert_eq!(worker.available_wft_permits().await, 2);
 }
 
 #[tokio::test]
@@ -1574,6 +1582,8 @@ async fn cache_miss_will_fetch_history() {
     let mut mock = build_mock_pollers(mh);
     mock.worker_cfg(|cfg| {
         cfg.max_cached_workflows = 1;
+        // Also verifies tying the WFT permit to the fetch request doesn't get us stuck
+        cfg.max_outstanding_workflow_tasks = 1;
     });
     let worker = mock_worker(mock);
 
@@ -1687,12 +1697,63 @@ async fn tasks_from_completion_are_delivered() {
     core.shutdown().await;
 }
 
+#[tokio::test]
+async fn pagination_works_with_tasks_from_completion() {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_we_signaled("sig", vec![]);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+    let get_exec_resp: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
+
+    let mut mock = mock_workflow_client();
+    let mut needs_pag_resp = hist_to_poll_resp(&t, wfid.to_owned(), ResponseType::OneTask(2)).resp;
+    needs_pag_resp.next_page_token = vec![1];
+    let complete_resp = RespondWorkflowTaskCompletedResponse {
+        workflow_task: Some(needs_pag_resp),
+        ..Default::default()
+    };
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(move |_| Ok(complete_resp.clone()));
+    mock.expect_complete_workflow_task()
+        .times(1)
+        .returning(|_| Ok(Default::default()));
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(get_exec_resp.clone()))
+        .times(1);
+    let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
+    let core = mock_worker(mock);
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        },]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
+        wf_task.run_id,
+        vec![CompleteWorkflowExecution { result: None }.into()],
+    ))
+    .await
+    .unwrap();
+    core.shutdown().await;
+}
+
 #[tokio::test]
 async fn poll_faster_than_complete_wont_overflow_cache() {
     // Make workflow tasks for 5 different runs
     let tasks: Vec<_> = (1..=5)
         .map(|i| FakeWfResponses {
-            wf_id: format!("wf-{}", i),
+            wf_id: format!("wf-{i}"),
             hist: canned_histories::single_timer("1"),
             response_batches: vec![ResponseType::ToTaskNum(1)],
         })
@@ -1901,7 +1962,7 @@ async fn autocompletes_wft_no_work() {
             seq: 1,
             activity_id: activity_id.to_string(),
             cancellation_type: ActivityCancellationType::Abandon as i32,
-            ..Default::default()
+            ..default_act_sched()
         }
         .into(),
     ))
@@ -2014,10 +2075,7 @@ async fn continue_as_new_preserves_some_values() {
     let mut mock_client = mock_workflow_client();
     let hist = {
         let mut t = TestHistoryBuilder::default();
-        t.add(
-            EventType::WorkflowExecutionStarted,
-            wes_attrs.clone().into(),
-        );
+        t.add(wes_attrs.clone());
         t.add_full_wf_task();
         t
     };
@@ -2061,21 +2119,19 @@ async fn continue_as_new_preserves_some_values() {
 async fn ignorable_events_are_ok(#[values(true, false)] attribs_unset: bool) {
     let mut t = TestHistoryBuilder::default();
     t.add_by_type(EventType::WorkflowExecutionStarted);
-    let id = t.add_get_event_id(
-        EventType::Unspecified,
-        Some(
-            history_event::Attributes::WorkflowPropertiesModifiedExternallyEventAttributes(
-                Default::default(),
-            ),
-        ),
-    );
-    t.modify_event(id, |e| e.worker_may_ignore = true);
-    if attribs_unset {
-        t.modify_event(id, |e| {
-            e.event_type = EventType::WorkflowPropertiesModifiedExternally as i32;
+    let id = t.add(WorkflowPropertiesModifiedExternallyEventAttributes::default());
+    t.modify_event(id, |e| {
+        e.worker_may_ignore = true;
+        // Ignorable events are ignored if we can't interpret the proto of either the event attribs
+        // or proto - otherwise (this is the _may_ part of may ignore), we'll still try to process
+        // it. That processing may ultimately still choose to do nothing, if we want to _explicitly_
+        // ignore it.
+        if attribs_unset {
             e.attributes = None;
-        });
-    }
+        } else {
+            e.event_type = EventType::Unspecified as i32;
+        }
+    });
     t.add_workflow_task_scheduled_and_started();
 
     let mock = mock_workflow_client();
@@ -2088,3 +2144,290 @@ async fn ignorable_events_are_ok(#[values(true, false)] attribs_unset: bool) {
         Some(workflow_activation_job::Variant::StartWorkflow(_))
     );
 }
+
+#[tokio::test]
+async fn fetching_to_continue_replay_works() {
+    let mut mock_client = mock_workflow_client();
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.add_full_wf_task(); // ends 7
+    let mut need_fetch_resp =
+        hist_to_poll_resp(&t, "wfid".to_owned(), ResponseType::AllHistory).resp;
+    need_fetch_resp.next_page_token = vec![1];
+
+    t.add_full_wf_task();
+    t.add_we_signaled("hi", vec![]); // Need to make there be two complete WFTs
+    t.add_full_wf_task(); // end 14
+    let mut fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_full_history_info().unwrap().into();
+    // Should only contain events after 7
+    if let Some(ref mut h) = fetch_resp.history {
+        h.events.retain(|e| e.event_id >= 8);
+    }
+    // And indicate that even *more* needs to be fetched after this, so we see a request for the
+    // next page happen.
+    fetch_resp.next_page_token = vec![2];
+
+    let timer_started_event_id = t.add_by_type(EventType::TimerStarted);
+    t.add_timer_fired(timer_started_event_id, "1".to_string());
+    t.add_full_wf_task();
+    let mut final_fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_full_history_info().unwrap().into();
+    // Should have only the final event
+    if let Some(ref mut h) = final_fetch_resp.history {
+        h.events.retain(|e| e.event_id >= 15);
+    }
+
+    let tasks = vec![
+        ResponseType::ToTaskNum(1),
+        ResponseType::Raw(need_fetch_resp),
+    ];
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(fetch_resp.clone()))
+        .times(1);
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Ok(final_fetch_resp.clone()))
+        .times(1);
+    let mut mock = single_hist_mock_sg("wfid", t, tasks, mock_client, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    let core = mock_worker(mock);
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant,
+        Some(workflow_activation_job::Variant::StartWorkflow(_))
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        }]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        act.run_id,
+        start_timer_cmd(1, Duration::from_secs(1)),
+    ))
+    .await
+    .unwrap();
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::FireTimer(_)),
+        }]
+    );
+}
+
+#[tokio::test]
+async fn fetching_error_evicts_wf() {
+    let mut mock_client = mock_workflow_client();
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_workflow_task_scheduled_and_started();
+    t.add_workflow_task_completed();
+    let mut need_fetch_resp =
+        hist_to_poll_resp(&t, "wfid".to_owned(), ResponseType::AllHistory).resp;
+    need_fetch_resp.next_page_token = vec![1];
+    let tasks = vec![
+        ResponseType::ToTaskNum(1),
+        ResponseType::Raw(need_fetch_resp),
+    ];
+    mock_client
+        .expect_get_workflow_execution_history()
+        .returning(move |_, _, _| Err(tonic::Status::not_found("Ahh broken")))
+        .times(1);
+    let mut mock = single_hist_mock_sg("wfid", t, tasks, mock_client, true);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
+    let core = mock_worker(mock);
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        act.jobs[0].variant,
+        Some(workflow_activation_job::Variant::StartWorkflow(_))
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+    let evict_act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        evict_act.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(r)),
+        }] => r.message.contains("Fetching history failed")
+    );
+}
+
+/// This test verifies that if we fail to fetch a page during a completion, that we don't get stuck
+/// in the complete waiting for the completion to finish.
+#[tokio::test]
+async fn ensure_fetching_fail_during_complete_sends_task_failure() {
+    let wfid = "fake_wf_id";
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task(); // started 3
+    t.add_we_signaled("sig1", vec![]);
+    t.add_full_wf_task(); // started 7
+    t.add_we_signaled("sig2", vec![]);
+    t.add_full_wf_task(); // started 11
+    t.add_workflow_execution_completed();
+
+    let mut first_poll = hist_to_poll_resp(&t, wfid, ResponseType::ToTaskNum(1)).resp;
+    first_poll.next_page_token = vec![1];
+    first_poll.previous_started_event_id = 3;
+
+    let mut next_page: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
+    next_page.next_page_token = vec![2];
+
+    let mut mock = mock_workflow_client();
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| {
+            error!("Called fetch!");
+            Ok(next_page.clone())
+        })
+        .times(1);
+    let mut really_empty_fetch_resp: GetWorkflowExecutionHistoryResponse =
+        t.get_history_info(1).unwrap().into();
+    really_empty_fetch_resp.history = Some(Default::default());
+    mock.expect_get_workflow_execution_history()
+        .returning(move |_, _, _| {
+            error!("Called fetch second time!");
+            Ok(really_empty_fetch_resp.clone())
+        })
+        .times(1);
+    mock.expect_fail_workflow_task()
+        .returning(|_, _, _| Ok(Default::default()))
+        .times(1);
+
+    let mut mock = single_hist_mock_sg(wfid, t, [ResponseType::Raw(first_poll)], mock, true);
+    mock.make_wft_stream_interminable();
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
+    let core = mock_worker(mock);
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+        },]
+    );
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
+        .await
+        .unwrap();
+
+    // Expect to see eviction b/c of history fetching error here.
+    let wf_task = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(
+        wf_task.jobs.as_slice(),
+        [WorkflowActivationJob {
+            variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
+        },]
+    );
+
+    core.shutdown().await;
+}
+
+#[tokio::test]
+async fn lang_internal_flags() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    t.set_flags_first_wft(&[], &[1]);
+    t.add_we_signaled("sig1", vec![]);
+    t.add_full_wf_task();
+    t.set_flags_last_wft(&[], &[2]);
+    t.add_we_signaled("sig2", vec![]);
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::ToTaskNum(2), ResponseType::AllHistory],
+        mock_workflow_client(),
+    );
+    mh.completion_asserts = Some(Box::new(|c| {
+        assert_matches!(c.sdk_metadata.lang_used_flags.as_slice(), &[2]);
+    }));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(act.available_internal_flags.as_slice(), [1]);
+    core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
+        .await
+        .unwrap();
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    let mut completion = WorkflowActivationCompletion::empty(act.run_id);
+    completion.add_internal_flags(2);
+    core.complete_workflow_activation(completion).await.unwrap();
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    assert_matches!(act.available_internal_flags.as_slice(), [1, 2]);
+    core.complete_execution(&act.run_id).await;
+    core.shutdown().await;
+}
+
+// Verify we send flags to server when they're used
+#[tokio::test]
+async fn core_internal_flags() {
+    let mut t = TestHistoryBuilder::default();
+    t.add_by_type(EventType::WorkflowExecutionStarted);
+    t.add_full_wf_task();
+    let act_scheduled_event_id = t.add_activity_task_scheduled("act-id");
+    let act_started_event_id = t.add_activity_task_started(act_scheduled_event_id);
+    t.add_activity_task_completed(
+        act_scheduled_event_id,
+        act_started_event_id,
+        Default::default(),
+    );
+    t.add_full_wf_task();
+    t.add_workflow_execution_completed();
+
+    let mut mh = MockPollCfg::from_resp_batches(
+        "fake_wf_id",
+        t,
+        [ResponseType::ToTaskNum(1), ResponseType::ToTaskNum(2)],
+        mock_workflow_client(),
+    );
+    let first_poll = AtomicBool::new(true);
+    mh.completion_asserts = Some(Box::new(move |c| {
+        if !first_poll.load(Ordering::Acquire) {
+            assert_matches!(c.sdk_metadata.core_used_flags.as_slice(), &[1]);
+        }
+        first_poll.store(false, Ordering::Release);
+    }));
+    let mut mock = build_mock_pollers(mh);
+    mock.worker_cfg(|wc| wc.max_cached_workflows = 1);
+    let core = mock_worker(mock);
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
+        act.run_id,
+        schedule_activity_cmd(
+            1,
+            "whatever",
+            "act-id",
+            ActivityCancellationType::TryCancel,
+            Duration::from_secs(60),
+            Duration::from_secs(60),
+        ),
+    ))
+    .await
+    .unwrap();
+
+    let act = core.poll_workflow_activation().await.unwrap();
+    core.complete_execution(&act.run_id).await;
+    core.shutdown().await;
+}