@temporalio/core-bridge 1.11.3 → 1.11.5

package/sdk-core/core/src/worker/tuner/resource_based.rs

@@ -3,7 +3,7 @@ use parking_lot::Mutex;
 use std::{
     marker::PhantomData,
     sync::{
-        atomic::{AtomicU64, Ordering},
+        atomic::{AtomicU64, AtomicUsize, Ordering},
         Arc, OnceLock,
     },
     time::{Duration, Instant},
@@ -11,7 +11,8 @@ use std::{
 use temporal_sdk_core_api::{
     telemetry::metrics::{CoreMeter, GaugeF64, MetricAttributes, TemporalMeter},
     worker::{
-        ActivitySlotKind, LocalActivitySlotKind, SlotKind, SlotReservationContext, SlotSupplier,
+        ActivitySlotKind, LocalActivitySlotKind, SlotInfo, SlotInfoTrait, SlotKind, SlotKindType,
+        SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext, SlotSupplier,
         SlotSupplierPermit, WorkerTuner, WorkflowSlotKind,
     },
 };
@@ -124,6 +125,12 @@ pub(crate) struct ResourceBasedSlotsForType<MI, SK> {
 
     last_slot_issued_tx: watch::Sender<Instant>,
     last_slot_issued_rx: watch::Receiver<Instant>,
+
+    // Only used for workflow slots - count of issued non-sticky slots
+    issued_nonsticky: AtomicUsize,
+    // Only used for workflow slots - count of issued sticky slots
+    issued_sticky: AtomicUsize,
+
     _slot_kind: PhantomData<SK>,
 }
 /// Allows for the full customization of the PID options for a resource based tuner
@@ -241,42 +248,51 @@ where
 
     async fn reserve_slot(&self, ctx: &dyn SlotReservationContext) -> SlotSupplierPermit {
         loop {
-            if ctx.num_issued_slots() < self.opts.min_slots {
-                return self.issue_slot();
+            if let Some(value) = self.issue_if_below_minimums(ctx) {
+                return value;
+            }
+            let must_wait_for = self
+                .opts
+                .ramp_throttle
+                .saturating_sub(self.time_since_last_issued());
+            if must_wait_for > Duration::from_millis(0) {
+                tokio::time::sleep(must_wait_for).await;
+            }
+            if let Some(p) = self.try_reserve_slot(ctx) {
+                return p;
             } else {
-                let must_wait_for = self
-                    .opts
-                    .ramp_throttle
-                    .saturating_sub(self.time_since_last_issued());
-                if must_wait_for > Duration::from_millis(0) {
-                    tokio::time::sleep(must_wait_for).await;
-                }
-                if let Some(p) = self.try_reserve_slot(ctx) {
-                    return p;
-                } else {
-                    tokio::time::sleep(Duration::from_millis(10)).await;
-                }
+                tokio::time::sleep(Duration::from_millis(10)).await;
             }
         }
     }
 
     fn try_reserve_slot(&self, ctx: &dyn SlotReservationContext) -> Option<SlotSupplierPermit> {
-        let num_issued = ctx.num_issued_slots();
-        if num_issued < self.opts.min_slots
-            || (self.time_since_last_issued() > self.opts.ramp_throttle
-                && num_issued < self.opts.max_slots
-                && self.inner.pid_decision()
-                && self.inner.can_reserve())
+        if let v @ Some(_) = self.issue_if_below_minimums(ctx) {
+            return v;
+        }
+        if self.time_since_last_issued() > self.opts.ramp_throttle
+            && ctx.num_issued_slots() < self.opts.max_slots
+            && self.inner.pid_decision()
+            && self.inner.can_reserve()
         {
-            Some(self.issue_slot())
+            Some(self.issue_slot(ctx))
         } else {
             None
         }
     }
 
-    fn mark_slot_used(&self, _info: SK::Info<'_>) {}
+    fn mark_slot_used(&self, _ctx: &dyn SlotMarkUsedContext<SlotKind = Self::SlotKind>) {}
 
-    fn release_slot(&self) {}
+    fn release_slot(&self, ctx: &dyn SlotReleaseContext<SlotKind = Self::SlotKind>) {
+        // Really could use specialization here
+        if let Some(SlotInfo::Workflow(info)) = ctx.info().map(|i| i.downcast()) {
+            if info.is_sticky {
+                self.issued_sticky.fetch_sub(1, Ordering::Relaxed);
+            } else {
+                self.issued_nonsticky.fetch_sub(1, Ordering::Relaxed);
+            }
+        }
+    }
 }
 
 impl<MI, SK> ResourceBasedSlotsForType<MI, SK>
@@ -291,11 +307,41 @@ where
             last_slot_issued_tx: tx,
             last_slot_issued_rx: rx,
             inner,
+            issued_nonsticky: Default::default(),
+            issued_sticky: Default::default(),
             _slot_kind: PhantomData,
         }
     }
 
-    fn issue_slot(&self) -> SlotSupplierPermit {
+    // Always be willing to hand out at least 1 slot for sticky and 1 for non-sticky to
+    // avoid getting stuck.
+    fn issue_if_below_minimums(
+        &self,
+        ctx: &dyn SlotReservationContext,
+    ) -> Option<SlotSupplierPermit> {
+        if ctx.num_issued_slots() < self.opts.min_slots {
+            return Some(self.issue_slot(ctx));
+        }
+        if SK::kind() == SlotKindType::Workflow
+            && (ctx.is_sticky() && self.issued_sticky.load(Ordering::Relaxed) == 0
+                || !ctx.is_sticky() && self.issued_nonsticky.load(Ordering::Relaxed) == 0)
+        {
+            return Some(self.issue_slot(ctx));
+        }
+
+        None
+    }
+
+    fn issue_slot(&self, ctx: &dyn SlotReservationContext) -> SlotSupplierPermit {
+        // Always be willing to hand out at least 1 slot for sticky and 1 for non-sticky to avoid
+        // getting stuck.
+        if SK::kind() == SlotKindType::Workflow {
+            if ctx.is_sticky() {
+                self.issued_sticky.fetch_add(1, Ordering::Relaxed);
+            } else {
+                self.issued_nonsticky.fetch_add(1, Ordering::Relaxed);
+            }
+        }
         let _ = self.last_slot_issued_tx.send(Instant::now());
         SlotSupplierPermit::default()
     }
@@ -406,7 +452,7 @@ impl<MI: SystemResourceInfo + Sync + Send> ResourceController<MI> {
 #[derive(Debug)]
 pub struct RealSysInfo {
     sys: Mutex<sysinfo::System>,
-    total_mem: u64,
+    total_mem: AtomicU64,
     cur_mem_usage: AtomicU64,
     cur_cpu_usage: AtomicU64,
     last_refresh: AtomicCell<Instant>,
@@ -421,7 +467,7 @@ impl RealSysInfo {
             last_refresh: AtomicCell::new(Instant::now()),
             cur_mem_usage: AtomicU64::new(0),
             cur_cpu_usage: AtomicU64::new(0),
-            total_mem,
+            total_mem: AtomicU64::new(total_mem),
         };
         s.refresh();
         s
@@ -441,17 +487,28 @@ impl RealSysInfo {
         lock.refresh_cpu_usage();
         let mem = lock.used_memory();
         let cpu = lock.global_cpu_usage() as f64 / 100.;
+        if let Some(cgroup_limits) = lock.cgroup_limits() {
+            self.total_mem
+                .store(cgroup_limits.total_memory, Ordering::Release);
+            self.cur_mem_usage.store(
+                cgroup_limits.total_memory - cgroup_limits.free_memory,
+                Ordering::Release,
+            );
+        }
         self.cur_mem_usage.store(mem, Ordering::Release);
         self.cur_cpu_usage.store(cpu.to_bits(), Ordering::Release);
         self.last_refresh.store(Instant::now());
     }
 }
+
 impl SystemResourceInfo for RealSysInfo {
     fn total_mem(&self) -> u64 {
-        self.total_mem
+        self.total_mem.load(Ordering::Acquire)
     }
 
     fn used_mem(&self) -> u64 {
+        // TODO: This should really happen on a background thread since it's getting called from
+        //   the async reserve
         self.refresh_if_needed();
         self.cur_mem_usage.load(Ordering::Acquire)
     }
@@ -512,10 +569,47 @@ mod tests {
             max_slots: 100,
             ramp_throttle: Duration::from_millis(0),
         });
-        let pd = MeteredPermitDealer::new(rbs.clone(), MetricsContext::no_op(), None);
+        let pd = MeteredPermitDealer::new(
+            rbs.clone(),
+            MetricsContext::no_op(),
+            None,
+            Arc::new(Default::default()),
+        );
+        let pd_s = pd.clone().into_sticky();
+        // Start with too high usage
+        used.store(90_000, Ordering::Release);
+        // Show workflow will always allow 1 each of sticky/non-sticky
+        assert!(rbs.try_reserve_slot(&pd).is_some());
+        assert!(rbs.try_reserve_slot(&pd_s).is_some());
+        assert!(rbs.try_reserve_slot(&pd).is_none());
+        assert!(rbs.try_reserve_slot(&pd_s).is_none());
+        used.store(0, Ordering::Release);
+        // Now it's willing to hand out slots again when usage is zero
         assert!(rbs.try_reserve_slot(&pd).is_some());
+        assert!(rbs.try_reserve_slot(&pd_s).is_some());
+    }
+
+    #[test]
+    fn mem_activity_sync() {
+        let (fmis, used) = FakeMIS::new();
+        let rbs = Arc::new(ResourceController::new_with_sysinfo(test_options(), fmis))
+            .as_kind::<ActivitySlotKind>(ResourceSlotOptions {
+            min_slots: 0,
+            max_slots: 100,
+            ramp_throttle: Duration::from_millis(0),
+        });
+        let pd = MeteredPermitDealer::new(
+            rbs.clone(),
+            MetricsContext::no_op(),
+            None,
+            Arc::new(Default::default()),
+        );
+        // Start with too high usage
         used.store(90_000, Ordering::Release);
         assert!(rbs.try_reserve_slot(&pd).is_none());
+        used.store(0, Ordering::Release);
+        // Now it's willing to hand out slots again when usage is zero
+        assert!(rbs.try_reserve_slot(&pd).is_some());
     }
 
     #[tokio::test]
@@ -528,7 +622,47 @@ mod tests {
             max_slots: 100,
             ramp_throttle: Duration::from_millis(0),
         });
-        let pd = MeteredPermitDealer::new(rbs.clone(), MetricsContext::no_op(), None);
+        let pd = MeteredPermitDealer::new(
+            rbs.clone(),
+            MetricsContext::no_op(),
+            None,
+            Arc::new(Default::default()),
+        );
+        let pd_s = pd.clone().into_sticky();
+        let order = crossbeam_queue::ArrayQueue::new(2);
+        // Show workflow will always allow 1 each of sticky/non-sticky
+        let _p1 = rbs.reserve_slot(&pd).await;
+        let _p2 = rbs.reserve_slot(&pd_s).await;
+        // Now we need to have some memory get freed before the next call resolves
+        let waits_free = async {
+            rbs.reserve_slot(&pd).await;
+            order.push(2).unwrap();
+        };
+        let frees = async {
+            used.store(70_000, Ordering::Release);
+            order.push(1).unwrap();
+        };
+        tokio::join!(waits_free, frees);
+        assert_eq!(order.pop(), Some(1));
+        assert_eq!(order.pop(), Some(2));
+    }
+
+    #[tokio::test]
+    async fn mem_activity_async() {
+        let (fmis, used) = FakeMIS::new();
+        used.store(90_000, Ordering::Release);
+        let rbs = Arc::new(ResourceController::new_with_sysinfo(test_options(), fmis))
+            .as_kind::<ActivitySlotKind>(ResourceSlotOptions {
+            min_slots: 0,
+            max_slots: 100,
+            ramp_throttle: Duration::from_millis(0),
+        });
+        let pd = MeteredPermitDealer::new(
+            rbs.clone(),
+            MetricsContext::no_op(),
+            None,
+            Arc::new(Default::default()),
+        );
         let order = crossbeam_queue::ArrayQueue::new(2);
         let waits_free = async {
             rbs.reserve_slot(&pd).await;
@@ -552,7 +686,12 @@ mod tests {
             max_slots: 100,
             ramp_throttle: Duration::from_millis(0),
         });
-        let pd = MeteredPermitDealer::new(rbs.clone(), MetricsContext::no_op(), None);
+        let pd = MeteredPermitDealer::new(
+            rbs.clone(),
+            MetricsContext::no_op(),
+            None,
+            Arc::new(Default::default()),
+        );
         used.store(90_000, Ordering::Release);
         let _p1 = pd.try_acquire_owned().unwrap();
         let _p2 = pd.try_acquire_owned().unwrap();

package/sdk-core/core/src/worker/tuner.rs

@@ -11,7 +11,7 @@ use std::sync::{Arc, OnceLock};
 use temporal_sdk_core_api::{
     telemetry::metrics::TemporalMeter,
     worker::{
-        ActivitySlotKind, LocalActivitySlotKind, SlotSupplier, WorkerConfig, WorkerTuner,
+        ActivitySlotKind, LocalActivitySlotKind, SlotKind, SlotSupplier, WorkerConfig, WorkerTuner,
         WorkflowSlotKind,
     },
 };
@@ -32,13 +32,13 @@ pub struct TunerHolder {
 pub struct TunerHolderOptions {
     /// Options for workflow slots
     #[builder(default, setter(strip_option))]
-    pub workflow_slot_options: Option<SlotSupplierOptions>,
+    pub workflow_slot_options: Option<SlotSupplierOptions<WorkflowSlotKind>>,
     /// Options for activity slots
     #[builder(default, setter(strip_option))]
-    pub activity_slot_options: Option<SlotSupplierOptions>,
+    pub activity_slot_options: Option<SlotSupplierOptions<ActivitySlotKind>>,
     /// Options for local activity slots
     #[builder(default, setter(strip_option))]
-    pub local_activity_slot_options: Option<SlotSupplierOptions>,
+    pub local_activity_slot_options: Option<SlotSupplierOptions<LocalActivitySlotKind>>,
     /// Options that will apply to all resource based slot suppliers. Must be set if any slot
     /// options are [SlotSupplierOptions::ResourceBased]
     #[builder(default, setter(strip_option))]
@@ -67,6 +67,9 @@ impl TunerHolderOptions {
                         .workflow_task_slot_supplier(),
                 );
             }
+            Some(SlotSupplierOptions::Custom(ss)) => {
+                builder.workflow_slot_supplier(ss);
+            }
             None => {}
         }
         match self.activity_slot_options {
@@ -82,6 +85,9 @@ impl TunerHolderOptions {
                         .activity_task_slot_supplier(),
                 );
             }
+            Some(SlotSupplierOptions::Custom(ss)) => {
+                builder.activity_slot_supplier(ss);
+            }
             None => {}
         }
         match self.local_activity_slot_options {
@@ -97,6 +103,9 @@ impl TunerHolderOptions {
                         .local_activity_slot_supplier(),
                 );
             }
+            Some(SlotSupplierOptions::Custom(ss)) => {
+                builder.local_activity_slot_supplier(ss);
+            }
             None => {}
         }
         Ok(builder.build())
@@ -104,8 +113,8 @@ impl TunerHolderOptions {
 }
 
 /// Options for known kinds of slot suppliers
-#[derive(Clone, Debug)]
-pub enum SlotSupplierOptions {
+#[derive(Clone, derive_more::Debug)]
+pub enum SlotSupplierOptions<SK: SlotKind> {
     /// Options for a [FixedSizeSlotSupplier]
     FixedSize {
         /// The number of slots the fixed supplier will have
@@ -113,6 +122,9 @@ pub enum SlotSupplierOptions {
     },
     /// Options for a [ResourceBasedSlots]
     ResourceBased(ResourceSlotOptions),
+    /// A user-implemented slot supplier
+    #[debug("Custom")]
+    Custom(Arc<dyn SlotSupplier<SlotKind = SK> + Send + Sync>),
 }
 
 impl TunerHolderOptionsBuilder {

package/sdk-core/core/src/worker/workflow/history_update.rs

@@ -5,9 +5,8 @@ use crate::{
         workflow::{CacheMissFetchReq, PermittedWFT, PreparedWFT},
     },
 };
-use futures::{future::BoxFuture, FutureExt, Stream, TryFutureExt};
+use futures_util::{future::BoxFuture, FutureExt, Stream, TryFutureExt};
 use itertools::Itertools;
-use once_cell::sync::Lazy;
 use std::{
     collections::VecDeque,
     fmt::Debug,
@@ -15,18 +14,21 @@ use std::{
     mem,
     mem::transmute,
     pin::Pin,
-    sync::Arc,
+    sync::{Arc, LazyLock},
     task::{Context, Poll},
 };
 use temporal_sdk_core_protos::temporal::api::{
     enums::v1::EventType,
-    history::v1::{history_event, History, HistoryEvent, WorkflowTaskCompletedEventAttributes},
+    history::v1::{
+        history_event, history_event::Attributes, History, HistoryEvent,
+        WorkflowTaskCompletedEventAttributes,
+    },
 };
 use tracing::Instrument;
 
-static EMPTY_FETCH_ERR: Lazy<tonic::Status> =
-    Lazy::new(|| tonic::Status::unknown("Fetched empty history page"));
-static EMPTY_TASK_ERR: Lazy<tonic::Status> = Lazy::new(|| {
+static EMPTY_FETCH_ERR: LazyLock<tonic::Status> =
+    LazyLock::new(|| tonic::Status::unknown("Fetched empty history page"));
+static EMPTY_TASK_ERR: LazyLock<tonic::Status> = LazyLock::new(|| {
     tonic::Status::unknown("Received an empty workflow task with no queries or history")
 });
 
@@ -166,7 +168,7 @@ impl HistoryPaginator {
     }
 
     pub(super) async fn from_fetchreq(
-        mut req: CacheMissFetchReq,
+        mut req: Box<CacheMissFetchReq>,
         client: Arc<dyn WorkerClient>,
     ) -> Result<PermittedWFT, tonic::Status> {
         let mut paginator = Self {
@@ -697,6 +699,7 @@ fn find_end_index_of_next_wft_seq(
     }
     let mut last_index = 0;
     let mut saw_any_command_event = false;
+    let mut wft_started_event_id_to_index = vec![];
     for (ix, e) in events.iter().enumerate() {
         last_index = ix;
 
@@ -714,12 +717,13 @@ fn find_end_index_of_next_wft_seq(
         }
 
         if e.event_type() == EventType::WorkflowTaskStarted {
+            wft_started_event_id_to_index.push((e.event_id, ix));
             if let Some(next_event) = events.get(ix + 1) {
-                let et = next_event.event_type();
+                let next_event_type = next_event.event_type();
                 // If the next event is WFT timeout or fail, or abrupt WF execution end, that
                 // doesn't conclude a WFT sequence.
                 if matches!(
-                    et,
+                    next_event_type,
                     EventType::WorkflowTaskFailed
                         | EventType::WorkflowTaskTimedOut
                         | EventType::WorkflowExecutionTimedOut
@@ -731,11 +735,38 @@ fn find_end_index_of_next_wft_seq(
                 // If we've never seen an interesting event and the next two events are a completion
                 // followed immediately again by scheduled, then this is a WFT heartbeat and also
                 // doesn't conclude the sequence.
-                else if et == EventType::WorkflowTaskCompleted {
+                else if next_event_type == EventType::WorkflowTaskCompleted {
                     if let Some(next_next_event) = events.get(ix + 2) {
                         if next_next_event.event_type() == EventType::WorkflowTaskScheduled {
                             continue;
                         } else {
+                            // If we see an update accepted command after WFT completed, we want to
+                            // conclude the WFT sequence where that update should have been
+                            // processed. We don't need to check for any other command types,
+                            // because the only thing that can run before an update validator is a
+                            // signal handler - but if a signal handler ran then there would have
+                            // been a previous signal event, and we would've already concluded the
+                            // previous WFT sequence.
+                            if let Some(
+                                Attributes::WorkflowExecutionUpdateAcceptedEventAttributes(
+                                    ref attr,
+                                ),
+                            ) = next_next_event.attributes
+                            {
+                                // Find index of closest WFT started before sequencing id
+                                if let Some(ret_ix) = wft_started_event_id_to_index
+                                    .iter()
+                                    .rev()
+                                    .find_map(|(eid, ix)| {
+                                        if *eid < attr.accepted_request_sequencing_event_id {
+                                            return Some(*ix);
+                                        }
+                                        None
+                                    })
+                                {
+                                    return NextWFTSeqEndIndex::Complete(ret_ix);
+                                }
+                            }
                             return NextWFTSeqEndIndex::Complete(ix);
                         }
                     } else if !has_last_wft && !saw_any_command_event {
@@ -766,8 +797,7 @@ mod tests {
         test_help::{canned_histories, hist_to_poll_resp, mock_sdk_cfg, MockPollCfg, ResponseType},
         worker::client::mocks::mock_workflow_client,
     };
-    use futures::StreamExt;
-    use futures_util::TryStreamExt;
+    use futures_util::{StreamExt, TryStreamExt};
     use std::sync::atomic::{AtomicUsize, Ordering};
     use temporal_client::WorkflowOptions;
     use temporal_sdk::WfContext;

package/sdk-core/core/src/worker/workflow/machines/transition_coverage.rs

@@ -4,7 +4,7 @@
 //! never ever be removed from behind `#[cfg(test)]` compilation.
 
 use dashmap::{mapref::entry::Entry, DashMap, DashSet};
-use once_cell::sync::Lazy;
+use std::sync::LazyLock;
 use std::{
     path::PathBuf,
     sync::{
@@ -16,11 +16,11 @@ use std::{
 };
 
 // During test we want to know about which transitions we've covered in state machines
-static COVERED_TRANSITIONS: Lazy<DashMap<String, DashSet<CoveredTransition>>> =
-    Lazy::new(DashMap::new);
-static COVERAGE_SENDER: Lazy<SyncSender<(String, CoveredTransition)>> =
-    Lazy::new(spawn_save_coverage_at_end);
-static THREAD_HANDLE: Lazy<Mutex<Option<JoinHandle<()>>>> = Lazy::new(|| Mutex::new(None));
+static COVERED_TRANSITIONS: LazyLock<DashMap<String, DashSet<CoveredTransition>>> =
+    LazyLock::new(DashMap::new);
+static COVERAGE_SENDER: LazyLock<SyncSender<(String, CoveredTransition)>> =
+    LazyLock::new(spawn_save_coverage_at_end);
+static THREAD_HANDLE: LazyLock<Mutex<Option<JoinHandle<()>>>> = LazyLock::new(|| Mutex::new(None));
 
 #[derive(Eq, PartialEq, Hash, Debug)]
 struct CoveredTransition {

package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs

@@ -635,8 +635,8 @@ impl WorkflowMachines {
                 event.attributes,
                 Some(history_event::Attributes::WorkflowExecutionUpdateAdmittedEventAttributes(_)),
             ) {
-                // The server has sent a durable update admitted event: create the message that would have been sent
-                // for a non-durable update request message.
+                // The server has sent a durable update admitted event: create the message that
+                // would have been sent for a non-durable update request message.
                 let msg = IncomingProtocolMessage::try_from(&event).context(
                     "Failed to create protocol message from WorkflowExecutionUpdateAdmittedEvent",
                 )?;
@@ -745,9 +745,10 @@ impl WorkflowMachines {
                 history_event::Attributes::WorkflowExecutionUpdateAcceptedEventAttributes(ref atts),
             ) = e.attributes
             {
-                // We've encountered an UpdateAccepted event during replay: pretend that we received the message we
-                // would have when receiving an update request under not-replay. If this event was preceded by an
-                // UpdateAdmitted event, then use the message that we created when we encountered that.
+                // We've encountered an UpdateAccepted event during replay: pretend that we received
+                // the message we would have when receiving an update request under not-replay. If
+                // this event was preceded by an UpdateAdmitted event, then use the message that we
+                // created when we encountered that.
                 delayed_actions.push(DelayedAction::ProtocolMessage(
                     update_admitted_event_messages
                         .remove(&atts.protocol_instance_id)

package/sdk-core/core/src/worker/workflow/managed_run.rs

@@ -369,7 +369,7 @@ impl ManagedRun {
         mut commands: Vec<WFCommand>,
         used_flags: Vec<u32>,
         resp_chan: Option<oneshot::Sender<ActivationCompleteResult>>,
-    ) -> Result<RunUpdateAct, NextPageReq> {
+    ) -> Result<RunUpdateAct, Box<NextPageReq>> {
         let activation_was_only_eviction = self.activation_is_eviction();
         let (task_token, has_pending_query, start_time) = if let Some(entry) = self.wft.as_ref() {
             (
@@ -447,10 +447,10 @@ impl ManagedRun {
                 return if let Some(paginator) = self.paginator.take() {
                     debug!("Need to fetch a history page before next WFT can be applied");
                     self.completion_waiting_on_page_fetch = Some(rac);
-                    Err(NextPageReq {
+                    Err(Box::new(NextPageReq {
                         paginator,
                         span: Span::current(),
-                    })
+                    }))
                 } else {
                     Ok(self.update_to_acts(Err(RunUpdateErr {
                         source: WFMachinesError::Fatal(

package/sdk-core/core/src/worker/workflow/mod.rs

@@ -38,8 +38,7 @@ use crate::{
     MetricsContext,
 };
 use anyhow::anyhow;
-use futures::{stream::BoxStream, Stream, StreamExt};
-use futures_util::{future::abortable, stream};
+use futures_util::{future::abortable, stream, stream::BoxStream, Stream, StreamExt};
 use itertools::Itertools;
 use prost_types::TimestampError;
 use std::{
@@ -121,7 +120,7 @@ pub(crate) struct Workflows {
     /// If set, can be used to reserve activity task slots for eager-return of new activity tasks.
     activity_tasks_handle: Option<ActivitiesFromWFTsHandle>,
     /// Ensures we stay at or below this worker's maximum concurrent workflow task limit
-    wft_semaphore: Arc<MeteredPermitDealer<WorkflowSlotKind>>,
+    wft_semaphore: MeteredPermitDealer<WorkflowSlotKind>,
     local_act_mgr: Arc<LocalActivityManager>,
     ever_polled: AtomicBool,
 }
@@ -149,7 +148,7 @@ impl Workflows {
         basics: WorkflowBasics,
         sticky_attrs: Option<StickyExecutionAttributes>,
         client: Arc<dyn WorkerClient>,
-        wft_semaphore: Arc<MeteredPermitDealer<WorkflowSlotKind>>,
+        wft_semaphore: MeteredPermitDealer<WorkflowSlotKind>,
         wft_stream: impl Stream<Item = WFTStreamIn> + Send + 'static,
         local_activity_request_sink: impl LocalActivityRequestSink,
         local_act_mgr: Arc<LocalActivityManager>,
@@ -195,8 +194,8 @@ impl Workflows {
                         local_activity_request_sink,
                     );
 
-                    // However, we want to avoid plowing ahead until we've been asked to poll at least
-                    // once. This supports activity-only workers.
+                    // However, we want to avoid plowing ahead until we've been asked to poll at
+                    // least once. This supports activity-only workers.
                     let do_poll = tokio::select! {
                         sp = start_polling_rx => {
                             sp.is_ok()
@@ -560,7 +559,7 @@ impl Workflows {
 
     /// Must be called after every activation completion has finished
     fn post_activation(&self, msg: PostActivationMsg) {
-        self.send_local(msg);
+        self.send_local(Box::new(msg));
     }
 
     /// Handle server errors from either completing or failing a workflow task. Un-handleable errors
@@ -715,6 +714,13 @@ impl Workflows {
             }
         }
     }
+
+    pub(super) fn get_sticky_queue_name(&self) -> Option<String> {
+        self.sticky_attrs
+            .as_ref()
+            .and_then(|sa| sa.worker_task_queue.as_ref())
+            .map(|tq| tq.name.clone())
+    }
 }
 
 /// Returned when a cache miss happens and we need to fetch history from the beginning to

package/sdk-core/core/src/worker/workflow/wft_extraction.rs

@@ -9,11 +9,10 @@ use crate::{
         },
     },
 };
-use futures::Stream;
-use futures_util::{stream, stream::PollNext, FutureExt, StreamExt};
+use futures_util::{stream, stream::PollNext, FutureExt, Stream, StreamExt};
 use std::{future, sync::Arc};
-use temporal_sdk_core_api::worker::{WorkflowSlotInfo, WorkflowSlotKind};
-use temporal_sdk_core_protos::TaskToken;
+use temporal_sdk_core_api::worker::WorkflowSlotKind;
+use temporal_sdk_core_protos::{coresdk::WorkflowSlotInfo, TaskToken};
 use tracing::Span;
 
 /// Transforms incoming validated WFTs and history fetching requests into [PermittedWFT]s ready
@@ -50,8 +49,8 @@ pub(crate) type WFTStreamIn = Result<
 >;
 #[derive(derive_more::From, Debug)]
 pub(super) enum HistoryFetchReq {
-    Full(CacheMissFetchReq, Arc<HistfetchRC>),
-    NextPage(NextPageReq, Arc<HistfetchRC>),
+    Full(Box<CacheMissFetchReq>, Arc<HistfetchRC>),
+    NextPage(Box<NextPageReq>, Arc<HistfetchRC>),
 }
 /// Used inside of `Arc`s to ensure we don't shutdown while there are outstanding fetches.
 #[derive(Debug)]
@@ -76,7 +75,8 @@ impl WFTExtractor {
                             Ok(match HistoryPaginator::from_poll(wft, client).await {
                                 Ok((pag, prep)) => WFTExtractorOutput::NewWFT(PermittedWFT {
                                     permit: permit.into_used(WorkflowSlotInfo {
-                                        workflow_type: prep.workflow_type.as_str(),
+                                        workflow_type: prep.workflow_type.clone(),
+                                        is_sticky: prep.is_incremental(),
                                     }),
                                     work: prep,
                                     paginator: pag,