@temporalio/core-bridge 1.12.2 → 1.12.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@temporalio/core-bridge",
-  "version": "1.12.2",
+  "version": "1.12.3",
   "description": "Temporal.io SDK Core<>Node bridge",
   "main": "index.js",
   "types": "lib/index.d.ts",
@@ -23,7 +23,7 @@
   "license": "MIT",
   "dependencies": {
     "@grpc/grpc-js": "^1.12.4",
-    "@temporalio/common": "1.12.2",
+    "@temporalio/common": "1.12.3",
     "arg": "^5.0.2",
     "cargo-cp-artifact": "^0.1.8",
     "which": "^4.0.0"
@@ -56,5 +56,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "98393e00b714b8d44a3dc25714d313d3366f4c50"
+  "gitHead": "e25f1d5ddaf0b5c755457b1cc1dde7c6e089a63b"
 }

package/sdk-core/.cargo/config.toml

@@ -1,6 +1,6 @@
 [env]
 # This temporarily overrides the version of the CLI used for integration tests, locally and in CI
-#CLI_VERSION_OVERRIDE = "v1.3.1-priority.0"
+CLI_VERSION_OVERRIDE = "v1.4.1-cloud-v1-29-0-139-2.0"
 
 [alias]
 integ-test = ["test", "--features", "temporal-sdk-core-protos/serde_serialize", "--package", "temporal-sdk-core", "--test", "integ_runner", "--"]

package/sdk-core/client/src/callback_based.rs

@@ -0,0 +1,123 @@
+//! This module implements support for callback-based gRPC service that has a callback invoked for
+//! every gRPC call instead of directly using the network.
+
+use anyhow::anyhow;
+use bytes::{BufMut, BytesMut};
+use futures_util::future::BoxFuture;
+use futures_util::stream;
+use http::{HeaderMap, Request, Response};
+use http_body_util::{BodyExt, StreamBody, combinators::BoxBody};
+use hyper::body::{Bytes, Frame};
+use std::{
+    sync::Arc,
+    task::{Context, Poll},
+};
+use tonic::{Status, metadata::GRPC_CONTENT_TYPE};
+use tower::Service;
+
+/// gRPC request for use by a callback.
+pub struct GrpcRequest {
+    /// Fully qualified gRPC service name.
+    pub service: String,
+    /// RPC name.
+    pub rpc: String,
+    /// Request headers.
+    pub headers: HeaderMap,
+    /// Protobuf bytes of the request.
+    pub proto: Bytes,
+}
+
+/// Successful gRPC response returned by a callback.
+pub struct GrpcSuccessResponse {
+    /// Response headers.
+    pub headers: HeaderMap,
+
+    /// Response proto bytes.
+    pub proto: Vec<u8>,
+}
+
+/// gRPC service that invokes the given callback on each call.
+#[derive(Clone)]
+pub struct CallbackBasedGrpcService {
+    /// Callback to invoke on each RPC call.
+    #[allow(clippy::type_complexity)] // Signature is not that complex
+    pub callback: Arc<
+        dyn Fn(GrpcRequest) -> BoxFuture<'static, Result<GrpcSuccessResponse, Status>>
+            + Send
+            + Sync,
+    >,
+}
+
+impl Service<Request<tonic::body::Body>> for CallbackBasedGrpcService {
+    type Response = http::Response<tonic::body::Body>;
+    type Error = anyhow::Error;
+    type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;
+
+    fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        Poll::Ready(Ok(()))
+    }
+
+    fn call(&mut self, req: Request<tonic::body::Body>) -> Self::Future {
+        let callback = self.callback.clone();
+
+        Box::pin(async move {
+            // Build req
+            let (parts, body) = req.into_parts();
+            let mut path_parts = parts.uri.path().trim_start_matches('/').split('/');
+            let req_body = body.collect().await.map_err(|e| anyhow!(e))?.to_bytes();
+            // Body is flag saying whether compressed (we do not support that), then 32-bit length,
+            // then the actual proto.
+            if req_body.len() < 5 {
+                return Err(anyhow!("Too few request bytes: {}", req_body.len()));
+            } else if req_body[0] != 0 {
+                return Err(anyhow!("Compression not supported"));
+            }
+            let req_proto_len =
+                u32::from_be_bytes([req_body[1], req_body[2], req_body[3], req_body[4]]) as usize;
+            if req_body.len() < 5 + req_proto_len {
+                return Err(anyhow!(
+                    "Expected request body length at least {}, got {}",
+                    5 + req_proto_len,
+                    req_body.len()
+                ));
+            }
+            let req = GrpcRequest {
+                service: path_parts.next().unwrap_or_default().to_owned(),
+                rpc: path_parts.next().unwrap_or_default().to_owned(),
+                headers: parts.headers,
+                proto: req_body.slice(5..5 + req_proto_len),
+            };
+
+            // Invoke and handle response
+            match (callback)(req).await {
+                Ok(success) => {
+                    // Create body bytes which requires a flag saying whether compressed, then
+                    // message len, then actual message. So we create a Bytes for those 5 prepend
+                    // parts, then stream it alongside the user-provided Vec. This allows us to
+                    // avoid copying the vec
+                    let mut body_prepend = BytesMut::with_capacity(5);
+                    body_prepend.put_u8(0); // 0 means no compression
+                    body_prepend.put_u32(success.proto.len() as u32);
+                    let stream = stream::iter(vec![
+                        Ok::<_, Status>(Frame::data(Bytes::from(body_prepend))),
+                        Ok::<_, Status>(Frame::data(Bytes::from(success.proto))),
+                    ]);
+                    let stream_body = StreamBody::new(stream);
+                    let full_body = BoxBody::new(stream_body).boxed();
+
+                    // Build response appending headers
+                    let mut resp_builder = Response::builder()
+                        .status(200)
+                        .header(http::header::CONTENT_TYPE, GRPC_CONTENT_TYPE);
+                    for (key, value) in success.headers.iter() {
+                        resp_builder = resp_builder.header(key, value);
+                    }
+                    Ok(resp_builder
+                        .body(tonic::body::Body::new(full_body))
+                        .map_err(|e| anyhow!(e))?)
+                }
+                Err(status) => Ok(status.into_http()),
+            }
+        })
+    }
+}

package/sdk-core/client/src/lib.rs

@@ -7,6 +7,7 @@
 #[macro_use]
 extern crate tracing;
 
+pub mod callback_based;
 mod metrics;
 mod proxy;
 mod raw;
@@ -35,7 +36,7 @@ pub use workflow_handle::{
 };
 
 use crate::{
-    metrics::{GrpcMetricSvc, MetricsContext},
+    metrics::{ChannelOrGrpcOverride, GrpcMetricSvc, MetricsContext},
     raw::{AttachMetricLabels, sealed::RawClientLike},
     sealed::WfHandleClient,
     workflow_handle::UntypedWorkflowHandle,
@@ -89,6 +90,8 @@ static TEMPORAL_NAMESPACE_HEADER_KEY: &str = "temporal-namespace";
 
 /// Key used to communicate when a GRPC message is too large
 pub static MESSAGE_TOO_LARGE_KEY: &str = "message-too-large";
+/// Key used to indicate a error was returned by the retryer because of the short-circuit predicate
+pub static ERROR_RETURNED_DUE_TO_SHORT_CIRCUIT: &str = "short-circuit";
 
 /// The server times out polls after 60 seconds. Set our timeout to be slightly beyond that.
 const LONG_POLL_TIMEOUT: Duration = Duration::from_secs(70);
@@ -432,34 +435,59 @@ impl ClientOptions {
         metrics_meter: Option<TemporalMeter>,
     ) -> Result<RetryClient<ConfiguredClient<TemporalServiceClientWithMetrics>>, ClientInitError>
     {
-        let channel = Channel::from_shared(self.target_url.to_string())?;
-        let channel = self.add_tls_to_channel(channel).await?;
-        let channel = if let Some(keep_alive) = self.keep_alive.as_ref() {
-            channel
-                .keep_alive_while_idle(true)
-                .http2_keep_alive_interval(keep_alive.interval)
-                .keep_alive_timeout(keep_alive.timeout)
-        } else {
-            channel
-        };
-        let channel = if let Some(origin) = self.override_origin.clone() {
-            channel.origin(origin)
-        } else {
-            channel
-        };
-        // If there is a proxy, we have to connect that way
-        let channel = if let Some(proxy) = self.http_connect_proxy.as_ref() {
-            proxy.connect_endpoint(&channel).await?
-        } else {
-            channel.connect().await?
-        };
-        let service = ServiceBuilder::new()
-            .layer_fn(move |channel| GrpcMetricSvc {
-                inner: channel,
+        self.connect_no_namespace_with_service_override(metrics_meter, None)
+            .await
+    }
+
+    /// Attempt to establish a connection to the Temporal server and return a gRPC client which is
+    /// intercepted with retry, default headers functionality, and metrics if provided. If a
+    /// service_override is present, network-specific options are ignored and the callback is
+    /// invoked for each gRPC call.
+    ///
+    /// See [RetryClient] for more
+    pub async fn connect_no_namespace_with_service_override(
+        &self,
+        metrics_meter: Option<TemporalMeter>,
+        service_override: Option<callback_based::CallbackBasedGrpcService>,
+    ) -> Result<RetryClient<ConfiguredClient<TemporalServiceClientWithMetrics>>, ClientInitError>
+    {
+        let service = if let Some(service_override) = service_override {
+            GrpcMetricSvc {
+                inner: ChannelOrGrpcOverride::GrpcOverride(service_override),
                 metrics: metrics_meter.clone().map(MetricsContext::new),
                 disable_errcode_label: self.disable_error_code_metric_tags,
-            })
-            .service(channel);
+            }
+        } else {
+            let channel = Channel::from_shared(self.target_url.to_string())?;
+            let channel = self.add_tls_to_channel(channel).await?;
+            let channel = if let Some(keep_alive) = self.keep_alive.as_ref() {
+                channel
+                    .keep_alive_while_idle(true)
+                    .http2_keep_alive_interval(keep_alive.interval)
+                    .keep_alive_timeout(keep_alive.timeout)
+            } else {
+                channel
+            };
+            let channel = if let Some(origin) = self.override_origin.clone() {
+                channel.origin(origin)
+            } else {
+                channel
+            };
+            // If there is a proxy, we have to connect that way
+            let channel = if let Some(proxy) = self.http_connect_proxy.as_ref() {
+                proxy.connect_endpoint(&channel).await?
+            } else {
+                channel.connect().await?
+            };
+            ServiceBuilder::new()
+                .layer_fn(move |channel| GrpcMetricSvc {
+                    inner: ChannelOrGrpcOverride::Channel(channel),
+                    metrics: metrics_meter.clone().map(MetricsContext::new),
+                    disable_errcode_label: self.disable_error_code_metric_tags,
+                })
+                .service(channel)
+        };
+
         let headers = Arc::new(RwLock::new(ClientHeaders {
             user_headers: self.headers.clone().unwrap_or_default(),
             api_key: self.api_key.clone(),
@@ -1140,7 +1168,7 @@ pub struct WorkflowOptions {
 /// The overall semantics of Priority are:
 /// (more will be added here later)
 /// 1. First, consider "priority_key": lower number goes first.
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
+#[derive(Debug, Clone, Default, PartialEq)]
 pub struct Priority {
     /// Priority key is a positive integer from 1 to n, where smaller integers
     /// correspond to higher priorities (tasks run sooner). In general, tasks in
@@ -1153,12 +1181,50 @@ pub struct Priority {
     /// The default priority is (min+max)/2. With the default max of 5 and min of
     /// 1, that comes out to 3.
     pub priority_key: u32,
+
+    /// Fairness key is a short string that's used as a key for a fairness
+    /// balancing mechanism. It may correspond to a tenant id, or to a fixed
+    /// string like "high" or "low". The default is the empty string.
+    ///
+    /// The fairness mechanism attempts to dispatch tasks for a given key in
+    /// proportion to its weight. For example, using a thousand distinct tenant
+    /// ids, each with a weight of 1.0 (the default) will result in each tenant
+    /// getting a roughly equal share of task dispatch throughput.
+    ///
+    /// (Note: this does not imply equal share of worker capacity! Fairness
+    /// decisions are made based on queue statistics, not
+    /// current worker load.)
+    ///
+    /// As another example, using keys "high" and "low" with weight 9.0 and 1.0
+    /// respectively will prefer dispatching "high" tasks over "low" tasks at a
+    /// 9:1 ratio, while allowing either key to use all worker capacity if the
+    /// other is not present.
+    ///
+    /// All fairness mechanisms, including rate limits, are best-effort and
+    /// probabilistic. The results may not match what a "perfect" algorithm with
+    /// infinite resources would produce. The more unique keys are used, the less
+    /// accurate the results will be.
+    ///
+    /// Fairness keys are limited to 64 bytes.
+    pub fairness_key: String,
+
+    /// Fairness weight for a task can come from multiple sources for
+    /// flexibility. From highest to lowest precedence:
+    /// 1. Weights for a small set of keys can be overridden in task queue
+    ///    configuration with an API.
+    /// 2. It can be attached to the workflow/activity in this field.
+    /// 3. The default weight of 1.0 will be used.
+    ///
+    /// Weight values are clamped by the server to the range [0.001, 1000].
+    pub fairness_weight: f32,
 }
 
 impl From<Priority> for common::v1::Priority {
     fn from(priority: Priority) -> Self {
         common::v1::Priority {
             priority_key: priority.priority_key as i32,
+            fairness_key: priority.fairness_key,
+            fairness_weight: priority.fairness_weight,
         }
     }
 }
@@ -1167,6 +1233,8 @@ impl From<common::v1::Priority> for Priority {
     fn from(priority: common::v1::Priority) -> Self {
         Self {
             priority_key: priority.priority_key as u32,
+            fairness_key: priority.fairness_key,
+            fairness_weight: priority.fairness_weight,
         }
     }
 }

package/sdk-core/client/src/metrics.rs

@@ -1,6 +1,9 @@
-use crate::{AttachMetricLabels, CallType, dbg_panic};
+use crate::{AttachMetricLabels, CallType, callback_based, dbg_panic};
+use futures_util::TryFutureExt;
+use futures_util::future::Either;
 use futures_util::{FutureExt, future::BoxFuture};
 use std::{
+    fmt,
     sync::Arc,
     task::{Context, Poll},
     time::{Duration, Instant},
@@ -205,19 +208,37 @@ fn code_as_screaming_snake(code: &Code) -> &'static str {
 /// Implements metrics functionality for gRPC (really, any http) calls
 #[derive(Debug, Clone)]
 pub struct GrpcMetricSvc {
-    pub(crate) inner: Channel,
+    pub(crate) inner: ChannelOrGrpcOverride,
     // If set to none, metrics are a no-op
     pub(crate) metrics: Option<MetricsContext>,
     pub(crate) disable_errcode_label: bool,
 }
 
+#[derive(Clone)]
+pub(crate) enum ChannelOrGrpcOverride {
+    Channel(Channel),
+    GrpcOverride(callback_based::CallbackBasedGrpcService),
+}
+
+impl fmt::Debug for ChannelOrGrpcOverride {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ChannelOrGrpcOverride::Channel(inner) => fmt::Debug::fmt(inner, f),
+            ChannelOrGrpcOverride::GrpcOverride(_) => f.write_str("<callback-based-grpc-service>"),
+        }
+    }
+}
+
 impl Service<http::Request<Body>> for GrpcMetricSvc {
     type Response = http::Response<Body>;
-    type Error = tonic::transport::Error;
+    type Error = Box<dyn std::error::Error + Send + Sync>;
     type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;
 
     fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        self.inner.poll_ready(cx).map_err(Into::into)
+        match &mut self.inner {
+            ChannelOrGrpcOverride::Channel(inner) => inner.poll_ready(cx).map_err(Into::into),
+            ChannelOrGrpcOverride::GrpcOverride(inner) => inner.poll_ready(cx).map_err(Into::into),
+        }
     }
 
     fn call(&mut self, mut req: http::Request<Body>) -> Self::Future {
@@ -245,7 +266,14 @@ impl Service<http::Request<Body>> for GrpcMetricSvc {
                     metrics
                 })
             });
-        let callfut = self.inner.call(req);
+        let callfut = match &mut self.inner {
+            ChannelOrGrpcOverride::Channel(inner) => {
+                Either::Left(inner.call(req).map_err(Into::into))
+            }
+            ChannelOrGrpcOverride::GrpcOverride(inner) => {
+                Either::Right(inner.call(req).map_err(Into::into))
+            }
+        };
         let errcode_label_disabled = self.disable_errcode_label;
         async move {
             let started = Instant::now();

package/sdk-core/client/src/raw.rs

@@ -1354,6 +1354,34 @@ proxier! {
             r.extensions_mut().insert(labels);
         }
     );
+    (
+        update_task_queue_config,
+        UpdateTaskQueueConfigRequest,
+        UpdateTaskQueueConfigResponse,
+        |r| {
+            let mut labels = namespaced_request!(r);
+            labels.task_q_str(r.get_ref().task_queue.clone());
+            r.extensions_mut().insert(labels);
+        }
+    );
+    (
+        fetch_worker_config,
+        FetchWorkerConfigRequest,
+        FetchWorkerConfigResponse,
+        |r| {
+            let labels = namespaced_request!(r);
+            r.extensions_mut().insert(labels);
+        }
+    );
+    (
+        update_worker_config,
+        UpdateWorkerConfigRequest,
+        UpdateWorkerConfigResponse,
+        |r| {
+            let labels = namespaced_request!(r);
+            r.extensions_mut().insert(labels);
+        }
+    );
 }
 
 proxier! {
@@ -1445,6 +1473,11 @@ proxier! {
     (update_namespace_export_sink, cloudreq::UpdateNamespaceExportSinkRequest, cloudreq::UpdateNamespaceExportSinkResponse);
     (delete_namespace_export_sink, cloudreq::DeleteNamespaceExportSinkRequest, cloudreq::DeleteNamespaceExportSinkResponse);
     (validate_namespace_export_sink, cloudreq::ValidateNamespaceExportSinkRequest, cloudreq::ValidateNamespaceExportSinkResponse);
+    (update_namespace_tags, cloudreq::UpdateNamespaceTagsRequest, cloudreq::UpdateNamespaceTagsResponse);
+    (create_connectivity_rule, cloudreq::CreateConnectivityRuleRequest, cloudreq::CreateConnectivityRuleResponse);
+    (get_connectivity_rule, cloudreq::GetConnectivityRuleRequest, cloudreq::GetConnectivityRuleResponse);
+    (get_connectivity_rules, cloudreq::GetConnectivityRulesRequest, cloudreq::GetConnectivityRulesResponse);
+    (delete_connectivity_rule, cloudreq::DeleteConnectivityRuleRequest, cloudreq::DeleteConnectivityRuleResponse);
 }
 
 proxier! {
@@ -1538,11 +1571,17 @@ mod tests {
             })
             .collect();
         let no_underscores: HashSet<_> = impl_list.iter().map(|x| x.replace('_', "")).collect();
+        let mut not_implemented = vec![];
         for method in methods {
             if !no_underscores.contains(&method.to_lowercase()) {
-                panic!("RPC method {method} is not implemented by raw client")
+                not_implemented.push(method);
             }
         }
+        if !not_implemented.is_empty() {
+            panic!(
+                "The following RPC methods are not implemented by raw client: {not_implemented:?}"
+            );
+        }
     }
     #[test]
     fn verify_all_workflow_service_methods_implemented() {

package/sdk-core/client/src/retry.rs

@@ -1,6 +1,6 @@
 use crate::{
-    Client, IsWorkerTaskLongPoll, MESSAGE_TOO_LARGE_KEY, NamespacedClient, NoRetryOnMatching,
-    Result, RetryConfig, raw::IsUserLongPoll,
+    Client, ERROR_RETURNED_DUE_TO_SHORT_CIRCUIT, IsWorkerTaskLongPoll, MESSAGE_TOO_LARGE_KEY,
+    NamespacedClient, NoRetryOnMatching, Result, RetryConfig, raw::IsUserLongPoll,
 };
 use backoff::{Clock, SystemClock, backoff::Backoff, exponential::ExponentialBackoff};
 use futures_retry::{ErrorHandler, FutureRetry, RetryPolicy};
@@ -214,6 +214,10 @@ where
         if let Some(sc) = self.retry_short_circuit.as_ref()
             && (sc.predicate)(&e)
         {
+            e.metadata_mut().insert(
+                ERROR_RETURNED_DUE_TO_SHORT_CIRCUIT,
+                tonic::metadata::MetadataValue::from(0),
+            );
             return RetryPolicy::ForwardError(e);
         }
 
@@ -441,7 +445,12 @@ mod tests {
             FixedClock(Instant::now()),
         );
         let result = err_handler.handle(1, Status::new(Code::ResourceExhausted, "leave me alone"));
-        assert_matches!(result, RetryPolicy::ForwardError(_))
+        let e = assert_matches!(result, RetryPolicy::ForwardError(e) => e);
+        assert!(
+            e.metadata()
+                .get(ERROR_RETURNED_DUE_TO_SHORT_CIRCUIT)
+                .is_some()
+        );
     }
 
     #[tokio::test]

package/sdk-core/core/src/lib.rs

@@ -103,7 +103,9 @@ where
         bail!("Client identity cannot be empty. Either lang or user should be setting this value");
     }
 
-    let heartbeat_fn = Arc::new(OnceLock::new());
+    let heartbeat_fn = worker_config
+        .heartbeat_interval
+        .map(|_| Arc::new(OnceLock::new()));
 
     let client_bag = Arc::new(WorkerClientBag::new(
         client,
@@ -118,7 +120,7 @@ where
         sticky_q,
         client_bag,
         Some(&runtime.telemetry),
-        Some(heartbeat_fn),
+        heartbeat_fn,
     ))
 }
 

package/sdk-core/core/src/pollers/poll_buffer.rs

@@ -18,7 +18,7 @@ use std::{
     },
     time::Duration,
 };
-use temporal_client::NoRetryOnMatching;
+use temporal_client::{ERROR_RETURNED_DUE_TO_SHORT_CIRCUIT, NoRetryOnMatching};
 use temporal_sdk_core_api::worker::{
     ActivitySlotKind, NexusSlotKind, PollerBehavior, SlotKind, WorkflowSlotKind,
 };
@@ -538,20 +538,27 @@ impl PollScalerReportHandle {
                 }
             }
             Err(e) => {
-                // We should only see (and react to) errors in autoscaling mode
-                if matches!(self.behavior, PollerBehavior::Autoscaling { .. })
-                    && self.ever_saw_scaling_decision.load(Ordering::Relaxed)
-                {
-                    debug!("Got error from server while polling: {:?}", e);
-                    if e.code() == Code::ResourceExhausted {
-                        // Scale down significantly for resource exhaustion
-                        self.change_target(usize::saturating_div, 2);
-                    } else {
-                        // Other codes that would normally have made us back off briefly can
-                        // reclaim this poller
-                        self.change_target(usize::saturating_sub, 1);
+                if matches!(self.behavior, PollerBehavior::Autoscaling { .. }) {
+                    // We should only react to errors in autoscaling mode if we saw a scaling
+                    // decision
+                    if self.ever_saw_scaling_decision.load(Ordering::Relaxed) {
+                        debug!("Got error from server while polling: {:?}", e);
+                        if e.code() == Code::ResourceExhausted {
+                            // Scale down significantly for resource exhaustion
+                            self.change_target(usize::saturating_div, 2);
+                        } else {
+                            // Other codes that would normally have made us back off briefly can
+                            // reclaim this poller
+                            self.change_target(usize::saturating_sub, 1);
+                        }
                     }
-                    return false;
+                    // Only propagate errors out if they weren't because of the short-circuiting
+                    // logic. IE: We don't want to fail callers because we said we wanted to know
+                    // about ResourceExhausted errors, but we haven't seen a scaling decision yet,
+                    // so we're not reacting to errors, only propagating them.
+                    return !e
+                        .metadata()
+                        .contains_key(ERROR_RETURNED_DUE_TO_SHORT_CIRCUIT);
                 }
             }
         }
@@ -748,4 +755,45 @@ mod tests {
         pb.poll().await.unwrap().unwrap();
         pb.shutdown().await;
     }
+
+    #[tokio::test]
+    async fn autoscale_wont_fail_caller_on_short_circuited_error() {
+        let mut mock_client = mock_manual_worker_client();
+        mock_client
+            .expect_poll_workflow_task()
+            .times(1)
+            .returning(move |_, _| {
+                async {
+                    let mut st = tonic::Status::cancelled("whatever");
+                    st.metadata_mut()
+                        .insert(ERROR_RETURNED_DUE_TO_SHORT_CIRCUIT, 1.into());
+                    Err(st)
+                }
+                .boxed()
+            });
+        mock_client
+            .expect_poll_workflow_task()
+            .returning(move |_, _| async { Ok(Default::default()) }.boxed());
+
+        let pb = LongPollBuffer::new_workflow_task(
+            Arc::new(mock_client),
+            "sometq".to_string(),
+            None,
+            PollerBehavior::Autoscaling {
+                minimum: 1,
+                maximum: 1,
+                initial: 1,
+            },
+            fixed_size_permit_dealer(1),
+            CancellationToken::new(),
+            None::<fn(usize)>,
+            WorkflowTaskOptions {
+                wft_poller_shared: Some(Arc::new(WFTPollerShared::new(Some(1)))),
+            },
+        );
+
+        // Should not see error, unwraps should get empty response
+        pb.poll().await.unwrap().unwrap();
+        pb.shutdown().await;
+    }
 }

package/sdk-core/core/src/worker/client.rs

@@ -50,7 +50,7 @@ pub(crate) struct WorkerClientBag {
     namespace: String,
     identity: String,
     worker_versioning_strategy: WorkerVersioningStrategy,
-    heartbeat_data: Arc<OnceLock<HeartbeatFn>>,
+    heartbeat_data: Option<Arc<OnceLock<HeartbeatFn>>>,
 }
 
 impl WorkerClientBag {
@@ -59,7 +59,7 @@ impl WorkerClientBag {
         namespace: String,
         identity: String,
         worker_versioning_strategy: WorkerVersioningStrategy,
-        heartbeat_data: Arc<OnceLock<HeartbeatFn>>,
+        heartbeat_data: Option<Arc<OnceLock<HeartbeatFn>>>,
     ) -> Self {
         Self {
             replaceable_client: RwLock::new(client),
@@ -129,10 +129,14 @@ impl WorkerClientBag {
     }
 
     fn capture_heartbeat(&self) -> Option<WorkerHeartbeat> {
-        if let Some(hb) = self.heartbeat_data.get() {
-            hb()
+        if let Some(heartbeat_data) = self.heartbeat_data.as_ref() {
+            if let Some(hb) = heartbeat_data.get() {
+                hb()
+            } else {
+                dbg_panic!("Heartbeat function never set");
+                None
+            }
         } else {
-            dbg_panic!("Heartbeat function never set");
             None
         }
     }