@temporalio/core-bridge 1.11.2 → 1.11.4

package/sdk-core/core/src/lib.rs

@@ -12,6 +12,8 @@ extern crate tracing;
 extern crate core;
 
 mod abstractions;
+#[cfg(feature = "debug-plugin")]
+pub mod debug_client;
 #[cfg(feature = "ephemeral-server")]
 pub mod ephemeral_server;
 mod internal_flags;
@@ -54,7 +56,7 @@ use crate::{
     worker::client::WorkerClientBag,
 };
 use anyhow::bail;
-use futures::Stream;
+use futures_util::Stream;
 use std::sync::Arc;
 use temporal_client::{ConfiguredClient, TemporalServiceClientWithMetrics};
 use temporal_sdk_core_api::{
@@ -198,6 +200,23 @@ pub struct CoreRuntime {
     runtime_handle: tokio::runtime::Handle,
 }
 
+/// Wraps a [tokio::runtime::Builder] to allow layering multiple on_thread_start functions
+pub struct TokioRuntimeBuilder<F> {
+    /// The underlying tokio runtime builder
+    pub inner: tokio::runtime::Builder,
+    /// A function to be called when setting the runtime builder's on thread start
+    pub lang_on_thread_start: Option<F>,
+}
+
+impl Default for TokioRuntimeBuilder<Box<dyn Fn() + Send + Sync>> {
+    fn default() -> Self {
+        TokioRuntimeBuilder {
+            inner: tokio::runtime::Builder::new_multi_thread(),
+            lang_on_thread_start: None,
+        }
+    }
+}
+
 impl CoreRuntime {
     /// Create a new core runtime with the provided telemetry options and tokio runtime builder.
     /// Also initialize telemetry for the thread this is being called on.
@@ -212,18 +231,25 @@ impl CoreRuntime {
     /// # Panics
     /// If a tokio runtime has already been initialized. To re-use an existing runtime, call
     /// [CoreRuntime::new_assume_tokio].
-    pub fn new(
+    pub fn new<F>(
         telemetry_options: TelemetryOptions,
-        mut tokio_builder: tokio::runtime::Builder,
-    ) -> Result<Self, anyhow::Error> {
+        mut tokio_builder: TokioRuntimeBuilder<F>,
+    ) -> Result<Self, anyhow::Error>
+    where
+        F: Fn() + Send + Sync + 'static,
+    {
         let telemetry = telemetry_init(telemetry_options)?;
         let subscriber = telemetry.trace_subscriber();
         let runtime = tokio_builder
+            .inner
             .enable_all()
             .on_thread_start(move || {
                 if let Some(sub) = subscriber.as_ref() {
                     set_trace_subscriber_for_current_thread(sub.clone());
                 }
+                if let Some(lang_on_thread_start) = tokio_builder.lang_on_thread_start.as_ref() {
+                    lang_on_thread_start();
+                }
             })
             .build()?;
         let _rg = runtime.enter();

package/sdk-core/core/src/pollers/mod.rs

@@ -14,7 +14,7 @@ use temporal_sdk_core_protos::temporal::api::workflowservice::v1::{
 };
 
 #[cfg(test)]
-use futures::Future;
+use futures_util::Future;
 #[cfg(test)]
 pub(crate) use poll_buffer::MockPermittedPollBuffer;
 use temporal_sdk_core_api::worker::{ActivitySlotKind, WorkflowSlotKind};

package/sdk-core/core/src/pollers/poll_buffer.rs

@@ -3,8 +3,7 @@ use crate::{
     pollers::{self, Poller},
     worker::client::WorkerClient,
 };
-use futures::{prelude::stream::FuturesUnordered, StreamExt};
-use futures_util::{future::BoxFuture, FutureExt};
+use futures_util::{future::BoxFuture, stream::FuturesUnordered, FutureExt, StreamExt};
 use governor::{Quota, RateLimiter};
 use std::{
     fmt::Debug,
@@ -73,7 +72,7 @@ where
 {
     pub(crate) fn new<FT, DelayFut>(
         poll_fn: impl Fn() -> FT + Send + Sync + 'static,
-        permit_dealer: Arc<MeteredPermitDealer<SK>>,
+        permit_dealer: MeteredPermitDealer<SK>,
         max_pollers: usize,
         shutdown: CancellationToken,
         num_pollers_handler: Option<impl Fn(usize) + Send + Sync + 'static>,
@@ -85,6 +84,7 @@ where
     {
         let (tx, rx) = unbounded_channel();
         let (starter, wait_for_start) = broadcast::channel(1);
+        let permit_dealer = Arc::new(permit_dealer);
         let active_pollers = Arc::new(AtomicUsize::new(0));
         let join_handles = FuturesUnordered::new();
         let pf = Arc::new(poll_fn);
@@ -243,7 +243,7 @@ pub(crate) fn new_workflow_task_buffer(
     client: Arc<dyn WorkerClient>,
     task_queue: TaskQueue,
     concurrent_pollers: usize,
-    permit_dealer: Arc<MeteredPermitDealer<WorkflowSlotKind>>,
+    permit_dealer: MeteredPermitDealer<WorkflowSlotKind>,
     shutdown: CancellationToken,
     num_pollers_handler: Option<impl Fn(usize) + Send + Sync + 'static>,
 ) -> PollWorkflowTaskBuffer {
@@ -268,7 +268,7 @@ pub(crate) fn new_activity_task_buffer(
     client: Arc<dyn WorkerClient>,
     task_queue: String,
     concurrent_pollers: usize,
-    semaphore: Arc<MeteredPermitDealer<ActivitySlotKind>>,
+    semaphore: MeteredPermitDealer<ActivitySlotKind>,
     max_tps: Option<f64>,
     shutdown: CancellationToken,
     num_pollers_handler: Option<impl Fn(usize) + Send + Sync + 'static>,
@@ -337,7 +337,7 @@ mod tests {
         abstractions::tests::fixed_size_permit_dealer,
         worker::client::mocks::mock_manual_workflow_client,
     };
-    use futures::FutureExt;
+    use futures_util::FutureExt;
     use std::time::Duration;
     use temporal_sdk_core_protos::temporal::api::enums::v1::TaskQueueKind;
     use tokio::{select, sync::mpsc::channel};
@@ -364,7 +364,7 @@ mod tests {
                 normal_name: "".to_string(),
             },
             1,
-            Arc::new(fixed_size_permit_dealer(10)),
+            fixed_size_permit_dealer(10),
             CancellationToken::new(),
             None::<fn(usize)>,
         );

package/sdk-core/core/src/replay/mod.rs

@@ -9,9 +9,9 @@ use crate::{
     },
     Worker,
 };
-use futures::{FutureExt, Stream, StreamExt};
-use once_cell::sync::OnceCell;
+use futures_util::{FutureExt, Stream, StreamExt};
 use parking_lot::Mutex;
+use std::sync::OnceLock;
 use std::{
     pin::Pin,
     sync::Arc,
@@ -179,7 +179,7 @@ impl Stream for HistoryFeederStream {
 pub(crate) struct Historator {
     iter: Pin<Box<dyn Stream<Item = HistoryForReplay> + Send>>,
     allow_stream: UnboundedReceiverStream<String>,
-    worker_closer: Arc<OnceCell<CancellationToken>>,
+    worker_closer: Arc<OnceLock<CancellationToken>>,
     dat: Arc<Mutex<HistoratorDat>>,
     replay_done_tx: UnboundedSender<String>,
 }
@@ -192,7 +192,7 @@ impl Historator {
         Self {
             iter: Box::pin(histories.fuse()),
             allow_stream: UnboundedReceiverStream::new(replay_done_rx),
-            worker_closer: Arc::new(OnceCell::new()),
+            worker_closer: Arc::new(OnceLock::new()),
             dat,
             replay_done_tx,
         }

package/sdk-core/core/src/telemetry/log_export.rs

@@ -1,4 +1,4 @@
-use futures::channel::mpsc::{channel, Receiver, Sender};
+use futures_channel::mpsc::{channel, Receiver, Sender};
 use parking_lot::Mutex;
 use ringbuf::{consumer::Consumer, producer::Producer, traits::Split, HeapRb};
 use std::{collections::HashMap, fmt, sync::Arc, time::SystemTime};
@@ -219,7 +219,7 @@ mod tests {
         telemetry::{construct_filter_string, CoreLogStreamConsumer},
         telemetry_init,
     };
-    use futures::stream::StreamExt;
+    use futures_util::stream::StreamExt;
     use std::{
         fmt,
         sync::{Arc, Mutex},

package/sdk-core/core/src/telemetry/metrics.rs

@@ -42,6 +42,12 @@ struct Instruments {
     act_execution_failed: Arc<dyn Counter>,
     act_sched_to_start_latency: Arc<dyn HistogramDuration>,
     act_exec_latency: Arc<dyn HistogramDuration>,
+    act_exec_succeeded_latency: Arc<dyn HistogramDuration>,
+    la_execution_cancelled: Arc<dyn Counter>,
+    la_execution_failed: Arc<dyn Counter>,
+    la_exec_latency: Arc<dyn HistogramDuration>,
+    la_exec_succeeded_latency: Arc<dyn HistogramDuration>,
+    la_total: Arc<dyn Counter>,
     worker_registered: Arc<dyn Counter>,
     num_pollers: Arc<dyn Gauge>,
     task_slots_available: Arc<dyn Gauge>,
@@ -177,6 +183,13 @@ impl MetricsContext {
         self.instruments.act_execution_failed.add(1, &self.kvs);
     }
 
+    /// Record end-to-end (sched-to-complete) time for successful activity executions
+    pub(crate) fn act_execution_succeeded(&self, dur: Duration) {
+        self.instruments
+            .act_exec_succeeded_latency
+            .record(dur, &self.kvs);
+    }
+
     /// Record activity task schedule to start time in millis
     pub(crate) fn act_sched_to_start_latency(&self, dur: Duration) {
         self.instruments
@@ -190,6 +203,28 @@ impl MetricsContext {
         self.instruments.act_exec_latency.record(dur, &self.kvs);
     }
 
+    pub(crate) fn la_execution_cancelled(&self) {
+        self.instruments.la_execution_cancelled.add(1, &self.kvs);
+    }
+
+    pub(crate) fn la_execution_failed(&self) {
+        self.instruments.la_execution_failed.add(1, &self.kvs);
+    }
+
+    pub(crate) fn la_exec_latency(&self, dur: Duration) {
+        self.instruments.la_exec_latency.record(dur, &self.kvs);
+    }
+
+    pub(crate) fn la_exec_succeeded_latency(&self, dur: Duration) {
+        self.instruments
+            .la_exec_succeeded_latency
+            .record(dur, &self.kvs);
+    }
+
+    pub(crate) fn la_executed(&self) {
+        self.instruments.la_total.add(1, &self.kvs);
+    }
+
     /// A worker was registered
     pub(crate) fn worker_registered(&self) {
         self.instruments.worker_registered.add(1, &self.kvs);
@@ -316,6 +351,39 @@ impl Instruments {
                 unit: "duration".into(),
                 description: "Histogram of activity execution latencies".into(),
             }),
+            act_exec_succeeded_latency: meter.histogram_duration(MetricParameters {
+                name: "activity_succeed_endtoend_latency".into(),
+                unit: "duration".into(),
+                description: "Histogram of activity execution latencies for successful activities"
+                    .into(),
+            }),
+            la_execution_cancelled: meter.counter(MetricParameters {
+                name: "local_activity_execution_cancelled".into(),
+                description: "Count of local activity executions that were cancelled".into(),
+                unit: "".into(),
+            }),
+            la_execution_failed: meter.counter(MetricParameters {
+                name: "local_activity_execution_failed".into(),
+                description: "Count of local activity executions that failed".into(),
+                unit: "".into(),
+            }),
+            la_exec_latency: meter.histogram_duration(MetricParameters {
+                name: "local_activity_execution_latency".into(),
+                unit: "duration".into(),
+                description: "Histogram of local activity execution latencies".into(),
+            }),
+            la_exec_succeeded_latency: meter.histogram_duration(MetricParameters {
+                name: "local_activity_succeed_endtoend_latency".into(),
+                unit: "duration".into(),
+                description:
+                    "Histogram of local activity execution latencies for successful local activities"
+                        .into(),
+            }),
+            la_total: meter.counter(MetricParameters {
+                name: "local_activity_total".into(),
+                description: "Count of local activities executed".into(),
+                unit: "".into(),
+            }),
             // name kept as worker start for compat with old sdk / what users expect
             worker_registered: meter.counter(MetricParameters {
                 name: "worker_start".into(),
@@ -807,7 +875,7 @@ mod tests {
         a1.set(Arc::new(DummyCustomAttrs(1))).unwrap();
         // Verify all metrics are created. This number will need to get updated any time a metric
         // is added.
-        let num_metrics = 24;
+        let num_metrics = 30;
         #[allow(clippy::needless_range_loop)] // Sorry clippy, this reads easier.
         for metric_num in 1..=num_metrics {
             let hole = assert_matches!(&events[metric_num],

package/sdk-core/core/src/telemetry/otel.rs

@@ -335,9 +335,9 @@ impl GaugeF64 for MemoryGauge<f64> {
 }
 
 fn default_resource_instance() -> &'static Resource {
-    use once_cell::sync::OnceCell;
+    use std::sync::OnceLock;
 
-    static INSTANCE: OnceCell<Resource> = OnceCell::new();
+    static INSTANCE: OnceLock<Resource> = OnceLock::new();
     INSTANCE.get_or_init(|| {
         let resource = Resource::default();
         if resource.get(Key::from("service.name")) == Some(Value::from("unknown_service")) {

package/sdk-core/core/src/test_help/mod.rs

@@ -15,7 +15,7 @@ use crate::{
 };
 use async_trait::async_trait;
 use bimap::BiMap;
-use futures::{future::BoxFuture, stream, stream::BoxStream, FutureExt, Stream, StreamExt};
+use futures_util::{future::BoxFuture, stream, stream::BoxStream, FutureExt, Stream, StreamExt};
 use mockall::TimesRange;
 use parking_lot::RwLock;
 use std::{
@@ -1014,9 +1014,9 @@ macro_rules! job_assert {
 #[macro_export]
 macro_rules! advance_fut {
     ($fut:ident) => {
-        ::futures::pin_mut!($fut);
+        ::futures_util::pin_mut!($fut);
         {
-            let waker = ::futures::task::noop_waker();
+            let waker = ::futures_util::task::noop_waker();
             let mut cx = core::task::Context::from_waker(&waker);
             for _ in 0..10 {
                 assert_matches!($fut.poll_unpin(&mut cx), core::task::Poll::Pending);

package/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs

@@ -3,7 +3,7 @@ use crate::{
     worker::{activities::PendingActivityCancel, client::WorkerClient},
     TaskToken,
 };
-use futures::StreamExt;
+use futures_util::StreamExt;
 use std::{
     collections::{hash_map::Entry, HashMap},
     sync::Arc,
@@ -94,7 +94,7 @@ impl ActivityHeartbeatManager {
             // The stream of incoming heartbeats uses unfold to carry state across each item in the
             // stream. The closure checks if, for any given activity, we should heartbeat or not
             // depending on its delay and when we last issued a heartbeat for it.
-            futures::stream::unfold(heartbeat_stream_state, move |mut hb_states| {
+            futures_util::stream::unfold(heartbeat_stream_state, move |mut hb_states| {
                 async move {
                     let hb = tokio::select! {
                         biased;
@@ -140,7 +140,7 @@ impl ActivityHeartbeatManager {
                                     .record_activity_heartbeat(tt.clone(), details.into_payloads())
                                     .await
                                 {
-                                    Ok(RecordActivityTaskHeartbeatResponse { cancel_requested }) => {
+                                    Ok(RecordActivityTaskHeartbeatResponse { cancel_requested, activity_paused: _ }) => {
                                         if cancel_requested {
                                             cancels_tx
                                                 .send(PendingActivityCancel::new(

package/sdk-core/core/src/worker/activities/activity_task_poller_stream.rs

@@ -1,5 +1,5 @@
 use crate::{pollers::BoxedActPoller, worker::activities::PermittedTqResp, MetricsContext};
-use futures::{stream, Stream};
+use futures_util::{stream, Stream};
 use temporal_sdk_core_protos::temporal::api::workflowservice::v1::PollActivityTaskQueueResponse;
 use tokio::select;
 use tokio_util::sync::CancellationToken;

package/sdk-core/core/src/worker/activities/local_activities.rs

@@ -2,25 +2,27 @@ use crate::{
     abstractions::{dbg_panic, MeteredPermitDealer, OwnedMeteredSemPermit, UsedMeteredSemPermit},
     protosext::ValidScheduleLA,
     retry_logic::RetryPolicyExt,
+    telemetry::metrics::{activity_type, workflow_type},
     worker::workflow::HeartbeatTimeoutMsg,
     MetricsContext, TaskToken,
 };
-use futures::{stream::BoxStream, Stream};
-use futures_util::{future, future::AbortRegistration, stream, StreamExt};
+use futures_util::{
+    future, future::AbortRegistration, stream, stream::BoxStream, Stream, StreamExt,
+};
 use parking_lot::{Mutex, MutexGuard};
 use std::{
     collections::{hash_map::Entry, HashMap},
     fmt::{Debug, Formatter},
     pin::Pin,
-    sync::Arc,
     task::{Context, Poll},
     time::{Duration, Instant, SystemTime},
 };
-use temporal_sdk_core_api::worker::{LocalActivitySlotInfo, LocalActivitySlotKind, SlotSupplier};
+use temporal_sdk_core_api::worker::LocalActivitySlotKind;
 use temporal_sdk_core_protos::{
     coresdk::{
         activity_result::{Cancellation, Failure as ActFail, Success},
         activity_task::{activity_task, ActivityCancelReason, ActivityTask, Cancel, Start},
+        LocalActivitySlotInfo,
     },
     temporal::api::{
         common::v1::WorkflowExecution,
@@ -173,6 +175,9 @@ pub(crate) struct LocalActivityManager {
     rcvs: tokio::sync::Mutex<RcvChans>,
     shutdown_complete_tok: CancellationToken,
     dat: Mutex<LAMData>,
+    /// Note that these metrics do *not* include the `worker_type` label, as every metric
+    /// emitted here is already specific to local activities via the metric name.
+    metrics: MetricsContext,
 }
 
 struct LocalActivityInfo {
@@ -205,20 +210,19 @@ impl LAMData {
 
 impl LocalActivityManager {
     pub(crate) fn new(
-        slot_supplier: Arc<dyn SlotSupplier<SlotKind = LocalActivitySlotKind> + Send + Sync>,
         namespace: String,
+        permit_dealer: MeteredPermitDealer<LocalActivitySlotKind>,
         heartbeat_timeout_tx: UnboundedSender<HeartbeatTimeoutMsg>,
         metrics_context: MetricsContext,
     ) -> Self {
         let (act_req_tx, act_req_rx) = unbounded_channel();
         let (cancels_req_tx, cancels_req_rx) = unbounded_channel();
         let shutdown_complete_tok = CancellationToken::new();
-        let semaphore = MeteredPermitDealer::new(slot_supplier, metrics_context, None);
         Self {
             namespace,
             rcvs: tokio::sync::Mutex::new(RcvChans::new(
                 act_req_rx,
-                semaphore,
+                permit_dealer,
                 cancels_req_rx,
                 shutdown_complete_tok.clone(),
             )),
@@ -233,16 +237,28 @@ impl LocalActivityManager {
                 next_tt_num: 0,
             }),
             workflows_have_shut_down: Default::default(),
+            metrics: metrics_context,
         }
     }
 
     #[cfg(test)]
     fn test(max_concurrent: usize) -> Self {
         use crate::worker::tuner::FixedSizeSlotSupplier;
+        use std::sync::Arc;
 
         let ss = Arc::new(FixedSizeSlotSupplier::new(max_concurrent));
         let (hb_tx, _hb_rx) = unbounded_channel();
-        Self::new(ss, "fake_ns".to_string(), hb_tx, MetricsContext::no_op())
+        Self::new(
+            "fake_ns".to_string(),
+            MeteredPermitDealer::new(
+                ss,
+                MetricsContext::no_op(),
+                None,
+                Arc::new(Default::default()),
+            ),
+            hb_tx,
+            MetricsContext::no_op(),
+        )
     }
 
     #[cfg(test)]
@@ -473,12 +489,18 @@ impl LocalActivityManager {
                 dispatch_time: Instant::now(),
                 attempt,
                 _permit: permit.into_used(LocalActivitySlotInfo {
-                    activity_type: new_la.workflow_type.as_str(),
+                    activity_type: new_la.workflow_type.clone(),
                 }),
             },
         );
 
         let (schedule_to_close, start_to_close) = sa.close_timeouts.into_sched_and_start();
+        self.metrics
+            .with_new_attrs([
+                activity_type(sa.activity_type.clone()),
+                workflow_type(new_la.workflow_type.clone()),
+            ])
+            .la_executed();
         Some(NextPendingLAAction::Dispatch(ActivityTask {
             task_token: tt.0,
             variant: Some(activity_task::Variant::Start(Start {
@@ -500,9 +522,7 @@ impl LocalActivityManager {
                     .ok(),
                 start_to_close_timeout: start_to_close
                     .or(schedule_to_close)
-                    .unwrap()
-                    .try_into()
-                    .ok(),
+                    .and_then(|t| t.try_into().ok()),
                 heartbeat_timeout: None,
                 retry_policy: Some(sa.retry_policy),
                 is_local: true,
@@ -536,6 +556,11 @@ impl LocalActivityManager {
                 }
             }
 
+            let la_metrics = self.metrics.with_new_attrs([
+                activity_type(info.la_info.schedule_cmd.activity_type.clone()),
+                workflow_type(info.la_info.workflow_type.clone()),
+            ]);
+
             enum Outcome {
                 FailurePath { backoff: Option<Duration> },
                 JustReport,
@@ -553,31 +578,48 @@ impl LocalActivityManager {
             }
 
             let mut is_timeout = false;
+            let runtime = info.dispatch_time.elapsed();
+            la_metrics.la_exec_latency(runtime);
             let outcome = match &status {
-                LocalActivityExecutionResult::Failed(fail) => Outcome::FailurePath {
-                    backoff: calc_backoff!(fail),
-                },
-                LocalActivityExecutionResult::TimedOut(fail)
-                    if matches!(status.get_timeout_type(), Some(TimeoutType::StartToClose)) =>
-                {
-                    // Start to close timeouts are retryable, other timeout types aren't.
-                    is_timeout = true;
+                LocalActivityExecutionResult::Failed(fail) => {
+                    la_metrics.la_execution_failed();
                     Outcome::FailurePath {
                         backoff: calc_backoff!(fail),
                     }
                 }
-                LocalActivityExecutionResult::TimedOut(_) => {
+                LocalActivityExecutionResult::TimedOut(fail) => {
+                    la_metrics.la_execution_failed();
                     is_timeout = true;
+                    // Start to close timeouts are retryable, other timeout types aren't.
+                    if matches!(status.get_timeout_type(), Some(TimeoutType::StartToClose)) {
+                        Outcome::FailurePath {
+                            backoff: calc_backoff!(fail),
+                        }
+                    } else {
+                        Outcome::JustReport
+                    }
+                }
+                LocalActivityExecutionResult::Completed(_) => {
+                    if let Some(rt) = info
+                        .la_info
+                        .schedule_cmd
+                        .original_schedule_time
+                        .and_then(|t| t.elapsed().ok())
+                    {
+                        la_metrics.la_exec_succeeded_latency(rt);
+                    }
+                    Outcome::JustReport
+                }
+                LocalActivityExecutionResult::Cancelled { .. } => {
+                    la_metrics.la_execution_cancelled();
                     Outcome::JustReport
                 }
-                LocalActivityExecutionResult::Completed(_)
-                | LocalActivityExecutionResult::Cancelled { .. } => Outcome::JustReport,
             };
 
             let mut resolution = LocalActivityResolution {
                 seq: info.la_info.schedule_cmd.seq,
                 result: status,
-                runtime: info.dispatch_time.elapsed(),
+                runtime,
                 attempt: info.attempt,
                 backoff: None,
                 original_schedule_time: info.la_info.schedule_cmd.original_schedule_time,
@@ -692,6 +734,8 @@ impl LocalActivityManager {
         while !self.set_shutdown_complete_if_ready(&mut self.dat.lock()) {
             self.complete_notify.notified().await;
         }
+        // This makes sure we drop any permits that might be held inside the stream
+        self.rcvs.lock().await.inner = stream::empty().boxed();
     }
 
     /// Try to close the activity stream as soon as worker shutdown is initiated. This is required