@temporalio/core-bridge 1.13.1 → 1.14.0

package/sdk-core/{core → crates/sdk-core}/src/telemetry/metrics.rs

@@ -1,4 +1,6 @@
-use crate::{abstractions::dbg_panic, telemetry::TelemetryInstance};
+#[cfg(test)]
+use crate::TelemetryInstance;
+use crate::abstractions::dbg_panic;
 
 use std::{
     fmt::{Debug, Display},
@@ -6,15 +8,15 @@ use std::{
     sync::Arc,
     time::Duration,
 };
-use temporal_sdk_core_api::telemetry::metrics::{
-    BufferAttributes, BufferInstrumentRef, CoreMeter, Counter, CounterBase, Gauge, GaugeBase,
-    GaugeF64, GaugeF64Base, Histogram, HistogramBase, HistogramDuration, HistogramDurationBase,
-    HistogramF64, HistogramF64Base, LazyBufferInstrument, MetricAttributable, MetricAttributes,
-    MetricCallBufferer, MetricEvent, MetricKeyValue, MetricKind, MetricParameters, MetricUpdateVal,
-    NewAttributes, NoOpCoreMeter,
-};
-use temporal_sdk_core_protos::temporal::api::{
-    enums::v1::WorkflowTaskFailedCause, failure::v1::Failure,
+use temporalio_common::{
+    protos::temporal::api::{enums::v1::WorkflowTaskFailedCause, failure::v1::Failure},
+    telemetry::metrics::{
+        BufferAttributes, BufferInstrumentRef, CoreMeter, Counter, CounterBase, Gauge, GaugeBase,
+        GaugeF64, GaugeF64Base, Histogram, HistogramBase, HistogramDuration, HistogramDurationBase,
+        HistogramF64, HistogramF64Base, LazyBufferInstrument, MetricAttributable, MetricAttributes,
+        MetricCallBufferer, MetricEvent, MetricKeyValue, MetricKind, MetricParameters,
+        MetricUpdateVal, NewAttributes, NoOpCoreMeter, TemporalMeter, WorkerHeartbeatMetrics,
+    },
 };
 
 /// Used to track context associated with metrics, and record/update them
@@ -23,6 +25,7 @@ pub(crate) struct MetricsContext {
     meter: Arc<dyn CoreMeter>,
     kvs: MetricAttributes,
     instruments: Arc<Instruments>,
+    in_memory_metrics: Option<Arc<WorkerHeartbeatMetrics>>,
 }
 
 #[derive(Clone)]
@@ -68,28 +71,41 @@ impl MetricsContext {
     pub(crate) fn no_op() -> Self {
         let meter = Arc::new(NoOpCoreMeter);
         let kvs = meter.new_attributes(Default::default());
-        let instruments = Arc::new(Instruments::new(meter.as_ref()));
+        let in_memory_metrics = Some(Arc::new(WorkerHeartbeatMetrics::default()));
+        let instruments = Arc::new(Instruments::new(meter.as_ref(), in_memory_metrics.clone()));
         Self {
             kvs,
             instruments,
             meter,
+            in_memory_metrics,
         }
     }
 
+    #[cfg(test)]
     pub(crate) fn top_level(namespace: String, tq: String, telemetry: &TelemetryInstance) -> Self {
-        if let Some(mut meter) = telemetry.get_temporal_metric_meter() {
+        MetricsContext::top_level_with_meter(namespace, tq, telemetry.get_temporal_metric_meter())
+    }
+
+    pub(crate) fn top_level_with_meter(
+        namespace: String,
+        tq: String,
+        temporal_meter: Option<TemporalMeter>,
+    ) -> Self {
+        if let Some(mut meter) = temporal_meter {
             meter
                 .default_attribs
                 .attributes
                 .push(MetricKeyValue::new(KEY_NAMESPACE, namespace));
             meter.default_attribs.attributes.push(task_queue(tq));
             let kvs = meter.inner.new_attributes(meter.default_attribs);
-            let mut instruments = Instruments::new(meter.inner.as_ref());
+            let in_memory_metrics = Some(Arc::new(WorkerHeartbeatMetrics::default()));
+            let mut instruments = Instruments::new(meter.inner.as_ref(), in_memory_metrics.clone());
             instruments.update_attributes(&kvs);
             Self {
                 kvs,
                 instruments: Arc::new(instruments),
                 meter: meter.inner,
+                in_memory_metrics,
             }
         } else {
             Self::no_op()
@@ -110,9 +126,14 @@ impl MetricsContext {
             instruments: Arc::new(instruments),
             kvs,
             meter: self.meter.clone(),
+            in_memory_metrics: self.in_memory_metrics.clone(),
         }
     }
 
+    pub(crate) fn in_memory_meter(&self) -> Option<Arc<WorkerHeartbeatMetrics>> {
+        self.in_memory_metrics.clone()
+    }
+
     /// A workflow task queue poll succeeded
     pub(crate) fn wf_tq_poll_ok(&self) {
         self.instruments.wf_task_queue_poll_succeed_counter.adds(1);
@@ -288,7 +309,31 @@ impl MetricsContext {
 }
 
 impl Instruments {
-    fn new(meter: &dyn CoreMeter) -> Self {
+    fn new(meter: &dyn CoreMeter, in_memory: Option<Arc<WorkerHeartbeatMetrics>>) -> Self {
+        let counter_with_in_mem = |params: MetricParameters| -> Counter {
+            in_memory
+                .clone()
+                .and_then(|in_mem| in_mem.get_metric(&params.name))
+                .map(|metric| meter.counter_with_in_memory(params.clone(), metric))
+                .unwrap_or_else(|| meter.counter(params))
+        };
+
+        let gauge_with_in_mem = |params: MetricParameters| -> Gauge {
+            in_memory
+                .clone()
+                .and_then(|in_mem| in_mem.get_metric(&params.name))
+                .map(|metric| meter.gauge_with_in_memory(params.clone(), metric))
+                .unwrap_or_else(|| meter.gauge(params))
+        };
+
+        let histogram_with_in_mem = |params: MetricParameters| -> HistogramDuration {
+            in_memory
+                .clone()
+                .and_then(|in_mem| in_mem.get_metric(&params.name))
+                .map(|metric| meter.histogram_duration_with_in_memory(params.clone(), metric))
+                .unwrap_or_else(|| meter.histogram_duration(params))
+        };
+
         Self {
             wf_completed_counter: meter.counter(MetricParameters {
                 name: "workflow_completed".into(),
@@ -320,12 +365,12 @@ impl Instruments {
                 description: "Count of workflow task queue poll timeouts (no new task)".into(),
                 unit: "".into(),
             }),
-            wf_task_queue_poll_succeed_counter: meter.counter(MetricParameters {
+            wf_task_queue_poll_succeed_counter: counter_with_in_mem(MetricParameters {
                 name: "workflow_task_queue_poll_succeed".into(),
                 description: "Count of workflow task queue poll successes".into(),
                 unit: "".into(),
             }),
-            wf_task_execution_failure_counter: meter.counter(MetricParameters {
+            wf_task_execution_failure_counter: counter_with_in_mem(MetricParameters {
                 name: "workflow_task_execution_failed".into(),
                 description: "Count of workflow task execution failures".into(),
                 unit: "".into(),
@@ -340,7 +385,7 @@ impl Instruments {
                 unit: "duration".into(),
                 description: "Histogram of workflow task replay latencies".into(),
             }),
-            wf_task_execution_latency: meter.histogram_duration(MetricParameters {
+            wf_task_execution_latency: histogram_with_in_mem(MetricParameters {
                 name: WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME.into(),
                 unit: "duration".into(),
                 description: "Histogram of workflow task execution (not replay) latencies".into(),
@@ -350,12 +395,12 @@ impl Instruments {
                 description: "Count of activity task queue poll timeouts (no new task)".into(),
                 unit: "".into(),
             }),
-            act_task_received_counter: meter.counter(MetricParameters {
+            act_task_received_counter: counter_with_in_mem(MetricParameters {
                 name: "activity_task_received".into(),
                 description: "Count of activity task queue poll successes".into(),
                 unit: "".into(),
             }),
-            act_execution_failed: meter.counter(MetricParameters {
+            act_execution_failed: counter_with_in_mem(MetricParameters {
                 name: "activity_execution_failed".into(),
                 description: "Count of activity task execution failures".into(),
                 unit: "".into(),
@@ -365,7 +410,7 @@ impl Instruments {
                 unit: "duration".into(),
                 description: "Histogram of activity schedule-to-start latencies".into(),
             }),
-            act_exec_latency: meter.histogram_duration(MetricParameters {
+            act_exec_latency: histogram_with_in_mem(MetricParameters {
                 name: ACTIVITY_EXEC_LATENCY_HISTOGRAM_NAME.into(),
                 unit: "duration".into(),
                 description: "Histogram of activity execution latencies".into(),
@@ -386,7 +431,7 @@ impl Instruments {
                 description: "Count of local activity executions that failed".into(),
                 unit: "".into(),
             }),
-            la_exec_latency: meter.histogram_duration(MetricParameters {
+            la_exec_latency: histogram_with_in_mem(MetricParameters {
                 name: "local_activity_execution_latency".into(),
                 unit: "duration".into(),
                 description: "Histogram of local activity execution latencies".into(),
@@ -398,7 +443,7 @@ impl Instruments {
                     "Histogram of local activity execution latencies for successful local activities"
                         .into(),
             }),
-            la_total: meter.counter(MetricParameters {
+            la_total: counter_with_in_mem(MetricParameters {
                 name: "local_activity_total".into(),
                 description: "Count of local activities executed".into(),
                 unit: "".into(),
@@ -418,12 +463,12 @@ impl Instruments {
                 unit: "duration".into(),
                 description: "Histogram of nexus task end-to-end latencies".into(),
             }),
-            nexus_task_execution_latency: meter.histogram_duration(MetricParameters {
+            nexus_task_execution_latency: histogram_with_in_mem(MetricParameters {
                 name: "nexus_task_execution_latency".into(),
                 unit: "duration".into(),
                 description: "Histogram of nexus task execution latencies".into(),
             }),
-            nexus_task_execution_failed: meter.counter(MetricParameters {
+            nexus_task_execution_failed: counter_with_in_mem(MetricParameters {
                 name: "nexus_task_execution_failed".into(),
                 description: "Count of nexus task execution failures".into(),
                 unit: "".into(),
@@ -434,35 +479,34 @@ impl Instruments {
                 description: "Count of the number of initialized workers".into(),
                 unit: "".into(),
             }),
-            num_pollers: meter.gauge(MetricParameters {
+            num_pollers: gauge_with_in_mem(MetricParameters {
                 name: NUM_POLLERS_NAME.into(),
                 description: "Current number of active pollers per queue type".into(),
                 unit: "".into(),
             }),
-            task_slots_available: meter.gauge(MetricParameters {
+            task_slots_available: gauge_with_in_mem(MetricParameters {
                 name: TASK_SLOTS_AVAILABLE_NAME.into(),
                 description: "Current number of available slots per task type".into(),
                 unit: "".into(),
             }),
-            task_slots_used: meter.gauge(MetricParameters {
+            task_slots_used: gauge_with_in_mem(MetricParameters {
                 name: TASK_SLOTS_USED_NAME.into(),
                 description: "Current number of used slots per task type".into(),
                 unit: "".into(),
             }),
-            sticky_cache_hit: meter.counter(MetricParameters {
+            sticky_cache_hit: counter_with_in_mem(MetricParameters {
                 name: "sticky_cache_hit".into(),
                 description: "Count of times the workflow cache was used for a new workflow task"
                     .into(),
                 unit: "".into(),
             }),
-            sticky_cache_miss: meter.counter(MetricParameters {
+            sticky_cache_miss: counter_with_in_mem(MetricParameters {
                 name: "sticky_cache_miss".into(),
                 description:
-                    "Count of times the workflow cache was missing a workflow for a sticky task"
-                        .into(),
+                "Count of times the workflow cache was missing a workflow for a sticky task".into(),
                 unit: "".into(),
             }),
-            sticky_cache_size: meter.gauge(MetricParameters {
+            sticky_cache_size: gauge_with_in_mem(MetricParameters {
                 name: STICKY_CACHE_SIZE_NAME.into(),
                 description: "Current number of cached workflows".into(),
                 unit: "".into(),
@@ -598,6 +642,7 @@ pub(crate) enum FailureReason {
     Timeout,
     NexusOperation(String),
     NexusHandlerError(String),
+    GrpcMessageTooLarge,
 }
 impl Display for FailureReason {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -607,6 +652,7 @@ impl Display for FailureReason {
             FailureReason::Timeout => "timeout".to_owned(),
             FailureReason::NexusOperation(op) => format!("operation_{op}"),
             FailureReason::NexusHandlerError(op) => format!("handler_error_{op}"),
+            FailureReason::GrpcMessageTooLarge => "GrpcMessageTooLarge".to_owned(),
         };
         write!(f, "{str}")
     }
@@ -1065,7 +1111,7 @@ impl<CM: CoreMeter> CoreMeter for PrefixedMetricsMeter<CM> {
 mod tests {
     use super::*;
     use std::any::Any;
-    use temporal_sdk_core_api::telemetry::{
+    use temporalio_common::telemetry::{
         METRIC_PREFIX,
         metrics::{BufferInstrumentRef, CustomMetricAttributes},
     };
@@ -1104,6 +1150,7 @@ mod tests {
             METRIC_PREFIX.to_string(),
             Some(call_buffer.clone()),
             true,
+            temporalio_common::telemetry::TaskQueueLabelStrategy::UseNormal,
         );
         let mc = MetricsContext::top_level("foo".to_string(), "q".to_string(), &telem_instance);
         mc.forced_cache_eviction();

package/sdk-core/{core → crates/sdk-core}/src/telemetry/mod.rs

@@ -39,8 +39,9 @@ use std::{
         atomic::{AtomicBool, Ordering},
     },
 };
-use temporal_sdk_core_api::telemetry::{
-    CoreLog, CoreTelemetry, Logger, TelemetryOptions, TelemetryOptionsBuilder,
+use temporalio_common::telemetry::{
+    CoreLog, CoreTelemetry, Logger, TaskQueueLabelStrategy, TelemetryOptions,
+    TelemetryOptionsBuilder,
     metrics::{CoreMeter, MetricKeyValue, NewAttributes, TemporalMeter},
 };
 use tracing::{Level, Subscriber};
@@ -54,7 +55,7 @@ const FORWARD_LOG_BUFFER_SIZE: usize = 2048;
 /// traces at `core_level` and all others (from 3rd party modules, etc) at `other_level`.
 pub fn construct_filter_string(core_level: Level, other_level: Level) -> String {
     format!(
-        "{other_level},temporal_sdk_core={core_level},temporal_client={core_level},temporal_sdk={core_level}"
+        "{other_level},temporalio_sdk_core={core_level},temporalio_client={core_level},temporalio_sdk={core_level}"
     )
 }
 
@@ -67,6 +68,7 @@ pub struct TelemetryInstance {
     /// the user has not opted into any tracing configuration.
     trace_subscriber: Option<Arc<dyn Subscriber + Send + Sync>>,
     attach_service_name: bool,
+    task_queue_label_strategy: TaskQueueLabelStrategy,
 }
 
 impl TelemetryInstance {
@@ -76,6 +78,7 @@ impl TelemetryInstance {
         metric_prefix: String,
         metrics: Option<Arc<dyn CoreMeter + 'static>>,
         attach_service_name: bool,
+        task_queue_label_strategy: TaskQueueLabelStrategy,
     ) -> Self {
         Self {
             metric_prefix,
@@ -83,6 +86,7 @@ impl TelemetryInstance {
             metrics,
             trace_subscriber,
             attach_service_name,
+            task_queue_label_strategy,
         }
     }
 
@@ -110,6 +114,7 @@ impl TelemetryInstance {
                 Arc::new(PrefixedMetricsMeter::new(self.metric_prefix.clone(), m))
                     as Arc<dyn CoreMeter>,
                 attribs,
+                self.task_queue_label_strategy,
             )
         })
     }
@@ -119,7 +124,7 @@ impl TelemetryInstance {
         self.metrics.clone().map(|m| {
             let kvs = self.default_kvs();
             let attribs = NewAttributes::new(kvs);
-            TemporalMeter::new(m, attribs)
+            TemporalMeter::new(m, attribs, self.task_queue_label_strategy)
         })
     }
 
@@ -240,10 +245,11 @@ pub fn telemetry_init(opts: TelemetryOptions) -> Result<TelemetryInstance, anyho
         opts.metric_prefix,
         opts.metrics,
         opts.attach_service_name,
+        opts.task_queue_label_strategy,
     ))
 }
 
-/// WARNING: Calling can cause panics because of https://github.com/tokio-rs/tracing/issues/1656
+/// WARNING: Calling can cause panics because of <https://github.com/tokio-rs/tracing/issues/1656>
 /// Lang must not start using until resolved
 ///
 /// Initialize telemetry/tracing globally. Useful for testing. Only takes affect when called
@@ -262,7 +268,7 @@ pub fn telemetry_init_global(opts: TelemetryOptions) -> Result<(), anyhow::Error
     Ok(())
 }
 
-/// WARNING: Calling can cause panics because of https://github.com/tokio-rs/tracing/issues/1656
+/// WARNING: Calling can cause panics because of <https://github.com/tokio-rs/tracing/issues/1656>
 /// Lang must not start using until resolved
 ///
 /// Initialize the fallback global handler. All lang SDKs should call this somewhere, once, at

package/sdk-core/{core → crates/sdk-core}/src/telemetry/otel.rs

@@ -22,7 +22,7 @@ use opentelemetry_sdk::{
     },
 };
 use std::{collections::HashMap, sync::Arc, time::Duration};
-use temporal_sdk_core_api::telemetry::{
+use temporalio_common::telemetry::{
     HistogramBucketOverrides, MetricTemporality, OtelCollectorOptions, OtlpProtocol,
     metrics::{
         CoreMeter, Counter, Gauge, GaugeF64, Histogram, HistogramBase, HistogramDuration,

package/sdk-core/{core → crates/sdk-core}/src/telemetry/prometheus_meter.rs

@@ -12,7 +12,7 @@ use std::{
     sync::Arc,
     time::Duration,
 };
-use temporal_sdk_core_api::telemetry::metrics::{
+use temporalio_common::telemetry::metrics::{
     CoreMeter, Counter, CounterBase, Gauge, GaugeBase, GaugeF64, GaugeF64Base, Histogram,
     HistogramBase, HistogramDuration, HistogramDurationBase, HistogramF64, HistogramF64Base,
     MetricAttributable, MetricAttributes, MetricParameters, NewAttributes, OrderedPromLabelSet,
@@ -315,8 +315,7 @@ where
             Ok(labels)
         } else {
             let e = anyhow!(
-                "Must use Prometheus attributes with a Prometheus metric implementation. Got: {:?}",
-                attributes
+                "Must use Prometheus attributes with a Prometheus metric implementation. Got: {attributes:?}"
             );
             dbg_panic!("{:?}", e);
             Err(e)
@@ -460,7 +459,7 @@ pub struct CorePrometheusMeter {
     registry: Registry,
     use_seconds_for_durations: bool,
     unit_suffix: bool,
-    bucket_overrides: temporal_sdk_core_api::telemetry::HistogramBucketOverrides,
+    bucket_overrides: temporalio_common::telemetry::HistogramBucketOverrides,
 }
 
 impl CorePrometheusMeter {
@@ -468,7 +467,7 @@ impl CorePrometheusMeter {
         registry: Registry,
         use_seconds_for_durations: bool,
         unit_suffix: bool,
-        bucket_overrides: temporal_sdk_core_api::telemetry::HistogramBucketOverrides,
+        bucket_overrides: temporalio_common::telemetry::HistogramBucketOverrides,
     ) -> Self {
         Self {
             registry,
@@ -639,7 +638,7 @@ mod tests {
     use super::*;
     use crate::telemetry::{TelemetryInstance, metrics::MetricsContext};
     use prometheus::{Encoder, TextEncoder};
-    use temporal_sdk_core_api::telemetry::{
+    use temporalio_common::telemetry::{
         METRIC_PREFIX,
         metrics::{MetricKeyValue, NewAttributes},
     };
@@ -651,7 +650,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporal_sdk_core_api::telemetry::HistogramBucketOverrides::default(),
+            temporalio_common::telemetry::HistogramBucketOverrides::default(),
         );
 
         let counter = meter.counter(MetricParameters {
@@ -691,7 +690,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporal_sdk_core_api::telemetry::HistogramBucketOverrides::default(),
+            temporalio_common::telemetry::HistogramBucketOverrides::default(),
         );
 
         let base_attrs = meter.new_attributes(NewAttributes::new(vec![
@@ -730,7 +729,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporal_sdk_core_api::telemetry::HistogramBucketOverrides::default(),
+            temporalio_common::telemetry::HistogramBucketOverrides::default(),
         );
 
         let histogram_ms = meter_ms.histogram_duration(MetricParameters {
@@ -760,7 +759,7 @@ mod tests {
             registry_s.clone(),
             true,
             false,
-            temporal_sdk_core_api::telemetry::HistogramBucketOverrides::default(),
+            temporalio_common::telemetry::HistogramBucketOverrides::default(),
         );
 
         let histogram_s = meter_s.histogram_duration(MetricParameters {
@@ -792,7 +791,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporal_sdk_core_api::telemetry::HistogramBucketOverrides::default(),
+            temporalio_common::telemetry::HistogramBucketOverrides::default(),
         );
         let counter = meter.counter(MetricParameters {
             name: "no_labels".into(),
@@ -813,7 +812,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporal_sdk_core_api::telemetry::HistogramBucketOverrides::default(),
+            temporalio_common::telemetry::HistogramBucketOverrides::default(),
         );
         let telem_instance = TelemetryInstance::new(
             None,
@@ -821,6 +820,7 @@ mod tests {
             METRIC_PREFIX.to_string(),
             Some(Arc::new(meter)),
             true,
+            temporalio_common::telemetry::TaskQueueLabelStrategy::UseNormal,
         );
         let mc = MetricsContext::top_level("foo".to_string(), "q".to_string(), &telem_instance);
         mc.worker_registered();
@@ -844,7 +844,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporal_sdk_core_api::telemetry::HistogramBucketOverrides::default(),
+            temporalio_common::telemetry::HistogramBucketOverrides::default(),
         );
         let dashes = meter.counter(MetricParameters {
             name: "dash-in-name".into(),
@@ -865,7 +865,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporal_sdk_core_api::telemetry::HistogramBucketOverrides::default(),
+            temporalio_common::telemetry::HistogramBucketOverrides::default(),
         );
         let dashes = meter.counter(MetricParameters {
             name: "not@permitted:symbols".into(),

package/sdk-core/{core → crates/sdk-core}/src/telemetry/prometheus_server.rs

@@ -10,7 +10,7 @@ use std::{
     net::{SocketAddr, TcpListener},
     sync::Arc,
 };
-use temporal_sdk_core_api::telemetry::PrometheusExporterOptions;
+use temporalio_common::telemetry::PrometheusExporterOptions;
 use tokio::{io, task::AbortHandle};
 
 pub struct StartedPromServer {
@@ -20,7 +20,7 @@ pub struct StartedPromServer {
 }
 
 /// Builds and runs a prometheus endpoint which can be scraped by prom instances for metrics export.
-/// Returns the meter that can be used as a [CoreMeter].
+/// Returns the meter that can be used as a [`CoreMeter`](temporalio_common::telemetry::metrics::CoreMeter).
 ///
 /// Requires a Tokio runtime to exist, and will block briefly while binding the server endpoint.
 pub fn start_prometheus_metric_exporter(