@temporalio/core-bridge 1.8.5 → 1.9.0-rc.0

package/sdk-core/core/src/telemetry/mod.rs

@@ -5,48 +5,42 @@ mod log_export;
 pub(crate) mod metrics;
 mod prometheus_server;
 
-use crate::telemetry::{
-    log_export::{CoreLogExportLayer, CoreLogsOut},
-    metrics::SDKAggSelector,
-    prometheus_server::PromServer,
+pub use metrics::{
+    build_otlp_metric_exporter, default_buckets_for, start_prometheus_metric_exporter,
+    MetricsCallBuffer,
 };
-use crossbeam::channel::Receiver;
+
+pub use log_export::{CoreLogBuffer, CoreLogBufferedConsumer, CoreLogStreamConsumer};
+
+use crate::telemetry::{log_export::CoreLogConsumerLayer, metrics::PrefixedMetricsMeter};
 use itertools::Itertools;
 use once_cell::sync::OnceCell;
-use opentelemetry::{
-    metrics::{Meter, MeterProvider},
-    runtime,
-    sdk::{
-        export::metrics::aggregation::{self, Temporality, TemporalitySelector},
-        trace::Config,
-        Resource,
-    },
-    KeyValue,
+use opentelemetry::KeyValue;
+use opentelemetry_sdk::{
+    metrics::{data::Temporality, reader::TemporalitySelector, InstrumentKind},
+    Resource,
 };
-use opentelemetry_otlp::WithExportConfig;
 use parking_lot::Mutex;
 use std::{
     cell::RefCell,
     collections::{HashMap, VecDeque},
-    convert::TryInto,
     env,
-    net::SocketAddr,
     sync::{
         atomic::{AtomicBool, Ordering},
         Arc,
     },
-    time::Duration,
 };
 use temporal_sdk_core_api::telemetry::{
-    CoreLog, CoreTelemetry, Logger, MetricTemporality, MetricsExporter, OtelCollectorOptions,
-    TelemetryOptions, TraceExporter,
+    metrics::{CoreMeter, MetricKeyValue, NewAttributes, TemporalMeter},
+    CoreLog, CoreTelemetry, Logger, MetricTemporality, TelemetryOptions,
 };
-use tonic::metadata::MetadataMap;
 use tracing::{Level, Subscriber};
 use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Layer};
 
 const TELEM_SERVICE_NAME: &str = "temporal-core-sdk";
 
+const FORWARD_LOG_BUFFER_SIZE: usize = 2048;
+
 /// Help you construct an [EnvFilter] compatible filter string which will forward all core module
 /// traces at `core_level` and all others (from 3rd party modules, etc) at `other_level`.
 pub fn construct_filter_string(core_level: Level, other_level: Level) -> String {
@@ -57,53 +51,75 @@ pub fn construct_filter_string(core_level: Level, other_level: Level) -> String
 
 /// Holds initialized tracing/metrics exporters, etc
 pub struct TelemetryInstance {
-    metric_prefix: &'static str,
-    logs_out: Option<Mutex<CoreLogsOut>>,
-    metrics: Option<(Box<dyn MeterProvider + Send + Sync + 'static>, Meter)>,
-    trace_subscriber: Arc<dyn Subscriber + Send + Sync>,
-    prom_binding: Option<SocketAddr>,
-    _keepalive_rx: Receiver<()>,
+    metric_prefix: String,
+    logs_out: Option<Mutex<CoreLogBuffer>>,
+    metrics: Option<Arc<dyn CoreMeter + 'static>>,
+    /// The tracing subscriber which is associated with this telemetry instance. May be `None` if
+    /// the user has not opted into any tracing configuration.
+    trace_subscriber: Option<Arc<dyn Subscriber + Send + Sync>>,
+    attach_service_name: bool,
 }
 
 impl TelemetryInstance {
     fn new(
-        trace_subscriber: Arc<dyn Subscriber + Send + Sync>,
-        logs_out: Option<Mutex<CoreLogsOut>>,
-        metric_prefix: &'static str,
-        mut meter_provider: Option<Box<dyn MeterProvider + Send + Sync + 'static>>,
-        prom_binding: Option<SocketAddr>,
-        keepalive_rx: Receiver<()>,
+        trace_subscriber: Option<Arc<dyn Subscriber + Send + Sync>>,
+        logs_out: Option<Mutex<CoreLogBuffer>>,
+        metric_prefix: String,
+        metrics: Option<Arc<dyn CoreMeter + 'static>>,
+        attach_service_name: bool,
     ) -> Self {
-        let metrics = meter_provider.take().map(|mp| {
-            let meter = mp.meter(TELEM_SERVICE_NAME);
-            (mp, meter)
-        });
         Self {
             metric_prefix,
             logs_out,
             metrics,
             trace_subscriber,
-            prom_binding,
-            _keepalive_rx: keepalive_rx,
+            attach_service_name,
         }
     }
 
-    /// Returns a trace subscriber which can be used with the tracing crate, or with our own
-    /// [set_trace_subscriber_for_current_thread] function.
-    pub fn trace_subscriber(&self) -> Arc<dyn Subscriber + Send + Sync> {
+    /// Return the trace subscriber associated with the telemetry options/instance. Can be used
+    /// to manually set the default for a thread or globally using the `tracing` crate, or with
+    /// [set_trace_subscriber_for_current_thread].
+    pub fn trace_subscriber(&self) -> Option<Arc<dyn Subscriber + Send + Sync>> {
         self.trace_subscriber.clone()
     }
 
-    /// Returns the address the Prometheus server is bound to if it is running
-    pub fn prom_port(&self) -> Option<SocketAddr> {
-        self.prom_binding
+    /// Some metric meters cannot be initialized until after a tokio runtime has started and after
+    /// other telemetry has initted (ex: prometheus). They can be attached here.
+    pub fn attach_late_init_metrics(&mut self, meter: Arc<dyn CoreMeter + 'static>) {
+        self.metrics = Some(meter);
     }
 
-    /// Returns our wrapper for OTel metric meters, can be used to, ex: initialize clients
+    /// Returns our wrapper for metric meters, including the `metric_prefix` from
+    /// [TelemetryOptions]. This should be used to initialize clients or for any other
+    /// temporal-owned metrics. User defined metrics should use [Self::get_metric_meter].
+    pub fn get_temporal_metric_meter(&self) -> Option<TemporalMeter> {
+        self.metrics.clone().map(|m| {
+            let kvs = self.default_kvs();
+            let attribs = NewAttributes::new(kvs);
+            TemporalMeter::new(
+                Arc::new(PrefixedMetricsMeter::new(self.metric_prefix.clone(), m))
+                    as Arc<dyn CoreMeter>,
+                attribs,
+            )
+        })
+    }
+
+    /// Returns our wrapper for metric meters, including attaching the service name if enabled.
     pub fn get_metric_meter(&self) -> Option<TemporalMeter> {
-        self.metrics
-            .as_ref()
-            .map(|(_, m)| TemporalMeter::new(m, self.metric_prefix))
+        self.metrics.clone().map(|m| {
+            let kvs = self.default_kvs();
+            let attribs = NewAttributes::new(kvs);
+            TemporalMeter::new(m, attribs)
+        })
+    }
+
+    fn default_kvs(&self) -> Vec<MetricKeyValue> {
+        if self.attach_service_name {
+            vec![MetricKeyValue::new("service_name", TELEM_SERVICE_NAME)]
+        } else {
+            vec![]
+        }
     }
 }
 
@@ -129,18 +145,10 @@ pub fn remove_trace_subscriber_for_current_thread() {
     SUB_GUARD.with(|sg| sg.take());
 }
 
-fn metric_prefix(opts: &TelemetryOptions) -> &'static str {
-    if opts.no_temporal_prefix_for_metrics {
-        ""
-    } else {
-        "temporal_"
-    }
-}
-
 impl CoreTelemetry for TelemetryInstance {
     fn fetch_buffered_logs(&self) -> Vec<CoreLog> {
         if let Some(logs_out) = self.logs_out.as_ref() {
-            logs_out.lock().pop_iter().collect()
+            logs_out.lock().drain()
         } else {
             vec![]
         }
@@ -160,169 +168,72 @@ pub fn telemetry_init(opts: TelemetryOptions) -> Result<TelemetryInstance, anyho
     // in one case or the other. There does not seem to be a way to tell from the current runtime
     // handle if it is single or multithreaded. Additionally, we can isolate metrics work this
     // way which is nice.
-    let (tx, rx) = crossbeam::channel::bounded(0);
-    let (keepalive_tx, keepalive_rx) = crossbeam::channel::bounded(0);
-    let jh = std::thread::spawn(move || -> Result<(), anyhow::Error> {
-        let runtime = tokio::runtime::Builder::new_multi_thread()
-            .thread_name("telemetry")
-            .worker_threads(2)
-            .enable_all()
-            .build()?;
-        // Parts of telem dat ====
-        let mut logs_out = None;
-        let metric_prefix = metric_prefix(&opts);
-        let mut prom_binding = None;
-        // =======================
-
-        // Tracing subscriber layers =========
-        let mut console_pretty_layer = None;
-        let mut console_compact_layer = None;
-        let mut forward_layer = None;
-        let mut export_layer = None;
-        // ===================================
-
-        if let Some(ref logger) = opts.logging {
-            match logger {
-                Logger::Console { filter } => {
-                    // This is silly dupe but can't be avoided without boxing.
-                    if env::var("TEMPORAL_CORE_PRETTY_LOGS").is_ok() {
-                        console_pretty_layer = Some(
-                            tracing_subscriber::fmt::layer()
-                                .with_target(false)
-                                .event_format(
-                                    tracing_subscriber::fmt::format()
-                                        .pretty()
-                                        .with_source_location(false),
-                                )
-                                .with_filter(EnvFilter::new(filter)),
-                        )
-                    } else {
-                        console_compact_layer = Some(
-                            tracing_subscriber::fmt::layer()
-                                .with_target(false)
-                                .event_format(
-                                    tracing_subscriber::fmt::format()
-                                        .compact()
-                                        .with_source_location(false),
-                                )
-                                .with_filter(EnvFilter::new(filter)),
-                        )
-                    }
-                }
-                Logger::Forward { filter } => {
-                    let (export_layer, lo) = CoreLogExportLayer::new();
-                    logs_out = Some(Mutex::new(lo));
-                    forward_layer = Some(export_layer.with_filter(EnvFilter::new(filter)));
-                }
-            };
-        };
-
-        let meter_provider = if let Some(ref metrics) = opts.metrics {
-            let aggregator = SDKAggSelector { metric_prefix };
-            match metrics {
-                MetricsExporter::Prometheus(addr) => {
-                    let srv = runtime.block_on(async {
-                        PromServer::new(
-                            *addr,
-                            aggregator,
-                            metric_temporality_to_selector(opts.metric_temporality),
-                            &opts.global_tags,
-                        )
-                    })?;
-                    prom_binding = Some(srv.bound_addr());
-                    let mp = srv.exporter.meter_provider()?;
-                    runtime.spawn(async move { srv.run().await });
-                    Some(Box::new(mp) as Box<dyn MeterProvider + Send + Sync>)
-                }
-                MetricsExporter::Otel(OtelCollectorOptions {
-                    url,
-                    headers,
-                    metric_periodicity,
-                }) => runtime.block_on(async {
-                    let metrics = opentelemetry_otlp::new_pipeline()
-                        .metrics(
-                            aggregator,
-                            metric_temporality_to_selector(opts.metric_temporality),
-                            runtime::Tokio,
-                        )
-                        .with_period(metric_periodicity.unwrap_or_else(|| Duration::from_secs(1)))
-                        .with_resource(default_resource(&opts.global_tags))
-                        .with_exporter(
-                            opentelemetry_otlp::new_exporter()
-                                .tonic()
-                                .with_endpoint(url.to_string())
-                                .with_metadata(MetadataMap::from_headers(headers.try_into()?)),
-                        )
-                        .build()?;
-                    Ok::<_, anyhow::Error>(Some(
-                        Box::new(metrics) as Box<dyn MeterProvider + Send + Sync>
-                    ))
-                })?,
-            }
-        } else {
-            None
-        };
-
-        if let Some(ref tracing) = opts.tracing {
-            match &tracing.exporter {
-                TraceExporter::Otel(OtelCollectorOptions { url, headers, .. }) => {
-                    runtime.block_on(async {
-                        let tracer_cfg =
-                            Config::default().with_resource(default_resource(&opts.global_tags));
-                        let tracer = opentelemetry_otlp::new_pipeline()
-                            .tracing()
-                            .with_exporter(
-                                opentelemetry_otlp::new_exporter()
-                                    .tonic()
-                                    .with_endpoint(url.to_string())
-                                    .with_metadata(MetadataMap::from_headers(headers.try_into()?)),
+    // Parts of telem dat ====
+    let mut logs_out = None;
+    // =======================
+
+    // Tracing subscriber layers =========
+    let mut console_pretty_layer = None;
+    let mut console_compact_layer = None;
+    let mut forward_layer = None;
+    // ===================================
+
+    let tracing_sub = opts.logging.map(|logger| {
+        match logger {
+            Logger::Console { filter } => {
+                // This is silly dupe but can't be avoided without boxing.
+                if env::var("TEMPORAL_CORE_PRETTY_LOGS").is_ok() {
+                    console_pretty_layer = Some(
+                        tracing_subscriber::fmt::layer()
+                            .with_target(false)
+                            .event_format(
+                                tracing_subscriber::fmt::format()
+                                    .pretty()
+                                    .with_source_location(false),
                             )
-                            .with_trace_config(tracer_cfg)
-                            .install_batch(runtime::Tokio)?;
-
-                        let opentelemetry = tracing_opentelemetry::layer()
-                            .with_tracer(tracer)
-                            .with_filter(EnvFilter::new(&tracing.filter));
-
-                        export_layer = Some(opentelemetry);
-                        Result::<(), anyhow::Error>::Ok(())
-                    })?;
+                            .with_filter(EnvFilter::new(filter)),
+                    )
+                } else {
+                    console_compact_layer = Some(
+                        tracing_subscriber::fmt::layer()
+                            .with_target(false)
+                            .event_format(
+                                tracing_subscriber::fmt::format()
+                                    .compact()
+                                    .with_source_location(false),
+                            )
+                            .with_filter(EnvFilter::new(filter)),
+                    )
                 }
-            };
+            }
+            Logger::Forward { filter } => {
+                let (export_layer, lo) =
+                    CoreLogConsumerLayer::new_buffered(FORWARD_LOG_BUFFER_SIZE);
+                logs_out = Some(Mutex::new(lo));
+                forward_layer = Some(export_layer.with_filter(EnvFilter::new(filter)));
+            }
+            Logger::Push { filter, consumer } => {
+                forward_layer =
+                    Some(CoreLogConsumerLayer::new(consumer).with_filter(EnvFilter::new(filter)));
+            }
         };
-
         let reg = tracing_subscriber::registry()
             .with(console_pretty_layer)
             .with(console_compact_layer)
-            .with(forward_layer)
-            .with(export_layer);
+            .with(forward_layer);
 
         #[cfg(feature = "tokio-console")]
         let reg = reg.with(console_subscriber::spawn());
-
-        tx.send(TelemetryInstance::new(
-            Arc::new(reg),
-            logs_out,
-            metric_prefix,
-            meter_provider,
-            prom_binding,
-            keepalive_rx,
-        ))
-        .expect("Must be able to send telem instance out of thread");
-        // Now keep the thread alive until the telemetry instance is dropped by trying to send
-        // something forever
-        let _ = keepalive_tx.send(());
-        Ok(())
+        Arc::new(reg) as Arc<dyn Subscriber + Send + Sync>
     });
-    match rx.recv() {
-        Ok(ti) => Ok(ti),
-        Err(_) => {
-            // Immediately join the thread since something went wrong in it
-            jh.join().expect("Telemetry must init cleanly")?;
-            // This can't happen. The rx channel can't be dropped unless the thread errored.
-            unreachable!("Impossible error in telemetry init thread");
-        }
-    }
+
+    Ok(TelemetryInstance::new(
+        tracing_sub,
+        logs_out,
+        opts.metric_prefix,
+        opts.metrics,
+        opts.attach_service_name,
+    ))
 }
 
 /// Initialize telemetry/tracing globally. Useful for testing. Only takes affect when called
@@ -334,7 +245,9 @@ pub fn telemetry_init_global(opts: TelemetryOptions) -> Result<(), anyhow::Error
         .is_ok()
     {
         let ti = telemetry_init(opts)?;
-        tracing::subscriber::set_global_default(ti.trace_subscriber())?;
+        if let Some(ts) = ti.trace_subscriber() {
+            tracing::subscriber::set_global_default(ts)?;
+        }
     }
     Ok(())
 }
@@ -351,21 +264,26 @@ fn default_resource(override_values: &HashMap<String, String>) -> Resource {
     Resource::new(default_resource_kvs().iter().cloned()).merge(&Resource::new(override_kvs))
 }
 
+#[derive(Clone)]
+struct ConstantTemporality(Temporality);
+impl TemporalitySelector for ConstantTemporality {
+    fn temporality(&self, _: InstrumentKind) -> Temporality {
+        self.0
+    }
+}
 fn metric_temporality_to_selector(
     t: MetricTemporality,
 ) -> impl TemporalitySelector + Send + Sync + Clone {
     match t {
-        MetricTemporality::Cumulative => {
-            aggregation::constant_temporality_selector(Temporality::Cumulative)
-        }
-        MetricTemporality::Delta => aggregation::constant_temporality_selector(Temporality::Delta),
+        MetricTemporality::Cumulative => ConstantTemporality(Temporality::Cumulative),
+        MetricTemporality::Delta => ConstantTemporality(Temporality::Delta),
     }
 }
 
 #[cfg(test)]
 pub mod test_initters {
     use super::*;
-    use temporal_sdk_core_api::telemetry::{TelemetryOptionsBuilder, TraceExportConfig};
+    use temporal_sdk_core_api::telemetry::TelemetryOptionsBuilder;
 
     #[allow(dead_code)] // Not always used, called to enable for debugging when needed
     pub fn test_telem_console() {
@@ -379,29 +297,7 @@ pub mod test_initters {
         )
         .unwrap();
     }
-
-    #[allow(dead_code)] // Not always used, called to enable for debugging when needed
-    pub fn test_telem_collector() {
-        telemetry_init_global(
-            TelemetryOptionsBuilder::default()
-                .logging(Logger::Console {
-                    filter: construct_filter_string(Level::DEBUG, Level::WARN),
-                })
-                .tracing(TraceExportConfig {
-                    filter: construct_filter_string(Level::DEBUG, Level::WARN),
-                    exporter: TraceExporter::Otel(OtelCollectorOptions {
-                        url: "grpc://localhost:4317".parse().unwrap(),
-                        headers: Default::default(),
-                        metric_periodicity: None,
-                    }),
-                })
-                .build()
-                .unwrap(),
-        )
-        .unwrap();
-    }
 }
-use crate::telemetry::metrics::TemporalMeter;
 #[cfg(test)]
 pub use test_initters::*;
 

package/sdk-core/core/src/telemetry/prometheus_server.rs

@@ -1,52 +1,57 @@
-use crate::telemetry::default_resource;
 use hyper::{
     header::CONTENT_TYPE,
     server::conn::AddrIncoming,
     service::{make_service_fn, service_fn},
     Body, Method, Request, Response, Server,
 };
-use opentelemetry::sdk::{
-    export::metrics::{aggregation::TemporalitySelector, AggregatorSelector},
-    metrics::{controllers, processors},
-};
-use opentelemetry_prometheus::{ExporterBuilder, PrometheusExporter};
-use prometheus::{Encoder, TextEncoder};
-use std::{collections::HashMap, convert::Infallible, net::SocketAddr, sync::Arc, time::Duration};
+use opentelemetry_prometheus::PrometheusExporter;
+use opentelemetry_sdk::metrics::reader::AggregationSelector;
+use prometheus::{Encoder, Registry, TextEncoder};
+use std::{convert::Infallible, net::SocketAddr};
+use temporal_sdk_core_api::telemetry::PrometheusExporterOptions;
 
 /// Exposes prometheus metrics for scraping
 pub(super) struct PromServer {
     bound_addr: AddrIncoming,
-    pub exporter: Arc<PrometheusExporter>,
+    registry: Registry,
 }
 
 impl PromServer {
     pub fn new(
-        addr: SocketAddr,
-        aggregation: impl AggregatorSelector + Send + Sync + 'static,
-        temporality: impl TemporalitySelector + Send + Sync + 'static,
-        tags: &HashMap<String, String>,
-    ) -> Result<Self, anyhow::Error> {
-        let controller =
-            controllers::basic(processors::factory(aggregation, temporality).with_memory(true))
-                // Because Prom is pull-based, make this always refresh
-                .with_collect_period(Duration::from_secs(0))
-                .with_resource(default_resource(tags))
-                .build();
-        let exporter = ExporterBuilder::new(controller).try_init()?;
-        let bound_addr = AddrIncoming::bind(&addr)?;
-        Ok(Self {
-            exporter: Arc::new(exporter),
-            bound_addr,
-        })
+        opts: &PrometheusExporterOptions,
+        aggregation: impl AggregationSelector + Send + Sync + 'static,
+    ) -> Result<(Self, PrometheusExporter), anyhow::Error> {
+        let registry = Registry::new();
+        let exporter = opentelemetry_prometheus::exporter()
+            .with_aggregation_selector(aggregation)
+            .without_scope_info()
+            .with_registry(registry.clone());
+        let exporter = if !opts.counters_total_suffix {
+            exporter.without_counter_suffixes()
+        } else {
+            exporter
+        };
+        let exporter = if !opts.unit_suffix {
+            exporter.without_units()
+        } else {
+            exporter
+        };
+        let bound_addr = AddrIncoming::bind(&opts.socket_addr)?;
+        Ok((
+            Self {
+                bound_addr,
+                registry,
+            },
+            exporter.build()?,
+        ))
     }
 
     pub async fn run(self) -> hyper::Result<()> {
         // Spin up hyper server to serve metrics for scraping. We use hyper since we already depend
         // on it via Tonic.
-        let expclone = self.exporter.clone();
         let svc = make_service_fn(move |_conn| {
-            let expclone = expclone.clone();
-            async move { Ok::<_, Infallible>(service_fn(move |req| metrics_req(req, expclone.clone()))) }
+            let regclone = self.registry.clone();
+            async move { Ok::<_, Infallible>(service_fn(move |req| metrics_req(req, regclone.clone()))) }
         });
         let server = Server::builder(self.bound_addr).serve(svc);
         server.await
@@ -60,13 +65,13 @@ impl PromServer {
 /// Serves prometheus metrics in the expected format for scraping
 async fn metrics_req(
     req: Request<Body>,
-    exporter: Arc<PrometheusExporter>,
+    registry: Registry,
 ) -> Result<Response<Body>, hyper::Error> {
     let response = match (req.method(), req.uri().path()) {
         (&Method::GET, "/metrics") => {
             let mut buffer = vec![];
             let encoder = TextEncoder::new();
-            let metric_families = exporter.registry().gather();
+            let metric_families = registry.gather();
             encoder.encode(&metric_families, &mut buffer).unwrap();
 
             Response::builder()