temporalio 0.1.1 → 0.2.0

data/bridge/sdk-core/core/src/telemetry/mod.rs

@@ -1,423 +0,0 @@
-//! This module helps with the initialization and management of telemetry. IE: Metrics and tracing.
-//! Logs from core are all traces, which may be exported to the console, in memory, or externally.
-
-mod log_export;
-pub(crate) mod metrics;
-mod prometheus_server;
-
-use crate::telemetry::{
-    log_export::{CoreLogExportLayer, CoreLogsOut},
-    metrics::SDKAggSelector,
-    prometheus_server::PromServer,
-};
-use crossbeam::channel::Receiver;
-use itertools::Itertools;
-use once_cell::sync::OnceCell;
-use opentelemetry::{
-    metrics::{Meter, MeterProvider},
-    runtime,
-    sdk::{
-        export::metrics::aggregation::{self, Temporality, TemporalitySelector},
-        trace::Config,
-        Resource,
-    },
-    KeyValue,
-};
-use opentelemetry_otlp::WithExportConfig;
-use parking_lot::Mutex;
-use std::{
-    cell::RefCell,
-    collections::VecDeque,
-    convert::TryInto,
-    env,
-    net::SocketAddr,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-    time::Duration,
-};
-use temporal_sdk_core_api::telemetry::{
-    CoreLog, CoreTelemetry, Logger, MetricTemporality, MetricsExporter, OtelCollectorOptions,
-    TelemetryOptions, TraceExporter,
-};
-use tonic::metadata::MetadataMap;
-use tracing::{Level, Subscriber};
-use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Layer};
-
-const TELEM_SERVICE_NAME: &str = "temporal-core-sdk";
-
-/// Help you construct an [EnvFilter] compatible filter string which will forward all core module
-/// traces at `core_level` and all others (from 3rd party modules, etc) at `other_level`.
-pub fn construct_filter_string(core_level: Level, other_level: Level) -> String {
-    format!(
-        "{other_level},temporal_sdk_core={core_level},temporal_client={core_level},temporal_sdk={core_level}"
-    )
-}
-
-/// Holds initialized tracing/metrics exporters, etc
-pub struct TelemetryInstance {
-    metric_prefix: &'static str,
-    logs_out: Option<Mutex<CoreLogsOut>>,
-    metrics: Option<(Box<dyn MeterProvider + Send + Sync + 'static>, Meter)>,
-    trace_subscriber: Arc<dyn Subscriber + Send + Sync>,
-    prom_binding: Option<SocketAddr>,
-    _keepalive_rx: Receiver<()>,
-}
-
-impl TelemetryInstance {
-    fn new(
-        trace_subscriber: Arc<dyn Subscriber + Send + Sync>,
-        logs_out: Option<Mutex<CoreLogsOut>>,
-        metric_prefix: &'static str,
-        mut meter_provider: Option<Box<dyn MeterProvider + Send + Sync + 'static>>,
-        prom_binding: Option<SocketAddr>,
-        keepalive_rx: Receiver<()>,
-    ) -> Self {
-        let metrics = meter_provider.take().map(|mp| {
-            let meter = mp.meter(TELEM_SERVICE_NAME);
-            (mp, meter)
-        });
-        Self {
-            metric_prefix,
-            logs_out,
-            metrics,
-            trace_subscriber,
-            prom_binding,
-            _keepalive_rx: keepalive_rx,
-        }
-    }
-
-    /// Returns a trace subscriber which can be used with the tracing crate, or with our own
-    /// [set_trace_subscriber_for_current_thread] function.
-    pub fn trace_subscriber(&self) -> Arc<dyn Subscriber + Send + Sync> {
-        self.trace_subscriber.clone()
-    }
-
-    /// Returns the address the Prometheus server is bound to if it is running
-    pub fn prom_port(&self) -> Option<SocketAddr> {
-        self.prom_binding
-    }
-
-    /// Returns our wrapper for OTel metric meters, can be used to, ex: initialize clients
-    pub fn get_metric_meter(&self) -> Option<TemporalMeter> {
-        self.metrics
-            .as_ref()
-            .map(|(_, m)| TemporalMeter::new(m, self.metric_prefix))
-    }
-}
-
-thread_local! {
-    static SUB_GUARD: RefCell<Option<tracing::subscriber::DefaultGuard>> = RefCell::new(None);
-}
-/// Set the trace subscriber for the current thread. This must be done in every thread which uses
-/// core stuff, otherwise traces/logs will not be collected on that thread. For example, if using
-/// a multithreaded Tokio runtime, you should ensure that said runtime uses
-/// [on_thread_start](https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html#method.on_thread_start)
-/// or a similar mechanism to call this for each thread within the runtime.
-pub fn set_trace_subscriber_for_current_thread(sub: impl Subscriber + Send + Sync + 'static) {
-    SUB_GUARD.with(|sg| {
-        if sg.borrow().is_none() {
-            let g = tracing::subscriber::set_default(sub);
-            *sg.borrow_mut() = Some(g);
-        }
-    })
-}
-
-/// Undoes [set_trace_subscriber_for_current_thread]
-pub fn remove_trace_subscriber_for_current_thread() {
-    SUB_GUARD.with(|sg| sg.take());
-}
-
-fn metric_prefix(opts: &TelemetryOptions) -> &'static str {
-    if opts.no_temporal_prefix_for_metrics {
-        ""
-    } else {
-        "temporal_"
-    }
-}
-
-impl CoreTelemetry for TelemetryInstance {
-    fn fetch_buffered_logs(&self) -> Vec<CoreLog> {
-        if let Some(logs_out) = self.logs_out.as_ref() {
-            logs_out.lock().pop_iter().collect()
-        } else {
-            vec![]
-        }
-    }
-}
-
-/// Initialize tracing subscribers/output and logging export, returning a [TelemetryInstance]
-/// which can be used to register default / global tracing subscribers.
-///
-/// You should only call this once per unique [TelemetryOptions]
-///
-/// See [TelemetryOptions] docs for more on configuration.
-pub fn telemetry_init(opts: TelemetryOptions) -> Result<TelemetryInstance, anyhow::Error> {
-    // This is a bit odd, but functional. It's desirable to create a separate tokio runtime for
-    // metrics handling, since tests typically use a single-threaded runtime and initializing
-    // pipeline requires us to know if the runtime is single or multithreaded, we will crash
-    // in one case or the other. There does not seem to be a way to tell from the current runtime
-    // handle if it is single or multithreaded. Additionally, we can isolate metrics work this
-    // way which is nice.
-    let (tx, rx) = crossbeam::channel::bounded(0);
-    let (keepalive_tx, keepalive_rx) = crossbeam::channel::bounded(0);
-    let jh = std::thread::spawn(move || -> Result<(), anyhow::Error> {
-        let runtime = tokio::runtime::Builder::new_multi_thread()
-            .thread_name("telemetry")
-            .worker_threads(2)
-            .enable_all()
-            .build()?;
-        // Parts of telem dat ====
-        let mut logs_out = None;
-        let metric_prefix = metric_prefix(&opts);
-        let mut prom_binding = None;
-        // =======================
-
-        // Tracing subscriber layers =========
-        let mut console_pretty_layer = None;
-        let mut console_compact_layer = None;
-        let mut forward_layer = None;
-        let mut export_layer = None;
-        // ===================================
-
-        if let Some(ref logger) = opts.logging {
-            match logger {
-                Logger::Console { filter } => {
-                    // This is silly dupe but can't be avoided without boxing.
-                    if env::var("TEMPORAL_CORE_PRETTY_LOGS").is_ok() {
-                        console_pretty_layer = Some(
-                            tracing_subscriber::fmt::layer()
-                                .with_target(false)
-                                .event_format(
-                                    tracing_subscriber::fmt::format()
-                                        .pretty()
-                                        .with_source_location(false),
-                                )
-                                .with_filter(EnvFilter::new(filter)),
-                        )
-                    } else {
-                        console_compact_layer = Some(
-                            tracing_subscriber::fmt::layer()
-                                .with_target(false)
-                                .event_format(
-                                    tracing_subscriber::fmt::format()
-                                        .compact()
-                                        .with_source_location(false),
-                                )
-                                .with_filter(EnvFilter::new(filter)),
-                        )
-                    }
-                }
-                Logger::Forward { filter } => {
-                    let (export_layer, lo) = CoreLogExportLayer::new();
-                    logs_out = Some(Mutex::new(lo));
-                    forward_layer = Some(export_layer.with_filter(EnvFilter::new(filter)));
-                }
-            };
-        };
-
-        let meter_provider = if let Some(ref metrics) = opts.metrics {
-            let aggregator = SDKAggSelector { metric_prefix };
-            match metrics {
-                MetricsExporter::Prometheus(addr) => {
-                    let srv = runtime.block_on(async move {
-                        PromServer::new(
-                            *addr,
-                            aggregator,
-                            metric_temporality_to_selector(opts.metric_temporality),
-                        )
-                    })?;
-                    prom_binding = Some(srv.bound_addr());
-                    let mp = srv.exporter.meter_provider()?;
-                    runtime.spawn(async move { srv.run().await });
-                    Some(Box::new(mp) as Box<dyn MeterProvider + Send + Sync>)
-                }
-                MetricsExporter::Otel(OtelCollectorOptions {
-                    url,
-                    headers,
-                    metric_periodicity,
-                }) => runtime.block_on(async {
-                    let metrics = opentelemetry_otlp::new_pipeline()
-                        .metrics(
-                            aggregator,
-                            metric_temporality_to_selector(opts.metric_temporality),
-                            runtime::Tokio,
-                        )
-                        .with_period(metric_periodicity.unwrap_or_else(|| Duration::from_secs(1)))
-                        .with_resource(default_resource())
-                        .with_exporter(
-                            opentelemetry_otlp::new_exporter()
-                                .tonic()
-                                .with_endpoint(url.to_string())
-                                .with_metadata(MetadataMap::from_headers(headers.try_into()?)),
-                        )
-                        .build()?;
-                    Ok::<_, anyhow::Error>(Some(
-                        Box::new(metrics) as Box<dyn MeterProvider + Send + Sync>
-                    ))
-                })?,
-            }
-        } else {
-            None
-        };
-
-        if let Some(ref tracing) = opts.tracing {
-            match &tracing.exporter {
-                TraceExporter::Otel(OtelCollectorOptions { url, headers, .. }) => {
-                    runtime.block_on(async {
-                        let tracer_cfg = Config::default().with_resource(default_resource());
-                        let tracer = opentelemetry_otlp::new_pipeline()
-                            .tracing()
-                            .with_exporter(
-                                opentelemetry_otlp::new_exporter()
-                                    .tonic()
-                                    .with_endpoint(url.to_string())
-                                    .with_metadata(MetadataMap::from_headers(headers.try_into()?)),
-                            )
-                            .with_trace_config(tracer_cfg)
-                            .install_batch(runtime::Tokio)?;
-
-                        let opentelemetry = tracing_opentelemetry::layer()
-                            .with_tracer(tracer)
-                            .with_filter(EnvFilter::new(&tracing.filter));
-
-                        export_layer = Some(opentelemetry);
-                        Result::<(), anyhow::Error>::Ok(())
-                    })?;
-                }
-            };
-        };
-
-        let reg = tracing_subscriber::registry()
-            .with(console_pretty_layer)
-            .with(console_compact_layer)
-            .with(forward_layer)
-            .with(export_layer);
-
-        tx.send(TelemetryInstance::new(
-            Arc::new(reg),
-            logs_out,
-            metric_prefix,
-            meter_provider,
-            prom_binding,
-            keepalive_rx,
-        ))
-        .expect("Must be able to send telem instance out of thread");
-        // Now keep the thread alive until the telemetry instance is dropped by trying to send
-        // something forever
-        let _ = keepalive_tx.send(());
-        Ok(())
-    });
-    match rx.recv() {
-        Ok(ti) => Ok(ti),
-        Err(_) => {
-            // Immediately join the thread since something went wrong in it
-            jh.join().expect("Telemetry must init cleanly")?;
-            // This can't happen. The rx channel can't be dropped unless the thread errored.
-            unreachable!("Impossible error in telemetry init thread");
-        }
-    }
-}
-
-/// Initialize telemetry/tracing globally. Useful for testing. Only takes affect when called
-/// the first time. Subsequent calls are ignored.
-pub fn telemetry_init_global(opts: TelemetryOptions) -> Result<(), anyhow::Error> {
-    static INITTED: AtomicBool = AtomicBool::new(false);
-    if INITTED
-        .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
-        .is_ok()
-    {
-        let ti = telemetry_init(opts)?;
-        tracing::subscriber::set_global_default(ti.trace_subscriber())?;
-    }
-    Ok(())
-}
-
-fn default_resource_kvs() -> &'static [KeyValue] {
-    static INSTANCE: OnceCell<[KeyValue; 1]> = OnceCell::new();
-    INSTANCE.get_or_init(|| [KeyValue::new("service.name", TELEM_SERVICE_NAME)])
-}
-fn default_resource() -> Resource {
-    Resource::new(default_resource_kvs().iter().cloned())
-}
-
-fn metric_temporality_to_selector(
-    t: MetricTemporality,
-) -> impl TemporalitySelector + Send + Sync + Clone {
-    match t {
-        MetricTemporality::Cumulative => {
-            aggregation::constant_temporality_selector(Temporality::Cumulative)
-        }
-        MetricTemporality::Delta => aggregation::constant_temporality_selector(Temporality::Delta),
-    }
-}
-
-#[cfg(test)]
-pub mod test_initters {
-    use super::*;
-    use temporal_sdk_core_api::telemetry::{TelemetryOptionsBuilder, TraceExportConfig};
-
-    #[allow(dead_code)] // Not always used, called to enable for debugging when needed
-    pub fn test_telem_console() {
-        telemetry_init_global(
-            TelemetryOptionsBuilder::default()
-                .logging(Logger::Console {
-                    filter: construct_filter_string(Level::DEBUG, Level::WARN),
-                })
-                .build()
-                .unwrap(),
-        )
-        .unwrap();
-    }
-
-    #[allow(dead_code)] // Not always used, called to enable for debugging when needed
-    pub fn test_telem_collector() {
-        telemetry_init_global(
-            TelemetryOptionsBuilder::default()
-                .logging(Logger::Console {
-                    filter: construct_filter_string(Level::DEBUG, Level::WARN),
-                })
-                .tracing(TraceExportConfig {
-                    filter: construct_filter_string(Level::DEBUG, Level::WARN),
-                    exporter: TraceExporter::Otel(OtelCollectorOptions {
-                        url: "grpc://localhost:4317".parse().unwrap(),
-                        headers: Default::default(),
-                        metric_periodicity: None,
-                    }),
-                })
-                .build()
-                .unwrap(),
-        )
-        .unwrap();
-    }
-}
-use crate::telemetry::metrics::TemporalMeter;
-#[cfg(test)]
-pub use test_initters::*;
-
-/// A trait for using [Display] on the contents of vecs, etc, which don't implement it.
-///
-/// Dislike this, but, there doesn't seem to be a great alternative. Calling itertools format
-/// inline in an `event!` macro can panic because it gets evaluated twice somehow.
-pub(crate) trait VecDisplayer {
-    fn display(&self) -> String;
-}
-
-impl<T> VecDisplayer for Vec<T>
-where
-    T: std::fmt::Display,
-{
-    fn display(&self) -> String {
-        format!("[{}]", self.iter().format(","))
-    }
-}
-
-impl<T> VecDisplayer for VecDeque<T>
-where
-    T: std::fmt::Display,
-{
-    fn display(&self) -> String {
-        format!("[{}]", self.iter().format(","))
-    }
-}

data/bridge/sdk-core/core/src/telemetry/prometheus_server.rs

@@ -1,83 +0,0 @@
-use crate::telemetry::default_resource;
-use hyper::{
-    header::CONTENT_TYPE,
-    server::conn::AddrIncoming,
-    service::{make_service_fn, service_fn},
-    Body, Method, Request, Response, Server,
-};
-use opentelemetry::sdk::{
-    export::metrics::{aggregation::TemporalitySelector, AggregatorSelector},
-    metrics::{controllers, processors},
-};
-use opentelemetry_prometheus::{ExporterBuilder, PrometheusExporter};
-use prometheus::{Encoder, TextEncoder};
-use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration};
-
-/// Exposes prometheus metrics for scraping
-pub(super) struct PromServer {
-    bound_addr: AddrIncoming,
-    pub exporter: Arc<PrometheusExporter>,
-}
-
-impl PromServer {
-    pub fn new(
-        addr: SocketAddr,
-        aggregation: impl AggregatorSelector + Send + Sync + 'static,
-        temporality: impl TemporalitySelector + Send + Sync + 'static,
-    ) -> Result<Self, anyhow::Error> {
-        let controller =
-            controllers::basic(processors::factory(aggregation, temporality).with_memory(true))
-                // Because Prom is pull-based, make this always refresh
-                .with_collect_period(Duration::from_secs(0))
-                .with_resource(default_resource())
-                .build();
-        let exporter = ExporterBuilder::new(controller).try_init()?;
-        let bound_addr = AddrIncoming::bind(&addr)?;
-        Ok(Self {
-            exporter: Arc::new(exporter),
-            bound_addr,
-        })
-    }
-
-    pub async fn run(self) -> hyper::Result<()> {
-        // Spin up hyper server to serve metrics for scraping. We use hyper since we already depend
-        // on it via Tonic.
-        let expclone = self.exporter.clone();
-        let svc = make_service_fn(move |_conn| {
-            let expclone = expclone.clone();
-            async move { Ok::<_, Infallible>(service_fn(move |req| metrics_req(req, expclone.clone()))) }
-        });
-        let server = Server::builder(self.bound_addr).serve(svc);
-        server.await
-    }
-
-    pub fn bound_addr(&self) -> SocketAddr {
-        self.bound_addr.local_addr()
-    }
-}
-
-/// Serves prometheus metrics in the expected format for scraping
-async fn metrics_req(
-    req: Request<Body>,
-    exporter: Arc<PrometheusExporter>,
-) -> Result<Response<Body>, hyper::Error> {
-    let response = match (req.method(), req.uri().path()) {
-        (&Method::GET, "/metrics") => {
-            let mut buffer = vec![];
-            let encoder = TextEncoder::new();
-            let metric_families = exporter.registry().gather();
-            encoder.encode(&metric_families, &mut buffer).unwrap();
-
-            Response::builder()
-                .status(200)
-                .header(CONTENT_TYPE, encoder.format_type())
-                .body(Body::from(buffer))
-                .unwrap()
-        }
-        _ => Response::builder()
-            .status(404)
-            .body(Body::empty())
-            .expect("Can't fail to construct empty resp"),
-    };
-    Ok(response)
-}