@temporalio/core-bridge 1.14.2-canary-release-testing.0 → 1.16.0

package/sdk-core/crates/{sdk-core → common}/src/telemetry/otel.rs

@@ -1,14 +1,17 @@
 use super::{
-    TELEM_SERVICE_NAME, default_buckets_for,
+    HistogramBucketOverrides, MetricTemporality, OtelCollectorOptions, OtlpProtocol,
+    TELEM_SERVICE_NAME,
     metrics::{
         ACTIVITY_EXEC_LATENCY_HISTOGRAM_NAME, ACTIVITY_SCHED_TO_START_LATENCY_HISTOGRAM_NAME,
-        DEFAULT_MS_BUCKETS, WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME,
-        WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME,
+        CoreMeter, Counter, DEFAULT_MS_BUCKETS, DEFAULT_S_BUCKETS, Gauge, GaugeF64, Histogram,
+        HistogramBase, HistogramDuration, HistogramDurationBase, HistogramF64, HistogramF64Base,
+        MetricAttributable, MetricAttributes, MetricParameters, NewAttributes,
+        WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME, WORKFLOW_TASK_EXECUTION_LATENCY_HISTOGRAM_NAME,
         WORKFLOW_TASK_REPLAY_LATENCY_HISTOGRAM_NAME,
-        WORKFLOW_TASK_SCHED_TO_START_LATENCY_HISTOGRAM_NAME,
+        WORKFLOW_TASK_SCHED_TO_START_LATENCY_HISTOGRAM_NAME, default_buckets_for,
     },
 };
-use crate::{abstractions::dbg_panic, telemetry::metrics::DEFAULT_S_BUCKETS};
+use crate::dbg_panic;
 use opentelemetry::{
     self, Key, KeyValue, Value,
     metrics::{Meter, MeterProvider as MeterProviderT},
@@ -22,14 +25,6 @@ use opentelemetry_sdk::{
     },
 };
 use std::{collections::HashMap, sync::Arc, time::Duration};
-use temporalio_common::telemetry::{
-    HistogramBucketOverrides, MetricTemporality, OtelCollectorOptions, OtlpProtocol,
-    metrics::{
-        CoreMeter, Counter, Gauge, GaugeF64, Histogram, HistogramBase, HistogramDuration,
-        HistogramDurationBase, HistogramF64, HistogramF64Base, MetricAttributable,
-        MetricAttributes, MetricParameters, NewAttributes,
-    },
-};
 use tonic::{metadata::MetadataMap, transport::ClientTlsConfig};
 
 fn histo_view(

package/sdk-core/crates/{sdk-core → common}/src/telemetry/prometheus_meter.rs

@@ -1,4 +1,12 @@
-use crate::{abstractions::dbg_panic, telemetry::default_buckets_for};
+use crate::{
+    dbg_panic,
+    telemetry::metrics::{
+        CoreMeter, Counter, CounterBase, Gauge, GaugeBase, GaugeF64, GaugeF64Base, Histogram,
+        HistogramBase, HistogramDuration, HistogramDurationBase, HistogramF64, HistogramF64Base,
+        MetricAttributable, MetricAttributes, MetricParameters, NewAttributes, OrderedPromLabelSet,
+        default_buckets_for,
+    },
+};
 use anyhow::anyhow;
 use parking_lot::RwLock;
 use prometheus::{
@@ -12,11 +20,6 @@ use std::{
     sync::Arc,
     time::Duration,
 };
-use temporalio_common::telemetry::metrics::{
-    CoreMeter, Counter, CounterBase, Gauge, GaugeBase, GaugeF64, GaugeF64Base, Histogram,
-    HistogramBase, HistogramDuration, HistogramDurationBase, HistogramF64, HistogramF64Base,
-    MetricAttributable, MetricAttributes, MetricParameters, NewAttributes, OrderedPromLabelSet,
-};
 
 #[derive(derive_more::From, derive_more::TryInto, Debug, Clone)]
 enum PromCollector {
@@ -459,7 +462,7 @@ pub struct CorePrometheusMeter {
     registry: Registry,
     use_seconds_for_durations: bool,
     unit_suffix: bool,
-    bucket_overrides: temporalio_common::telemetry::HistogramBucketOverrides,
+    bucket_overrides: crate::telemetry::HistogramBucketOverrides,
 }
 
 impl CorePrometheusMeter {
@@ -467,7 +470,7 @@ impl CorePrometheusMeter {
         registry: Registry,
         use_seconds_for_durations: bool,
         unit_suffix: bool,
-        bucket_overrides: temporalio_common::telemetry::HistogramBucketOverrides,
+        bucket_overrides: crate::telemetry::HistogramBucketOverrides,
     ) -> Self {
         Self {
             registry,
@@ -636,12 +639,11 @@ impl MetricAttributable<Box<dyn HistogramDurationBase>> for DurationHistogram {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::telemetry::{TelemetryInstance, metrics::MetricsContext};
-    use prometheus::{Encoder, TextEncoder};
-    use temporalio_common::telemetry::{
-        METRIC_PREFIX,
-        metrics::{MetricKeyValue, NewAttributes},
+    use crate::telemetry::{
+        HistogramBucketOverrides, TELEM_SERVICE_NAME,
+        metrics::{MetricKeyValue, NewAttributes, WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME},
     };
+    use prometheus::{Encoder, TextEncoder};
 
     #[test]
     fn test_prometheus_meter_dynamic_labels() {
@@ -650,7 +652,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporalio_common::telemetry::HistogramBucketOverrides::default(),
+            HistogramBucketOverrides::default(),
         );
 
         let counter = meter.counter(MetricParameters {
@@ -690,7 +692,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporalio_common::telemetry::HistogramBucketOverrides::default(),
+            HistogramBucketOverrides::default(),
         );
 
         let base_attrs = meter.new_attributes(NewAttributes::new(vec![
@@ -729,15 +731,11 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporalio_common::telemetry::HistogramBucketOverrides::default(),
+            HistogramBucketOverrides::default(),
         );
 
         let histogram_ms = meter_ms.histogram_duration(MetricParameters {
-            name: format!(
-                "temporal_{}",
-                crate::telemetry::WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME
-            )
-            .into(),
+            name: format!("temporal_{}", WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME).into(),
             description: "Test workflow e2e latency".into(),
             unit: "duration".into(),
         });
@@ -759,15 +757,11 @@ mod tests {
             registry_s.clone(),
             true,
             false,
-            temporalio_common::telemetry::HistogramBucketOverrides::default(),
+            HistogramBucketOverrides::default(),
         );
 
         let histogram_s = meter_s.histogram_duration(MetricParameters {
-            name: format!(
-                "temporal_{}",
-                crate::telemetry::WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME
-            )
-            .into(),
+            name: format!("temporal_{}", WORKFLOW_E2E_LATENCY_HISTOGRAM_NAME).into(),
             description: "Test workflow e2e latency".into(),
             unit: "duration".into(),
         });
@@ -791,7 +785,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporalio_common::telemetry::HistogramBucketOverrides::default(),
+            HistogramBucketOverrides::default(),
         );
         let counter = meter.counter(MetricParameters {
             name: "no_labels".into(),
@@ -806,35 +800,40 @@ mod tests {
     }
 
     #[test]
-    fn works_with_recreated_metrics_context() {
-        let registry = Registry::new(HashMap::new());
+    fn test_overlapping_fields() {
+        let registry = Registry::new(HashMap::from([(
+            "service_name".to_string(),
+            TELEM_SERVICE_NAME.to_string(),
+        )]));
         let meter = CorePrometheusMeter::new(
             registry.clone(),
             false,
             false,
-            temporalio_common::telemetry::HistogramBucketOverrides::default(),
-        );
-        let telem_instance = TelemetryInstance::new(
-            None,
-            None,
-            METRIC_PREFIX.to_string(),
-            Some(Arc::new(meter)),
-            true,
-            temporalio_common::telemetry::TaskQueueLabelStrategy::UseNormal,
+            HistogramBucketOverrides::default(),
         );
-        let mc = MetricsContext::top_level("foo".to_string(), "q".to_string(), &telem_instance);
-        mc.worker_registered();
-        drop(mc);
 
-        let mc = MetricsContext::top_level("foo".to_string(), "q".to_string(), &telem_instance);
-        mc.worker_registered();
+        let counter = meter.counter(MetricParameters {
+            name: "temporal_worker_start".into(),
+            description: "".into(),
+            unit: "".into(),
+        });
+
+        let attrs1 = meter.new_attributes(NewAttributes::new(vec![
+            MetricKeyValue::new("namespace", "foo"),
+            MetricKeyValue::new("task_queue", "q"),
+        ]));
+        counter.add(2, &attrs1);
 
-        let mc = MetricsContext::top_level("foo".to_string(), "q2".to_string(), &telem_instance);
-        mc.worker_registered();
+        let attrs2 = meter.new_attributes(NewAttributes::new(vec![
+            MetricKeyValue::new("namespace", "foo"),
+            MetricKeyValue::new("task_queue", "q2"),
+        ]));
+        counter.add(1, &attrs2);
 
         let output = output_string(&registry);
-        assert!(output.contains("temporal_worker_start{namespace=\"foo\",service_name=\"temporal-core-sdk\",task_queue=\"q\"} 2"));
-        assert!(output.contains("temporal_worker_start{namespace=\"foo\",service_name=\"temporal-core-sdk\",task_queue=\"q2\"} 1"));
+
+        assert!(output.contains("temporal_worker_start{namespace=\"foo\",task_queue=\"q\",service_name=\"temporal-core-sdk\"} 2"));
+        assert!(output.contains("temporal_worker_start{namespace=\"foo\",task_queue=\"q2\",service_name=\"temporal-core-sdk\"} 1"));
     }
 
     #[test]
@@ -844,7 +843,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporalio_common::telemetry::HistogramBucketOverrides::default(),
+            HistogramBucketOverrides::default(),
         );
         let dashes = meter.counter(MetricParameters {
             name: "dash-in-name".into(),
@@ -865,7 +864,7 @@ mod tests {
             registry.clone(),
             false,
             false,
-            temporalio_common::telemetry::HistogramBucketOverrides::default(),
+            HistogramBucketOverrides::default(),
         );
         let dashes = meter.counter(MetricParameters {
             name: "not@permitted:symbols".into(),

package/sdk-core/crates/{sdk-core → common}/src/telemetry/prometheus_server.rs

@@ -1,4 +1,4 @@
-use crate::telemetry::prometheus_meter::Registry;
+use crate::telemetry::{PrometheusExporterOptions, prometheus_meter::Registry};
 use http_body_util::Full;
 use hyper::{Method, Request, Response, body::Bytes, header::CONTENT_TYPE, service::service_fn};
 use hyper_util::{
@@ -10,7 +10,6 @@ use std::{
     net::{SocketAddr, TcpListener},
     sync::Arc,
 };
-use temporalio_common::telemetry::PrometheusExporterOptions;
 use tokio::{io, task::AbortHandle};
 
 pub struct StartedPromServer {
@@ -20,7 +19,7 @@ pub struct StartedPromServer {
 }
 
 /// Builds and runs a prometheus endpoint which can be scraped by prom instances for metrics export.
-/// Returns the meter that can be used as a [`CoreMeter`](temporalio_common::telemetry::metrics::CoreMeter).
+/// Returns the meter that can be used as a [`CoreMeter`](crate::telemetry::metrics::CoreMeter).
 ///
 /// Requires a Tokio runtime to exist, and will block briefly while binding the server endpoint.
 pub fn start_prometheus_metric_exporter(

package/sdk-core/crates/common/src/telemetry.rs

@@ -1,20 +1,51 @@
 //! Contains tracing/logging and metrics related functionality
 
+/// Metric instrument types and the [`CoreMeter`] trait.
 pub mod metrics;
 
-use crate::telemetry::metrics::CoreMeter;
+#[cfg(feature = "core-based-sdk")]
+mod log_export;
+#[cfg(feature = "otel")]
+mod otel;
+#[cfg(feature = "prometheus")]
+mod prometheus_meter;
+#[cfg(feature = "prometheus")]
+mod prometheus_server;
+
+use crate::telemetry::metrics::{
+    CoreMeter, MetricKeyValue, NewAttributes, PrefixedMetricsMeter, TemporalMeter,
+};
 use std::{
+    cell::RefCell,
     collections::HashMap,
+    env,
     fmt::{Debug, Formatter},
     net::SocketAddr,
-    sync::Arc,
+    sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering},
+    },
     time::{Duration, SystemTime, UNIX_EPOCH},
 };
-use tracing_core::{Level, Subscriber};
+use tracing::{Level, Subscriber};
+use tracing_subscriber::{EnvFilter, Layer, layer::SubscriberExt};
 use url::Url;
 
+#[cfg(feature = "core-based-sdk")]
+use crate::telemetry::log_export::CoreLogConsumerLayer;
+
+#[cfg(feature = "core-based-sdk")]
+pub use log_export::{CoreLogBuffer, CoreLogBufferedConsumer, CoreLogStreamConsumer};
+#[cfg(feature = "otel")]
+pub use otel::build_otlp_metric_exporter;
+#[cfg(feature = "prometheus")]
+pub use prometheus_server::start_prometheus_metric_exporter;
+
+/// The default prefix applied to all Temporal metric names.
 pub static METRIC_PREFIX: &str = "temporal_";
 
+const TELEM_SERVICE_NAME: &str = "temporal-core-sdk";
+
 /// Each core runtime instance has a telemetry subsystem associated with it, this trait defines the
 /// operations that lang might want to perform on that telemetry after it's initialized.
 pub trait CoreTelemetry {
@@ -28,31 +59,30 @@ pub trait CoreTelemetry {
     fn fetch_buffered_logs(&self) -> Vec<CoreLog>;
 }
 
-/// Telemetry configuration options. Construct with [TelemetryOptionsBuilder]
-#[derive(Clone, derive_builder::Builder)]
+/// Telemetry configuration options. Construct with [TelemetryOptions::builder]
+#[derive(Clone, bon::Builder)]
 #[non_exhaustive]
 pub struct TelemetryOptions {
     /// Optional logger - set as None to disable.
-    #[builder(setter(into, strip_option), default)]
+    #[builder(into)]
     pub logging: Option<Logger>,
     /// Optional metrics exporter - set as None to disable.
-    #[builder(setter(into, strip_option), default)]
+    #[builder(into)]
     pub metrics: Option<Arc<dyn CoreMeter>>,
     /// If set true (the default) explicitly attach a `service_name` label to all metrics. Turn this
     /// off if your collection system supports the `target_info` metric from the OpenMetrics spec.
     /// For more, see
     /// [here](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems)
-    #[builder(default = "true")]
+    #[builder(default = true)]
     pub attach_service_name: bool,
     /// A prefix to be applied to all core-created metrics. Defaults to "temporal_".
-    #[builder(default = "METRIC_PREFIX.to_string()")]
+    #[builder(default = METRIC_PREFIX.to_string())]
     pub metric_prefix: String,
     /// If provided, logging config will be ignored and this explicit subscriber will be used for
     /// all logging and traces.
-    #[builder(setter(strip_option), default)]
     pub subscriber_override: Option<Arc<dyn Subscriber + Send + Sync>>,
     /// See [TaskQueueLabelStrategy].
-    #[builder(default = "TaskQueueLabelStrategy::UseNormal")]
+    #[builder(default = TaskQueueLabelStrategy::UseNormal)]
     pub task_queue_label_strategy: TaskQueueLabelStrategy,
 }
 impl Debug for TelemetryOptions {
@@ -96,19 +126,19 @@ pub enum TaskQueueLabelStrategy {
 }
 
 /// Options for exporting to an OpenTelemetry Collector
-#[derive(Debug, Clone, derive_builder::Builder)]
+#[derive(Debug, Clone, bon::Builder)]
 pub struct OtelCollectorOptions {
     /// The url of the OTel collector to export telemetry and metrics to. Lang SDK should also
     /// export to this same collector.
     pub url: Url,
     /// Optional set of HTTP headers to send to the Collector, e.g for authentication.
-    #[builder(default = "HashMap::new()")]
+    #[builder(default = HashMap::new())]
     pub headers: HashMap<String, String>,
     /// Optionally specify how frequently metrics should be exported. Defaults to 1 second.
-    #[builder(default = "Duration::from_secs(1)")]
+    #[builder(default = Duration::from_secs(1))]
     pub metric_periodicity: Duration,
     /// Specifies the aggregation temporality for metric export. Defaults to cumulative.
-    #[builder(default = "MetricTemporality::Cumulative")]
+    #[builder(default = MetricTemporality::Cumulative)]
     pub metric_temporality: MetricTemporality,
     /// A map of tags to be applied to all metrics
     #[builder(default)]
@@ -120,15 +150,16 @@ pub struct OtelCollectorOptions {
     #[builder(default)]
     pub histogram_bucket_overrides: HistogramBucketOverrides,
     /// Protocol to use for communication with the collector
-    #[builder(default = "OtlpProtocol::Grpc")]
+    #[builder(default = OtlpProtocol::Grpc)]
     pub protocol: OtlpProtocol,
 }
 
 /// Options for exporting metrics to Prometheus
-#[derive(Debug, Clone, derive_builder::Builder)]
+#[derive(Debug, Clone, bon::Builder)]
 pub struct PrometheusExporterOptions {
+    /// The address the Prometheus exporter HTTP server will bind to.
     pub socket_addr: SocketAddr,
-    // A map of tags to be applied to all metrics
+    /// A map of tags to be applied to all metrics
     #[builder(default)]
     pub global_tags: HashMap<String, String>,
     /// If set true, all counters will include a "_total" suffix
@@ -169,11 +200,13 @@ pub enum Logger {
         /// An [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html) filter string.
         filter: String,
     },
+    #[cfg(feature = "core-based-sdk")]
     /// Forward logs to Lang - collectable with `fetch_global_buffered_logs`.
     Forward {
         /// An [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html) filter string.
         filter: String,
     },
+    #[cfg(feature = "core-based-sdk")]
     /// Push logs to Lang. Can be used with
     /// temporalio_sdk_core::telemetry::log_export::CoreLogBufferedConsumer to buffer.
     Push {
@@ -205,7 +238,7 @@ pub enum OtlpProtocol {
 
 impl Default for TelemetryOptions {
     fn default() -> Self {
-        TelemetryOptionsBuilder::default().build().unwrap()
+        TelemetryOptions::builder().build()
     }
 }
 
@@ -243,3 +276,229 @@ pub trait CoreLogConsumer: Send + Sync + Debug {
     /// Invoked synchronously for every single log.
     fn on_log(&self, log: CoreLog);
 }
+
+#[cfg(feature = "core-based-sdk")]
+const FORWARD_LOG_BUFFER_SIZE: usize = 2048;
+
+/// Help you construct an [EnvFilter] compatible filter string which will forward all core module
+/// traces at `core_level` and all others (from 3rd party modules, etc) at `other_level`.
+pub fn construct_filter_string(core_level: Level, other_level: Level) -> String {
+    format!(
+        "{other_level},temporalio_common={core_level},temporalio_sdk_core={core_level},temporalio_client={core_level},temporalio_sdk={core_level}"
+    )
+}
+
+/// Holds initialized tracing/metrics exporters, etc
+pub struct TelemetryInstance {
+    metric_prefix: String,
+    #[cfg(feature = "core-based-sdk")]
+    logs_out: Option<parking_lot::Mutex<CoreLogBuffer>>,
+    metrics: Option<Arc<dyn CoreMeter + 'static>>,
+    /// The tracing subscriber which is associated with this telemetry instance. May be `None` if
+    /// the user has not opted into any tracing configuration.
+    trace_subscriber: Option<Arc<dyn Subscriber + Send + Sync>>,
+    attach_service_name: bool,
+    task_queue_label_strategy: TaskQueueLabelStrategy,
+}
+
+impl TelemetryInstance {
+    /// Return the trace subscriber associated with the telemetry options/instance. Can be used
+    /// to manually set the default for a thread or globally using the `tracing` crate, or with
+    /// [set_trace_subscriber_for_current_thread].
+    pub fn trace_subscriber(&self) -> Option<Arc<dyn Subscriber + Send + Sync>> {
+        self.trace_subscriber.clone()
+    }
+
+    /// Some metric meters cannot be initialized until after a tokio runtime has started and after
+    /// other telemetry has initted (ex: prometheus). They can be attached here.
+    pub fn attach_late_init_metrics(&mut self, meter: Arc<dyn CoreMeter + 'static>) {
+        self.metrics = Some(meter);
+    }
+
+    /// Returns our wrapper for metric meters, including the `metric_prefix` from
+    /// [TelemetryOptions]. This should be used to initialize clients or for any other
+    /// temporal-owned metrics. User defined metrics should use [Self::get_metric_meter].
+    pub fn get_temporal_metric_meter(&self) -> Option<TemporalMeter> {
+        self.metrics.clone().map(|m| {
+            let kvs = self.default_kvs();
+            let attribs = NewAttributes::new(kvs);
+            TemporalMeter::new(
+                Arc::new(PrefixedMetricsMeter::new(self.metric_prefix.clone(), m))
+                    as Arc<dyn CoreMeter>,
+                attribs,
+                self.task_queue_label_strategy,
+            )
+        })
+    }
+
+    /// Returns our wrapper for metric meters, including attaching the service name if enabled.
+    pub fn get_metric_meter(&self) -> Option<TemporalMeter> {
+        self.metrics.clone().map(|m| {
+            let kvs = self.default_kvs();
+            let attribs = NewAttributes::new(kvs);
+            TemporalMeter::new(m, attribs, self.task_queue_label_strategy)
+        })
+    }
+
+    fn default_kvs(&self) -> Vec<MetricKeyValue> {
+        if self.attach_service_name {
+            vec![MetricKeyValue::new("service_name", TELEM_SERVICE_NAME)]
+        } else {
+            vec![]
+        }
+    }
+}
+
+thread_local! {
+    static SUB_GUARD: RefCell<Option<tracing::subscriber::DefaultGuard>> =
+        const { RefCell::new(None) };
+}
+/// Set the trace subscriber for the current thread. This must be done in every thread which uses
+/// core stuff, otherwise traces/logs will not be collected on that thread. For example, if using
+/// a multithreaded Tokio runtime, you should ensure that said runtime uses
+/// [on_thread_start](https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html#method.on_thread_start)
+/// or a similar mechanism to call this for each thread within the runtime.
+pub fn set_trace_subscriber_for_current_thread(sub: impl Subscriber + Send + Sync + 'static) {
+    SUB_GUARD.with(|sg| {
+        if sg.borrow().is_none() {
+            let g = tracing::subscriber::set_default(sub);
+            *sg.borrow_mut() = Some(g);
+        }
+    })
+}
+
+/// Undoes [set_trace_subscriber_for_current_thread]
+pub fn remove_trace_subscriber_for_current_thread() {
+    SUB_GUARD.take();
+}
+
+#[cfg(feature = "core-based-sdk")]
+impl CoreTelemetry for TelemetryInstance {
+    fn fetch_buffered_logs(&self) -> Vec<CoreLog> {
+        if let Some(logs_out) = self.logs_out.as_ref() {
+            logs_out.lock().drain()
+        } else {
+            vec![]
+        }
+    }
+}
+
+/// Initialize tracing subscribers/output and logging export, returning a [TelemetryInstance]
+/// which can be used to register default / global tracing subscribers.
+///
+/// You should only call this once per unique [TelemetryOptions]
+///
+/// See [TelemetryOptions] docs for more on configuration.
+pub fn telemetry_init(opts: TelemetryOptions) -> Result<TelemetryInstance, anyhow::Error> {
+    #[cfg(feature = "core-based-sdk")]
+    let mut logs_out = None;
+
+    // Tracing subscriber layers =========
+    let mut console_pretty_layer = None;
+    let mut console_compact_layer = None;
+    #[cfg(feature = "core-based-sdk")]
+    let mut forward_layer = None;
+    // ===================================
+
+    let tracing_sub = if let Some(ts) = opts.subscriber_override {
+        Some(ts)
+    } else {
+        opts.logging.map(|logger| {
+            match logger {
+                Logger::Console { filter } => {
+                    // This is silly dupe but can't be avoided without boxing.
+                    if env::var("TEMPORAL_CORE_PRETTY_LOGS").is_ok() {
+                        console_pretty_layer = Some(
+                            tracing_subscriber::fmt::layer()
+                                .with_target(false)
+                                .event_format(
+                                    tracing_subscriber::fmt::format()
+                                        .pretty()
+                                        .with_source_location(false),
+                                )
+                                .with_filter(EnvFilter::new(filter)),
+                        )
+                    } else {
+                        console_compact_layer = Some(
+                            tracing_subscriber::fmt::layer()
+                                .with_target(false)
+                                .event_format(
+                                    tracing_subscriber::fmt::format()
+                                        .compact()
+                                        .with_source_location(false),
+                                )
+                                .with_filter(EnvFilter::new(filter)),
+                        )
+                    }
+                }
+                #[cfg(feature = "core-based-sdk")]
+                Logger::Forward { filter } => {
+                    let (export_layer, lo) =
+                        CoreLogConsumerLayer::new_buffered(FORWARD_LOG_BUFFER_SIZE);
+                    logs_out = Some(parking_lot::Mutex::new(lo));
+                    forward_layer = Some(export_layer.with_filter(EnvFilter::new(filter)));
+                }
+                #[cfg(feature = "core-based-sdk")]
+                Logger::Push { filter, consumer } => {
+                    forward_layer = Some(
+                        CoreLogConsumerLayer::new(consumer).with_filter(EnvFilter::new(filter)),
+                    );
+                }
+            };
+            let reg = tracing_subscriber::registry()
+                .with(console_pretty_layer)
+                .with(console_compact_layer);
+            #[cfg(feature = "core-based-sdk")]
+            let reg = reg.with(forward_layer);
+
+            Arc::new(reg) as Arc<dyn Subscriber + Send + Sync>
+        })
+    };
+
+    Ok(TelemetryInstance {
+        metric_prefix: opts.metric_prefix,
+        #[cfg(feature = "core-based-sdk")]
+        logs_out,
+        metrics: opts.metrics,
+        trace_subscriber: tracing_sub,
+        attach_service_name: opts.attach_service_name,
+        task_queue_label_strategy: opts.task_queue_label_strategy,
+    })
+}
+
+/// WARNING: Calling can cause panics because of <https://github.com/tokio-rs/tracing/issues/1656>
+/// Lang must not start using until resolved
+///
+/// Initialize telemetry/tracing globally. Useful for testing. Only takes affect when called
+/// the first time. Subsequent calls are ignored.
+pub fn telemetry_init_global(opts: TelemetryOptions) -> Result<(), anyhow::Error> {
+    static INITTED: AtomicBool = AtomicBool::new(false);
+    if INITTED
+        .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
+        .is_ok()
+    {
+        let ti = telemetry_init(opts)?;
+        if let Some(ts) = ti.trace_subscriber() {
+            tracing::subscriber::set_global_default(ts)?;
+        }
+    }
+    Ok(())
+}
+
+/// WARNING: Calling can cause panics because of <https://github.com/tokio-rs/tracing/issues/1656>
+/// Lang must not start using until resolved
+///
+/// Initialize the fallback global handler. All lang SDKs should call this somewhere, once, at
+/// startup, as it initializes a fallback handler for any dependencies (looking at you, otel) that
+/// don't provide good ways to customize their tracing usage. It sets a WARN-level global filter
+/// that uses the default console logger.
+pub fn telemetry_init_fallback() -> Result<(), anyhow::Error> {
+    telemetry_init_global(
+        TelemetryOptions::builder()
+            .logging(Logger::Console {
+                filter: construct_filter_string(Level::DEBUG, Level::WARN),
+            })
+            .build(),
+    )?;
+    Ok(())
+}