@temporalio/core-bridge 1.9.2 → 1.10.0

package/sdk-core/core/src/telemetry/metrics.rs

@@ -1,37 +1,18 @@
-use crate::{
-    abstractions::dbg_panic,
-    telemetry::{
-        default_resource, metric_temporality_to_selector, prometheus_server::PromServer,
-        TelemetryInstance, TELEM_SERVICE_NAME,
-    },
-};
-use opentelemetry::{
-    self,
-    metrics::{Meter, MeterProvider as MeterProviderT, Unit},
-    KeyValue,
-};
-use opentelemetry_otlp::WithExportConfig;
-use opentelemetry_sdk::{
-    metrics::{
-        new_view,
-        reader::{AggregationSelector, DefaultAggregationSelector},
-        Aggregation, Instrument, InstrumentKind, MeterProvider, MeterProviderBuilder,
-        PeriodicReader, View,
-    },
-    runtime, AttributeSet,
+use crate::{abstractions::dbg_panic, telemetry::TelemetryInstance};
+
+use std::{
+    fmt::{Debug, Display},
+    iter::Iterator,
+    sync::Arc,
+    time::Duration,
 };
-use parking_lot::RwLock;
-use std::{collections::HashMap, fmt::Debug, net::SocketAddr, sync::Arc, time::Duration};
-use temporal_sdk_core_api::telemetry::{
-    metrics::{
-        BufferAttributes, BufferInstrumentRef, CoreMeter, Counter, Gauge, Histogram,
-        LazyBufferInstrument, MetricAttributes, MetricCallBufferer, MetricEvent, MetricKeyValue,
-        MetricKind, MetricParameters, MetricUpdateVal, NewAttributes, NoOpCoreMeter,
-    },
-    OtelCollectorOptions, PrometheusExporterOptions,
+use temporal_sdk_core_api::telemetry::metrics::{
+    BufferAttributes, BufferInstrumentRef, CoreMeter, Counter, Gauge, GaugeF64, Histogram,
+    HistogramDuration, HistogramF64, LazyBufferInstrument, MetricAttributes, MetricCallBufferer,
+    MetricEvent, MetricKeyValue, MetricKind, MetricParameters, MetricUpdateVal, NewAttributes,
+    NoOpCoreMeter,
 };
-use tokio::task::AbortHandle;
-use tonic::metadata::MetadataMap;
+use temporal_sdk_core_protos::temporal::api::enums::v1::WorkflowTaskFailedCause;
 
 /// Used to track context associated with metrics, and record/update them
 ///
@@ -49,21 +30,22 @@ struct Instruments {
     wf_canceled_counter: Arc<dyn Counter>,
     wf_failed_counter: Arc<dyn Counter>,
     wf_cont_counter: Arc<dyn Counter>,
-    wf_e2e_latency: Arc<dyn Histogram>,
+    wf_e2e_latency: Arc<dyn HistogramDuration>,
     wf_task_queue_poll_empty_counter: Arc<dyn Counter>,
     wf_task_queue_poll_succeed_counter: Arc<dyn Counter>,
     wf_task_execution_failure_counter: Arc<dyn Counter>,
-    wf_task_sched_to_start_latency: Arc<dyn Histogram>,
-    wf_task_replay_latency: Arc<dyn Histogram>,
-    wf_task_execution_latency: Arc<dyn Histogram>,
+    wf_task_sched_to_start_latency: Arc<dyn HistogramDuration>,
+    wf_task_replay_latency: Arc<dyn HistogramDuration>,
+    wf_task_execution_latency: Arc<dyn HistogramDuration>,
     act_poll_no_task: Arc<dyn Counter>,
     act_task_received_counter: Arc<dyn Counter>,
     act_execution_failed: Arc<dyn Counter>,
-    act_sched_to_start_latency: Arc<dyn Histogram>,
-    act_exec_latency: Arc<dyn Histogram>,
+    act_sched_to_start_latency: Arc<dyn HistogramDuration>,
+    act_exec_latency: Arc<dyn HistogramDuration>,
     worker_registered: Arc<dyn Counter>,
     num_pollers: Arc<dyn Gauge>,
     task_slots_available: Arc<dyn Gauge>,
+    task_slots_used: Arc<dyn Gauge>,
     sticky_cache_hit: Arc<dyn Counter>,
     sticky_cache_miss: Arc<dyn Counter>,
     sticky_cache_size: Arc<dyn Gauge>,
@@ -156,30 +138,28 @@ impl MetricsContext {
 
     /// Record workflow total execution time in milliseconds
     pub(crate) fn wf_e2e_latency(&self, dur: Duration) {
-        self.instruments
-            .wf_e2e_latency
-            .record(dur.as_millis() as u64, &self.kvs);
+        self.instruments.wf_e2e_latency.record(dur, &self.kvs);
     }
 
     /// Record workflow task schedule to start time in millis
     pub(crate) fn wf_task_sched_to_start_latency(&self, dur: Duration) {
         self.instruments
             .wf_task_sched_to_start_latency
-            .record(dur.as_millis() as u64, &self.kvs);
+            .record(dur, &self.kvs);
     }
 
     /// Record workflow task execution time in milliseconds
     pub(crate) fn wf_task_latency(&self, dur: Duration) {
         self.instruments
             .wf_task_execution_latency
-            .record(dur.as_millis() as u64, &self.kvs);
+            .record(dur, &self.kvs);
     }
 
     /// Record time it takes to catch up on replaying a WFT
     pub(crate) fn wf_task_replay_latency(&self, dur: Duration) {
         self.instruments
             .wf_task_replay_latency
-            .record(dur.as_millis() as u64, &self.kvs);
+            .record(dur, &self.kvs);
     }
 
     /// An activity long poll timed out
@@ -201,15 +181,13 @@ impl MetricsContext {
     pub(crate) fn act_sched_to_start_latency(&self, dur: Duration) {
         self.instruments
             .act_sched_to_start_latency
-            .record(dur.as_millis() as u64, &self.kvs);
+            .record(dur, &self.kvs);
     }
 
     /// Record time it took to complete activity execution, from the time core generated the
     /// activity task, to the time lang responded with a completion (failure or success).
     pub(crate) fn act_execution_latency(&self, dur: Duration) {
-        self.instruments
-            .act_exec_latency
-            .record(dur.as_millis() as u64, &self.kvs);
+        self.instruments.act_exec_latency.record(dur, &self.kvs);
     }
 
     /// A worker was registered
@@ -224,6 +202,11 @@ impl MetricsContext {
             .record(num as u64, &self.kvs)
     }
 
+    /// Record current number of used task slots. Context should have worker type set.
+    pub(crate) fn task_slots_used(&self, num: u64) {
+        self.instruments.task_slots_used.record(num, &self.kvs)
+    }
+
     /// Record current number of pollers. Context should include poller type / task queue tag.
     pub(crate) fn record_num_pollers(&self, num: usize) {
         self.instruments.num_pollers.record(num as u64, &self.kvs);
@@ -273,9 +256,9 @@ impl Instruments {
                 description: "Count of continued-as-new workflows".into(),
                 unit: "".into(),
             }),
-            wf_e2e_latency: meter.histogram(MetricParameters {
+            wf_e2e_latency: meter.histogram_duration(MetricParameters {
                 name: WF_E2E_LATENCY_NAME.into(),
-                unit: "ms".into(),
+                unit: "duration".into(),
                 description: "Histogram of total workflow execution latencies".into(),
             }),
             wf_task_queue_poll_empty_counter: meter.counter(MetricParameters {
@@ -293,19 +276,19 @@ impl Instruments {
                 description: "Count of workflow task execution failures".into(),
                 unit: "".into(),
             }),
-            wf_task_sched_to_start_latency: meter.histogram(MetricParameters {
+            wf_task_sched_to_start_latency: meter.histogram_duration(MetricParameters {
                 name: WF_TASK_SCHED_TO_START_LATENCY_NAME.into(),
-                unit: "ms".into(),
+                unit: "duration".into(),
                 description: "Histogram of workflow task schedule-to-start latencies".into(),
             }),
-            wf_task_replay_latency: meter.histogram(MetricParameters {
+            wf_task_replay_latency: meter.histogram_duration(MetricParameters {
                 name: WF_TASK_REPLAY_LATENCY_NAME.into(),
-                unit: "ms".into(),
+                unit: "duration".into(),
                 description: "Histogram of workflow task replay latencies".into(),
             }),
-            wf_task_execution_latency: meter.histogram(MetricParameters {
+            wf_task_execution_latency: meter.histogram_duration(MetricParameters {
                 name: WF_TASK_EXECUTION_LATENCY_NAME.into(),
-                unit: "ms".into(),
+                unit: "duration".into(),
                 description: "Histogram of workflow task execution (not replay) latencies".into(),
             }),
             act_poll_no_task: meter.counter(MetricParameters {
@@ -323,14 +306,14 @@ impl Instruments {
                 description: "Count of activity task execution failures".into(),
                 unit: "".into(),
             }),
-            act_sched_to_start_latency: meter.histogram(MetricParameters {
+            act_sched_to_start_latency: meter.histogram_duration(MetricParameters {
                 name: ACT_SCHED_TO_START_LATENCY_NAME.into(),
-                unit: "ms".into(),
+                unit: "duration".into(),
                 description: "Histogram of activity schedule-to-start latencies".into(),
             }),
-            act_exec_latency: meter.histogram(MetricParameters {
+            act_exec_latency: meter.histogram_duration(MetricParameters {
                 name: ACT_EXEC_LATENCY_NAME.into(),
-                unit: "ms".into(),
+                unit: "duration".into(),
                 description: "Histogram of activity execution latencies".into(),
             }),
             // name kept as worker start for compat with old sdk / what users expect
@@ -349,6 +332,11 @@ impl Instruments {
                 description: "Current number of available slots per task type".into(),
                 unit: "".into(),
             }),
+            task_slots_used: meter.gauge(MetricParameters {
+                name: TASK_SLOTS_USED_NAME.into(),
+                description: "Current number of used slots per task type".into(),
+                unit: "".into(),
+            }),
             sticky_cache_hit: meter.counter(MetricParameters {
                 name: "sticky_cache_hit".into(),
                 description: "Count of times the workflow cache was used for a new workflow task"
@@ -383,6 +371,7 @@ const KEY_ACT_TYPE: &str = "activity_type";
 const KEY_POLLER_TYPE: &str = "poller_type";
 const KEY_WORKER_TYPE: &str = "worker_type";
 const KEY_EAGER: &str = "eager";
+const KEY_TASK_FAILURE_TYPE: &str = "failure_reason";
 
 pub(crate) fn workflow_poller() -> MetricKeyValue {
     MetricKeyValue::new(KEY_POLLER_TYPE, "workflow_task")
@@ -414,217 +403,115 @@ pub(crate) fn local_activity_worker_type() -> MetricKeyValue {
 pub(crate) fn eager(is_eager: bool) -> MetricKeyValue {
     MetricKeyValue::new(KEY_EAGER, is_eager)
 }
-
-const WF_E2E_LATENCY_NAME: &str = "workflow_endtoend_latency";
-const WF_TASK_SCHED_TO_START_LATENCY_NAME: &str = "workflow_task_schedule_to_start_latency";
-const WF_TASK_REPLAY_LATENCY_NAME: &str = "workflow_task_replay_latency";
-const WF_TASK_EXECUTION_LATENCY_NAME: &str = "workflow_task_execution_latency";
-const ACT_SCHED_TO_START_LATENCY_NAME: &str = "activity_schedule_to_start_latency";
-const ACT_EXEC_LATENCY_NAME: &str = "activity_execution_latency";
-const NUM_POLLERS_NAME: &str = "num_pollers";
-const TASK_SLOTS_AVAILABLE_NAME: &str = "worker_task_slots_available";
-const STICKY_CACHE_SIZE_NAME: &str = "sticky_cache_size";
-
-/// Artisanal, handcrafted latency buckets for workflow e2e latency which should expose a useful
-/// set of buckets for < 1 day runtime workflows. Beyond that, this metric probably isn't very
-/// helpful
-static WF_LATENCY_MS_BUCKETS: &[f64] = &[
-    100.,
-    500.,
-    1000.,
-    1500.,
-    2000.,
-    5000.,
-    10_000.,
-    30_000.,
-    60_000.,
-    120_000.,
-    300_000.,
-    600_000.,
-    1_800_000.,  // 30 min
-    3_600_000.,  //  1 hr
-    30_600_000., // 10 hrs
-    8.64e7,      // 24 hrs
-];
-
-/// Task latencies are expected to be fast, no longer than a second which was generally the deadlock
-/// timeout in old SDKs. Here it's a bit different since a WFT may represent multiple activations.
-static WF_TASK_MS_BUCKETS: &[f64] = &[1., 10., 20., 50., 100., 200., 500., 1000.];
-
-/// Activity are generally expected to take at least a little time, and sometimes quite a while,
-/// since they're doing side-effecty things, etc.
-static ACT_EXE_MS_BUCKETS: &[f64] = &[50., 100., 500., 1000., 5000., 10_000., 60_000.];
-
-/// Schedule-to-start latency buckets for both WFT and AT
-static TASK_SCHED_TO_START_MS_BUCKETS: &[f64] =
-    &[100., 500., 1000., 5000., 10_000., 100_000., 1_000_000.];
-
-/// Default buckets. Should never really be used as they will be meaningless for many things, but
-/// broadly it's trying to represent latencies in millis.
-pub(super) static DEFAULT_MS_BUCKETS: &[f64] = &[50., 100., 500., 1000., 2500., 10_000.];
-
-/// Returns the default histogram buckets that lang should use for a given metric name if they
-/// have not been overridden by the user.
-///
-/// The name must *not* be prefixed with `temporal_`
-pub fn default_buckets_for(histo_name: &str) -> &'static [f64] {
-    match histo_name {
-        WF_E2E_LATENCY_NAME => WF_LATENCY_MS_BUCKETS,
-        WF_TASK_EXECUTION_LATENCY_NAME | WF_TASK_REPLAY_LATENCY_NAME => WF_TASK_MS_BUCKETS,
-        WF_TASK_SCHED_TO_START_LATENCY_NAME | ACT_SCHED_TO_START_LATENCY_NAME => {
-            TASK_SCHED_TO_START_MS_BUCKETS
-        }
-        ACT_EXEC_LATENCY_NAME => ACT_EXE_MS_BUCKETS,
-        _ => DEFAULT_MS_BUCKETS,
-    }
-}
-
-/// Chooses appropriate aggregators for our metrics
-#[derive(Debug, Clone, Default)]
-pub struct SDKAggSelector {
-    default: DefaultAggregationSelector,
+pub(crate) enum FailureReason {
+    Nondeterminism,
+    Workflow,
 }
-impl AggregationSelector for SDKAggSelector {
-    fn aggregation(&self, kind: InstrumentKind) -> Aggregation {
-        match kind {
-            InstrumentKind::Histogram => Aggregation::ExplicitBucketHistogram {
-                boundaries: DEFAULT_MS_BUCKETS.to_vec(),
-                record_min_max: true,
-            },
-            _ => self.default.aggregation(kind),
-        }
+impl Display for FailureReason {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let str = match self {
+            FailureReason::Nondeterminism => "NonDeterminismError",
+            FailureReason::Workflow => "WorkflowError",
+        };
+        write!(f, "{}", str)
     }
 }
-
-fn histo_view(
-    metric_name: &'static str,
-    buckets: &[f64],
-) -> opentelemetry::metrics::Result<Box<dyn View>> {
-    new_view(
-        Instrument::new().name(format!("*{metric_name}")),
-        opentelemetry_sdk::metrics::Stream::new().aggregation(
-            Aggregation::ExplicitBucketHistogram {
-                boundaries: buckets.to_vec(),
-                record_min_max: true,
-            },
-        ),
-    )
-}
-
-pub(super) fn augment_meter_provider_with_defaults(
-    mpb: MeterProviderBuilder,
-    global_tags: &HashMap<String, String>,
-) -> opentelemetry::metrics::Result<MeterProviderBuilder> {
-    // Some histograms are actually gauges, but we have to use histograms otherwise they forget
-    // their value between collections since we don't use callbacks.
-    Ok(mpb
-        .with_view(histo_view(WF_E2E_LATENCY_NAME, WF_LATENCY_MS_BUCKETS)?)
-        .with_view(histo_view(
-            WF_TASK_EXECUTION_LATENCY_NAME,
-            WF_TASK_MS_BUCKETS,
-        )?)
-        .with_view(histo_view(WF_TASK_REPLAY_LATENCY_NAME, WF_TASK_MS_BUCKETS)?)
-        .with_view(histo_view(
-            WF_TASK_SCHED_TO_START_LATENCY_NAME,
-            TASK_SCHED_TO_START_MS_BUCKETS,
-        )?)
-        .with_view(histo_view(
-            ACT_SCHED_TO_START_LATENCY_NAME,
-            TASK_SCHED_TO_START_MS_BUCKETS,
-        )?)
-        .with_view(histo_view(ACT_EXEC_LATENCY_NAME, ACT_EXE_MS_BUCKETS)?)
-        .with_resource(default_resource(global_tags)))
-}
-
-/// OTel has no built-in synchronous Gauge. Histograms used to be able to serve that purpose, but
-/// they broke that. Lovely. So, we need to implement one by hand.
-pub(crate) struct MemoryGaugeU64 {
-    labels_to_values: Arc<RwLock<HashMap<AttributeSet, u64>>>,
-}
-
-impl MemoryGaugeU64 {
-    fn new(params: MetricParameters, meter: &Meter) -> Self {
-        let gauge = meter
-            .u64_observable_gauge(params.name)
-            .with_unit(Unit::new(params.unit))
-            .with_description(params.description)
-            .init();
-        let map = Arc::new(RwLock::new(HashMap::<AttributeSet, u64>::new()));
-        let map_c = map.clone();
-        meter
-            .register_callback(&[gauge.as_any()], move |o| {
-                // This whole thing is... extra stupid.
-                // See https://github.com/open-telemetry/opentelemetry-rust/issues/1181
-                // The performance is likely bad here, but, given this is only called when metrics
-                // are exported it should be livable for now.
-                let map_rlock = map_c.read();
-                for (kvs, val) in map_rlock.iter() {
-                    let kvs: Vec<_> = kvs
-                        .iter()
-                        .map(|(k, v)| KeyValue::new(k.clone(), v.clone()))
-                        .collect();
-                    o.observe_u64(&gauge, *val, kvs.as_slice())
-                }
-            })
-            .expect("instrument must exist we just created it");
-        MemoryGaugeU64 {
-            labels_to_values: map,
+impl From<WorkflowTaskFailedCause> for FailureReason {
+    fn from(v: WorkflowTaskFailedCause) -> Self {
+        match v {
+            WorkflowTaskFailedCause::NonDeterministicError => FailureReason::Nondeterminism,
+            _ => FailureReason::Workflow,
         }
     }
-    fn record(&self, val: u64, kvs: &[KeyValue]) {
-        self.labels_to_values
-            .write()
-            .insert(AttributeSet::from(kvs), val);
-    }
-}
-
-/// Create an OTel meter that can be used as a [CoreMeter] to export metrics over OTLP.
-pub fn build_otlp_metric_exporter(
-    opts: OtelCollectorOptions,
-) -> Result<CoreOtelMeter, anyhow::Error> {
-    let exporter = opentelemetry_otlp::TonicExporterBuilder::default()
-        .with_endpoint(opts.url.to_string())
-        .with_metadata(MetadataMap::from_headers((&opts.headers).try_into()?))
-        .build_metrics_exporter(
-            Box::<SDKAggSelector>::default(),
-            Box::new(metric_temporality_to_selector(opts.metric_temporality)),
-        )?;
-    let reader = PeriodicReader::builder(exporter, runtime::Tokio)
-        .with_interval(opts.metric_periodicity)
-        .build();
-    let mp = augment_meter_provider_with_defaults(
-        MeterProvider::builder().with_reader(reader),
-        &opts.global_tags,
-    )?
-    .build();
-    Ok::<_, anyhow::Error>(CoreOtelMeter(mp.meter(TELEM_SERVICE_NAME)))
 }
-
-pub struct StartedPromServer {
-    pub meter: Arc<CoreOtelMeter>,
-    pub bound_addr: SocketAddr,
-    pub abort_handle: AbortHandle,
+pub(crate) fn failure_reason(reason: FailureReason) -> MetricKeyValue {
+    MetricKeyValue::new(KEY_TASK_FAILURE_TYPE, reason.to_string())
+}
+
+pub(super) const WF_E2E_LATENCY_NAME: &str = "workflow_endtoend_latency";
+pub(super) const WF_TASK_SCHED_TO_START_LATENCY_NAME: &str =
+    "workflow_task_schedule_to_start_latency";
+pub(super) const WF_TASK_REPLAY_LATENCY_NAME: &str = "workflow_task_replay_latency";
+pub(super) const WF_TASK_EXECUTION_LATENCY_NAME: &str = "workflow_task_execution_latency";
+pub(super) const ACT_SCHED_TO_START_LATENCY_NAME: &str = "activity_schedule_to_start_latency";
+pub(super) const ACT_EXEC_LATENCY_NAME: &str = "activity_execution_latency";
+pub(super) const NUM_POLLERS_NAME: &str = "num_pollers";
+pub(super) const TASK_SLOTS_AVAILABLE_NAME: &str = "worker_task_slots_available";
+pub(super) const TASK_SLOTS_USED_NAME: &str = "worker_task_slots_used";
+pub(super) const STICKY_CACHE_SIZE_NAME: &str = "sticky_cache_size";
+
+/// Helps define buckets once in terms of millis, but also generates a seconds version
+macro_rules! define_latency_buckets {
+    ($(($metric_name:pat, $name:ident, $sec_name:ident, [$($bucket:expr),*])),*) => {
+        $(
+            pub(super) static $name: &[f64] = &[$($bucket,)*];
+            pub(super) static $sec_name: &[f64] = &[$( $bucket / 1000.0, )*];
+        )*
+
+        /// Returns the default histogram buckets that lang should use for a given metric name if
+        /// they have not been overridden by the user. If `use_seconds` is true, returns buckets
+        /// in terms of seconds rather than milliseconds.
+        ///
+        /// The name must *not* be prefixed with `temporal_`
+        pub fn default_buckets_for(histo_name: &str, use_seconds: bool) -> &'static [f64] {
+            match histo_name {
+                $(
+                    $metric_name => { if use_seconds { &$sec_name } else { &$name } },
+                )*
+            }
+        }
+    };
 }
 
-/// Builds and runs a prometheus endpoint which can be scraped by prom instances for metrics export.
-/// Returns the meter that can be used as a [CoreMeter].
-pub fn start_prometheus_metric_exporter(
-    opts: PrometheusExporterOptions,
-) -> Result<StartedPromServer, anyhow::Error> {
-    let (srv, exporter) = PromServer::new(&opts, SDKAggSelector::default())?;
-    let meter_provider = augment_meter_provider_with_defaults(
-        MeterProvider::builder().with_reader(exporter),
-        &opts.global_tags,
-    )?
-    .build();
-    let bound_addr = srv.bound_addr();
-    let handle = tokio::spawn(async move { srv.run().await });
-    Ok(StartedPromServer {
-        meter: Arc::new(CoreOtelMeter(meter_provider.meter(TELEM_SERVICE_NAME))),
-        bound_addr,
-        abort_handle: handle.abort_handle(),
-    })
-}
+define_latency_buckets!(
+    (
+        WF_E2E_LATENCY_NAME,
+        WF_LATENCY_MS_BUCKETS,
+        WF_LATENCY_S_BUCKETS,
+        [
+            100.,
+            500.,
+            1000.,
+            1500.,
+            2000.,
+            5000.,
+            10_000.,
+            30_000.,
+            60_000.,
+            120_000.,
+            300_000.,
+            600_000.,
+            1_800_000.,  // 30 min
+            3_600_000.,  //  1 hr
+            30_600_000., // 10 hrs
+            8.64e7       // 24 hrs
+        ]
+    ),
+    (
+        WF_TASK_EXECUTION_LATENCY_NAME | WF_TASK_REPLAY_LATENCY_NAME,
+        WF_TASK_MS_BUCKETS,
+        WF_TASK_S_BUCKETS,
+        [1., 10., 20., 50., 100., 200., 500., 1000.]
+    ),
+    (
+        ACT_EXEC_LATENCY_NAME,
+        ACT_EXE_MS_BUCKETS,
+        ACT_EXE_S_BUCKETS,
+        [50., 100., 500., 1000., 5000., 10_000., 60_000.]
+    ),
+    (
+        WF_TASK_SCHED_TO_START_LATENCY_NAME | ACT_SCHED_TO_START_LATENCY_NAME,
+        TASK_SCHED_TO_START_MS_BUCKETS,
+        TASK_SCHED_TO_START_S_BUCKETS,
+        [100., 500., 1000., 5000., 10_000., 100_000., 1_000_000.]
+    ),
+    (
+        _,
+        DEFAULT_MS_BUCKETS,
+        DEFAULT_S_BUCKETS,
+        [50., 100., 500., 1000., 2500., 10_000.]
+    )
+);
 
 /// Buffers [MetricEvent]s for periodic consumption by lang
 #[derive(Debug)]
@@ -668,7 +555,6 @@ where
             populate_into: hole.clone(),
         });
         BufferInstrument {
-            kind,
             instrument_ref: hole,
             tx: self.calls_tx.clone(),
         }
@@ -716,9 +602,21 @@ where
         Arc::new(self.new_instrument(params, MetricKind::Histogram))
     }
 
+    fn histogram_f64(&self, params: MetricParameters) -> Arc<dyn HistogramF64> {
+        Arc::new(self.new_instrument(params, MetricKind::HistogramF64))
+    }
+
+    fn histogram_duration(&self, params: MetricParameters) -> Arc<dyn HistogramDuration> {
+        Arc::new(self.new_instrument(params, MetricKind::HistogramDuration))
+    }
+
     fn gauge(&self, params: MetricParameters) -> Arc<dyn Gauge> {
         Arc::new(self.new_instrument(params, MetricKind::Gauge))
     }
+
+    fn gauge_f64(&self, params: MetricParameters) -> Arc<dyn GaugeF64> {
+        Arc::new(self.new_instrument(params, MetricKind::GaugeF64))
+    }
 }
 impl<I> MetricCallBufferer<I> for MetricsCallBuffer<I>
 where
@@ -730,7 +628,6 @@ where
 }
 
 struct BufferInstrument<I: BufferInstrumentRef> {
-    kind: MetricKind,
     instrument_ref: LazyBufferInstrument<I>,
     tx: LogErrOnFullSender<MetricEvent<I>>,
 }
@@ -738,17 +635,14 @@ impl<I> BufferInstrument<I>
 where
     I: Clone + BufferInstrumentRef,
 {
-    fn send(&self, value: u64, attributes: &MetricAttributes) {
+    fn send(&self, value: MetricUpdateVal, attributes: &MetricAttributes) {
         let attributes = match attributes {
             MetricAttributes::Buffer(l) => l.clone(),
             _ => panic!("MetricsCallBuffer only works with MetricAttributes::Lang"),
         };
         self.tx.send(MetricEvent::Update {
             instrument: self.instrument_ref.clone(),
-            update: match self.kind {
-                MetricKind::Counter => MetricUpdateVal::Delta(value),
-                MetricKind::Gauge | MetricKind::Histogram => MetricUpdateVal::Value(value),
-            },
+            update: value,
             attributes: attributes.clone(),
         });
     }
@@ -758,7 +652,7 @@ where
     I: BufferInstrumentRef + Send + Sync + Clone,
 {
     fn add(&self, value: u64, attributes: &MetricAttributes) {
-        self.send(value, attributes)
+        self.send(MetricUpdateVal::Delta(value), attributes)
     }
 }
 impl<I> Gauge for BufferInstrument<I>
@@ -766,7 +660,15 @@ where
     I: BufferInstrumentRef + Send + Sync + Clone,
 {
     fn record(&self, value: u64, attributes: &MetricAttributes) {
-        self.send(value, attributes)
+        self.send(MetricUpdateVal::Value(value), attributes)
+    }
+}
+impl<I> GaugeF64 for BufferInstrument<I>
+where
+    I: BufferInstrumentRef + Send + Sync + Clone,
+{
+    fn record(&self, value: f64, attributes: &MetricAttributes) {
+        self.send(MetricUpdateVal::ValueF64(value), attributes)
     }
 }
 impl<I> Histogram for BufferInstrument<I>
@@ -774,65 +676,23 @@ where
     I: BufferInstrumentRef + Send + Sync + Clone,
 {
     fn record(&self, value: u64, attributes: &MetricAttributes) {
-        self.send(value, attributes)
+        self.send(MetricUpdateVal::Value(value), attributes)
     }
 }
-
-#[derive(Debug)]
-pub struct CoreOtelMeter(Meter);
-impl CoreMeter for CoreOtelMeter {
-    fn new_attributes(&self, attribs: NewAttributes) -> MetricAttributes {
-        MetricAttributes::OTel {
-            kvs: Arc::new(attribs.attributes.into_iter().map(KeyValue::from).collect()),
-        }
-    }
-
-    fn extend_attributes(
-        &self,
-        existing: MetricAttributes,
-        attribs: NewAttributes,
-    ) -> MetricAttributes {
-        if let MetricAttributes::OTel { mut kvs } = existing {
-            Arc::make_mut(&mut kvs).extend(attribs.attributes.into_iter().map(Into::into));
-            MetricAttributes::OTel { kvs }
-        } else {
-            dbg_panic!("Must use OTel attributes with an OTel metric implementation");
-            existing
-        }
-    }
-
-    fn counter(&self, params: MetricParameters) -> Arc<dyn Counter> {
-        Arc::new(
-            self.0
-                .u64_counter(params.name)
-                .with_unit(Unit::new(params.unit))
-                .with_description(params.description)
-                .init(),
-        )
-    }
-
-    fn histogram(&self, params: MetricParameters) -> Arc<dyn Histogram> {
-        Arc::new(
-            self.0
-                .u64_histogram(params.name)
-                .with_unit(Unit::new(params.unit))
-                .with_description(params.description)
-                .init(),
-        )
-    }
-
-    fn gauge(&self, params: MetricParameters) -> Arc<dyn Gauge> {
-        Arc::new(MemoryGaugeU64::new(params, &self.0))
+impl<I> HistogramF64 for BufferInstrument<I>
+where
+    I: BufferInstrumentRef + Send + Sync + Clone,
+{
+    fn record(&self, value: f64, attributes: &MetricAttributes) {
+        self.send(MetricUpdateVal::ValueF64(value), attributes)
     }
 }
-
-impl Gauge for MemoryGaugeU64 {
-    fn record(&self, value: u64, attributes: &MetricAttributes) {
-        if let MetricAttributes::OTel { kvs } = attributes {
-            self.record(value, kvs);
-        } else {
-            dbg_panic!("Must use OTel attributes with an OTel metric implementation");
-        }
+impl<I> HistogramDuration for BufferInstrument<I>
+where
+    I: BufferInstrumentRef + Send + Sync + Clone,
+{
+    fn record(&self, value: Duration, attributes: &MetricAttributes) {
+        self.send(MetricUpdateVal::Duration(value), attributes)
     }
 }
 
@@ -864,10 +724,25 @@ impl<CM: CoreMeter> CoreMeter for PrefixedMetricsMeter<CM> {
         self.meter.histogram(params)
     }
 
+    fn histogram_f64(&self, mut params: MetricParameters) -> Arc<dyn HistogramF64> {
+        params.name = (self.prefix.clone() + &*params.name).into();
+        self.meter.histogram_f64(params)
+    }
+
+    fn histogram_duration(&self, mut params: MetricParameters) -> Arc<dyn HistogramDuration> {
+        params.name = (self.prefix.clone() + &*params.name).into();
+        self.meter.histogram_duration(params)
+    }
+
     fn gauge(&self, mut params: MetricParameters) -> Arc<dyn Gauge> {
         params.name = (self.prefix.clone() + &*params.name).into();
         self.meter.gauge(params)
     }
+
+    fn gauge_f64(&self, mut params: MetricParameters) -> Arc<dyn GaugeF64> {
+        params.name = (self.prefix.clone() + &*params.name).into();
+        self.meter.gauge_f64(params)
+    }
 }
 
 #[cfg(test)]
@@ -932,7 +807,7 @@ mod tests {
         a1.set(Arc::new(DummyCustomAttrs(1))).unwrap();
         // Verify all metrics are created. This number will need to get updated any time a metric
         // is added.
-        let num_metrics = 23;
+        let num_metrics = 24;
         #[allow(clippy::needless_range_loop)] // Sorry clippy, this reads easier.
         for metric_num in 1..=num_metrics {
             let hole = assert_matches!(&events[metric_num],
@@ -970,9 +845,10 @@ mod tests {
             MetricEvent::Update {
                 instrument,
                 attributes,
-                update: MetricUpdateVal::Value(1000) // milliseconds
+                update: MetricUpdateVal::Duration(d)
             }
             if DummyCustomAttrs::as_id(attributes) == 2 && instrument.get().0 == 11
+               && d == &Duration::from_secs(1)
         );
     }
 
@@ -994,6 +870,11 @@ mod tests {
             description: "a counter".into(),
             unit: "bleezles".into(),
         });
+        let histo_dur = call_buffer.histogram_duration(MetricParameters {
+            name: "histo_dur".into(),
+            description: "a duration histogram".into(),
+            unit: "seconds".into(),
+        });
         let attrs_1 = call_buffer.new_attributes(NewAttributes {
             attributes: vec![MetricKeyValue::new("hi", "yo")],
         });
@@ -1003,6 +884,7 @@ mod tests {
         ctr.add(1, &attrs_1);
         histo.record(2, &attrs_1);
         gauge.record(3, &attrs_2);
+        histo_dur.record(Duration::from_secs_f64(1.2), &attrs_1);
 
         let mut calls = call_buffer.retrieve();
         calls.reverse();
@@ -1039,6 +921,17 @@ mod tests {
             => populate_into
         );
         gauge_3.set(Arc::new(DummyInstrumentRef(3))).unwrap();
+        let hist_4 = assert_matches!(
+            calls.pop(),
+            Some(MetricEvent::Create {
+                params,
+                populate_into,
+                kind: MetricKind::HistogramDuration
+            })
+            if params.name == "histo_dur"
+            => populate_into
+        );
+        hist_4.set(Arc::new(DummyInstrumentRef(4))).unwrap();
         let a1 = assert_matches!(
             calls.pop(),
             Some(MetricEvent::CreateAttributes {
@@ -1086,7 +979,17 @@ mod tests {
                 attributes,
                 update: MetricUpdateVal::Value(3)
             })
-            if DummyCustomAttrs::as_id(&attributes) == 2&& instrument.get().0 == 3
+            if DummyCustomAttrs::as_id(&attributes) == 2 && instrument.get().0 == 3
+        );
+        assert_matches!(
+            calls.pop(),
+            Some(MetricEvent::Update{
+                instrument,
+                attributes,
+                update: MetricUpdateVal::Duration(d)
+            })
+            if DummyCustomAttrs::as_id(&attributes) == 1 && instrument.get().0 == 4
+               && d == Duration::from_secs_f64(1.2)
         );
     }
 }