@temporalio/core-bridge 0.19.2 → 0.20.2

package/sdk-core/core/src/replay/mod.rs

@@ -2,7 +2,7 @@
 //! to replay canned histories. It should be used by Lang SDKs to provide replay capabilities to
 //! users during testing.
 
-use crate::{init_mock_gateway, CoreInitOptionsBuilder, CoreSDK, TelemetryOptions};
+use crate::{worker::client::mocks::mock_manual_workflow_client, WorkerClientBag};
 use futures::FutureExt;
 use std::{
     sync::{
@@ -11,154 +11,26 @@ use std::{
     },
     time::Duration,
 };
-use temporal_client::{
-    mocks::{mock_gateway, mock_manual_gateway},
-    ServerGatewayApis,
-};
-use temporal_sdk_core_api::{
-    errors::{
-        CompleteActivityError, CompleteWfError, PollActivityError, PollWfError,
-        WorkerRegistrationError,
-    },
-    worker::WorkerConfig,
-    Core, CoreLog,
-};
-use temporal_sdk_core_protos::{
-    coresdk::{
-        activity_task::ActivityTask, workflow_activation::WorkflowActivation,
-        workflow_completion::WorkflowActivationCompletion, ActivityHeartbeat,
-        ActivityTaskCompletion,
-    },
-    temporal::api::{
-        common::v1::WorkflowExecution,
-        history::v1::History,
-        workflowservice::v1::{
-            RespondWorkflowTaskCompletedResponse, RespondWorkflowTaskFailedResponse,
-            StartWorkflowExecutionResponse,
-        },
+use temporal_sdk_core_protos::temporal::api::{
+    common::v1::WorkflowExecution,
+    history::v1::History,
+    workflowservice::v1::{
+        RespondWorkflowTaskCompletedResponse, RespondWorkflowTaskFailedResponse,
     },
 };
-
 pub use temporal_sdk_core_protos::{
     default_wes_attribs, HistoryInfo, TestHistoryBuilder, DEFAULT_WORKFLOW_TYPE,
 };
 
-/// Create a core instance that can be used for replay. See the [ReplayCore] trait for adding
-/// canned history.
-pub fn init_core_replay(opts: TelemetryOptions) -> ReplayCoreImpl {
-    let shared_mock_gateway = mock_gateway();
-    let init_opts = CoreInitOptionsBuilder::default()
-        .gateway_opts(shared_mock_gateway.get_options().clone())
-        .telemetry_opts(opts)
-        .build()
-        .expect("replay core options init properly");
-    let replay_core =
-        init_mock_gateway(init_opts, shared_mock_gateway).expect("init replay core works");
-    ReplayCoreImpl { inner: replay_core }
-}
-
-/// An extension of the [Core] trait to be used for testing workflows against canned histories.
-pub trait ReplayCore: Core {
-    /// Make a fake worker which will use the provided history to simulate poll responses containing
-    /// that entire history. The configuration should have a unique task queue name.
-    ///
-    /// Note that some of the worker config options do not apply, and some will be overridden.
-    /// Replay workers always are cached, have only 1 poller, and do not poll for activities.
-    fn make_replay_worker(
-        &self,
-        config: WorkerConfig,
-        history: &History,
-    ) -> Result<(), anyhow::Error>;
-}
-
-/// Implements [ReplayCore] functionality
-pub struct ReplayCoreImpl {
-    pub(crate) inner: CoreSDK,
-}
-
-impl ReplayCore for ReplayCoreImpl {
-    fn make_replay_worker(
-        &self,
-        mut config: WorkerConfig,
-        history: &History,
-    ) -> Result<(), anyhow::Error> {
-        let mock_g = mock_gateway_from_history(history, config.task_queue.clone());
-        config.max_cached_workflows = 1;
-        config.max_concurrent_wft_polls = 1;
-        config.no_remote_activities = true;
-        self.inner
-            .register_replay_worker(config, Arc::new(mock_g), history)
-    }
-}
-
-#[async_trait::async_trait]
-impl Core for ReplayCoreImpl {
-    fn register_worker(&self, _config: WorkerConfig) -> Result<(), WorkerRegistrationError> {
-        panic!("Do not use `register_worker` with a replay core instance")
-    }
-
-    async fn poll_workflow_activation(
-        &self,
-        task_queue: &str,
-    ) -> Result<WorkflowActivation, PollWfError> {
-        self.inner.poll_workflow_activation(task_queue).await
-    }
-
-    async fn poll_activity_task(
-        &self,
-        task_queue: &str,
-    ) -> Result<ActivityTask, PollActivityError> {
-        self.inner.poll_activity_task(task_queue).await
-    }
-
-    async fn complete_workflow_activation(
-        &self,
-        completion: WorkflowActivationCompletion,
-    ) -> Result<(), CompleteWfError> {
-        self.inner.complete_workflow_activation(completion).await
-    }
-
-    async fn complete_activity_task(
-        &self,
-        completion: ActivityTaskCompletion,
-    ) -> Result<(), CompleteActivityError> {
-        self.inner.complete_activity_task(completion).await
-    }
-
-    fn record_activity_heartbeat(&self, _details: ActivityHeartbeat) {
-        // do nothing
-    }
-
-    fn request_workflow_eviction(&self, task_queue: &str, run_id: &str) {
-        self.inner.request_workflow_eviction(task_queue, run_id)
-    }
-
-    fn server_gateway(&self) -> Arc<dyn ServerGatewayApis + Send + Sync> {
-        self.inner.server_gateway()
-    }
-
-    async fn shutdown(&self) {
-        self.inner.shutdown().await
-    }
-
-    async fn shutdown_worker(&self, task_queue: &str) {
-        self.inner.shutdown_worker(task_queue).await
-    }
-
-    fn fetch_buffered_logs(&self) -> Vec<CoreLog> {
-        self.inner.fetch_buffered_logs()
-    }
-}
-
-/// Create a mock gateway which can be used by a replay worker to serve up canned history.
+/// Create a mock client which can be used by a replay worker to serve up canned history.
 /// It will return the entire history in one workflow task, after that it will return default
 /// responses (with a 10s wait). If a workflow task failure is sent to the mock, it will send
 /// the complete response again.
-pub fn mock_gateway_from_history(
+pub(crate) fn mock_client_from_history(
     history: &History,
     task_queue: impl Into<String>,
-) -> impl ServerGatewayApis {
-    let mut mg = mock_manual_gateway();
+) -> WorkerClientBag {
+    let mut mg = mock_manual_workflow_client();
 
     let hist_info = HistoryInfo::new_from_history(history, None).unwrap();
     let wf = WorkflowExecution {
@@ -166,17 +38,6 @@ pub fn mock_gateway_from_history(
         run_id: hist_info.orig_run_id().to_string(),
     };
 
-    let wf_clone = wf.clone();
-    mg.expect_start_workflow().returning(move |_, _, _, _, _| {
-        let wf_clone = wf_clone.clone();
-        async move {
-            Ok(StartWorkflowExecutionResponse {
-                run_id: wf_clone.run_id.clone(),
-            })
-        }
-        .boxed()
-    });
-
     let did_send = Arc::new(AtomicBool::new(false));
     let did_send_clone = did_send.clone();
     let tq = task_queue.into();
@@ -206,5 +67,5 @@ pub fn mock_gateway_from_history(
         async move { Ok(RespondWorkflowTaskFailedResponse {}) }.boxed()
     });
 
-    mg
+    WorkerClientBag::new(Box::new(mg), "fake_namespace".to_string())
 }

package/sdk-core/core/src/telemetry/metrics.rs

@@ -8,7 +8,7 @@ use opentelemetry::{
     },
     KeyValue,
 };
-use std::{sync::Arc, time::Duration};
+use std::{borrow::Cow, sync::Arc, time::Duration};
 
 /// Used to track context associated with metrics, and record/update them
 ///
@@ -31,6 +31,11 @@ impl MetricsContext {
         Self::new(vec![KeyValue::new(KEY_NAMESPACE, namespace)])
     }
 
+    pub(crate) fn with_task_q(mut self, tq: String) -> Self {
+        Arc::make_mut(&mut self.kvs).push(task_queue(tq));
+        self
+    }
+
     /// Extend an existing metrics context with new attributes
     pub(crate) fn with_new_attrs(&self, new_kvs: impl IntoIterator<Item = KeyValue>) -> Self {
         let mut kvs = self.kvs.clone();
@@ -119,6 +124,11 @@ impl MetricsContext {
         WORKER_REGISTERED.add(1, &self.kvs);
     }
 
+    /// Record current number of available task slots. Context should have worker type set.
+    pub(crate) fn available_task_slots(&self, num: usize) {
+        TASK_SLOTS_AVAILABLE.record(num as u64, &self.kvs)
+    }
+
     /// Record current number of pollers. Context should include poller type / task queue tag.
     pub(crate) fn record_num_pollers(&self, num: usize) {
         NUM_POLLERS.record(num as u64, &self.kvs);
@@ -174,6 +184,7 @@ const KEY_WF_TYPE: &str = "workflow_type";
 const KEY_TASK_QUEUE: &str = "task_queue";
 const KEY_ACT_TYPE: &str = "activity_type";
 const KEY_POLLER_TYPE: &str = "poller_type";
+const KEY_WORKER_TYPE: &str = "worker_type";
 
 pub(crate) fn workflow_poller() -> KeyValue {
     KeyValue::new(KEY_POLLER_TYPE, "workflow_task")
@@ -193,6 +204,24 @@ pub(crate) fn activity_type(ty: String) -> KeyValue {
 pub(crate) fn workflow_type(ty: String) -> KeyValue {
     KeyValue::new(KEY_WF_TYPE, ty)
 }
+pub(crate) const fn workflow_worker_type() -> KeyValue {
+    KeyValue {
+        key: opentelemetry::Key::from_static_str(KEY_WORKER_TYPE),
+        value: opentelemetry::Value::String(Cow::Borrowed("WorkflowWorker")),
+    }
+}
+pub(crate) const fn activity_worker_type() -> KeyValue {
+    KeyValue {
+        key: opentelemetry::Key::from_static_str(KEY_WORKER_TYPE),
+        value: opentelemetry::Value::String(Cow::Borrowed("ActivityWorker")),
+    }
+}
+pub(crate) const fn local_activity_worker_type() -> KeyValue {
+    KeyValue {
+        key: opentelemetry::Key::from_static_str(KEY_WORKER_TYPE),
+        value: opentelemetry::Value::String(Cow::Borrowed("LocalActivityWorker")),
+    }
+}
 
 tm!(ctr, WF_COMPLETED_COUNTER, "workflow_completed");
 tm!(ctr, WF_CANCELED_COUNTER, "workflow_canceled");
@@ -248,6 +277,8 @@ tm!(vr_u64, ACT_EXEC_LATENCY, ACT_EXEC_LATENCY_NAME);
 tm!(ctr, WORKER_REGISTERED, "worker_start");
 const NUM_POLLERS_NAME: &str = "num_pollers";
 tm!(vr_u64, NUM_POLLERS, NUM_POLLERS_NAME);
+const TASK_SLOTS_AVAILABLE_NAME: &str = "worker_task_slots_available";
+tm!(vr_u64, TASK_SLOTS_AVAILABLE, TASK_SLOTS_AVAILABLE_NAME);
 
 tm!(ctr, STICKY_CACHE_HIT, "sticky_cache_hit");
 tm!(ctr, STICKY_CACHE_MISS, "sticky_cache_miss");
@@ -305,7 +336,9 @@ impl AggregatorSelector for SDKAggSelector {
         if *descriptor.instrument_kind() == InstrumentKind::ValueRecorder {
             // Some recorders are just gauges
             match descriptor.name() {
-                STICKY_CACHE_SIZE_NAME | NUM_POLLERS_NAME => return Some(Arc::new(last_value())),
+                STICKY_CACHE_SIZE_NAME | NUM_POLLERS_NAME | TASK_SLOTS_AVAILABLE_NAME => {
+                    return Some(Arc::new(last_value()))
+                }
                 _ => (),
             }
 

package/sdk-core/core/src/telemetry/mod.rs

@@ -4,13 +4,14 @@ mod prometheus_server;
 use crate::{
     log_export::CoreExportLogger,
     telemetry::{metrics::SDKAggSelector, prometheus_server::PromServer},
-    CoreLog,
+    CoreLog, METRIC_METER,
 };
 use itertools::Itertools;
 use log::LevelFilter;
 use once_cell::sync::OnceCell;
 use opentelemetry::{
     global,
+    metrics::Meter,
     sdk::{metrics::PushController, trace::Config, Resource},
     util::tokio_interval_stream,
     KeyValue,
@@ -18,6 +19,7 @@ use opentelemetry::{
 use opentelemetry_otlp::WithExportConfig;
 use parking_lot::{const_mutex, Mutex};
 use std::{collections::VecDeque, net::SocketAddr, time::Duration};
+use temporal_sdk_core_api::CoreTelemetry;
 use tracing_subscriber::{layer::SubscriberExt, EnvFilter};
 use url::Url;
 
@@ -27,6 +29,14 @@ static DEFAULT_FILTER: &str = "temporal_sdk_core=INFO";
 static GLOBAL_TELEM_DAT: OnceCell<GlobalTelemDat> = OnceCell::new();
 static TELETM_MUTEX: Mutex<()> = const_mutex(());
 
+fn default_resource_kvs() -> &'static [KeyValue] {
+    static INSTANCE: OnceCell<[KeyValue; 1]> = OnceCell::new();
+    INSTANCE.get_or_init(|| [KeyValue::new("service.name", TELEM_SERVICE_NAME)])
+}
+fn default_resource() -> Resource {
+    Resource::new(default_resource_kvs().iter().cloned())
+}
+
 /// Telemetry configuration options. Construct with [TelemetryOptionsBuilder]
 #[derive(Debug, Clone, derive_builder::Builder)]
 #[non_exhaustive]
@@ -56,6 +66,10 @@ pub struct TelemetryOptions {
     /// to the OTel collector if it is also set, only traces will be.
     #[builder(setter(into, strip_option), default)]
     pub prometheus_export_bind_address: Option<SocketAddr>,
+    /// If set, no resources are dedicated to telemetry and no metrics or traces are emited.
+    /// Supercedes all other options.
+    #[builder(default)]
+    pub totally_disable: bool,
 }
 
 impl Default for TelemetryOptions {
@@ -66,7 +80,7 @@ impl Default for TelemetryOptions {
 
 /// Things that need to not be dropped while telemetry is ongoing
 #[derive(Default)]
-pub(crate) struct GlobalTelemDat {
+pub struct GlobalTelemDat {
     metric_push_controller: Option<PushController>,
     core_export_logger: Option<CoreExportLogger>,
     runtime: Option<tokio::runtime::Runtime>,
@@ -87,14 +101,24 @@ impl GlobalTelemDat {
     }
 }
 
-/// Initialize tracing subscribers and output. Core [crate::init] calls this, but it may be called
-/// separately so that tests may choose to initialize tracing differently. If this function is
-/// called more than once, subsequent calls do nothing.
+impl CoreTelemetry for GlobalTelemDat {
+    fn fetch_buffered_logs(&self) -> Vec<CoreLog> {
+        fetch_global_buffered_logs()
+    }
+
+    fn get_metric_meter(&self) -> Option<&Meter> {
+        if GLOBAL_TELEM_DAT.get().is_some() {
+            return Some(&METRIC_METER);
+        }
+        None
+    }
+}
+
+/// Initialize tracing subscribers/output and logging export. If this function is called more than
+/// once, subsequent calls do nothing.
 ///
 /// See [TelemetryOptions] docs for more on configuration.
-pub(crate) fn telemetry_init(
-    opts: &TelemetryOptions,
-) -> Result<&'static GlobalTelemDat, anyhow::Error> {
+pub fn telemetry_init(opts: &TelemetryOptions) -> Result<&'static GlobalTelemDat, anyhow::Error> {
     // TODO: Per-layer filtering has been implemented but does not yet support
     //   env-filter. When it does, allow filtering logs/telemetry separately.
 
@@ -116,6 +140,15 @@ pub(crate) fn telemetry_init(
             // Ensure closure captures the mutex guard
             let _ = &*guard;
 
+            if opts.totally_disable {
+                return Ok(GlobalTelemDat {
+                    metric_push_controller: None,
+                    core_export_logger: None,
+                    runtime: None,
+                    prom_srv: None,
+                });
+            }
+
             let runtime = tokio::runtime::Builder::new_multi_thread()
                 .thread_name("telemetry")
                 .worker_threads(2)
@@ -146,11 +179,7 @@ pub(crate) fn telemetry_init(
 
             if let Some(otel_url) = opts.otel_collector_url.as_ref() {
                 runtime.block_on(async {
-                    let tracer_cfg =
-                        Config::default().with_resource(Resource::new(vec![KeyValue::new(
-                            "service.name",
-                            TELEM_SERVICE_NAME,
-                        )]));
+                    let tracer_cfg = Config::default().with_resource(default_resource());
                     let tracer = opentelemetry_otlp::new_pipeline()
                         .tracing()
                         .with_exporter(
@@ -168,6 +197,7 @@ pub(crate) fn telemetry_init(
                             .metrics(|f| runtime.spawn(f), tokio_interval_stream)
                             .with_aggregator_selector(SDKAggSelector)
                             .with_period(Duration::from_secs(1))
+                            .with_resource(default_resource_kvs().iter().cloned())
                             .with_exporter(
                                 // No joke exporter builder literally not cloneable for some insane
                                 // reason
@@ -210,8 +240,9 @@ pub(crate) fn telemetry_init(
     .expect("Telemetry initialization panicked")
 }
 
-/// Returned buffered logs for export to lang from the global logging instance
-pub(crate) fn fetch_global_buffered_logs() -> Vec<CoreLog> {
+/// Returned buffered logs for export to lang from the global logging instance.
+/// If [telemetry_init] has not been called, always returns an empty vec.
+pub fn fetch_global_buffered_logs() -> Vec<CoreLog> {
     if let Some(loggr) = GLOBAL_TELEM_DAT
         .get()
         .and_then(|gd| gd.core_export_logger.as_ref())
@@ -227,9 +258,10 @@ pub(crate) fn fetch_global_buffered_logs() -> Vec<CoreLog> {
 pub(crate) fn test_telem_console() {
     telemetry_init(&TelemetryOptions {
         otel_collector_url: None,
-        tracing_filter: "temporal_sdk_core=DEBUG".to_string(),
+        tracing_filter: "temporal_sdk_core=DEBUG,temporal_sdk=DEBUG".to_string(),
         log_forwarding_level: LevelFilter::Off,
         prometheus_export_bind_address: None,
+        totally_disable: false,
     })
     .unwrap();
 }
@@ -242,6 +274,7 @@ pub(crate) fn test_telem_collector() {
         tracing_filter: "temporal_sdk_core=DEBUG".to_string(),
         log_forwarding_level: LevelFilter::Off,
         prometheus_export_bind_address: None,
+        totally_disable: false,
     })
     .unwrap();
 }

package/sdk-core/core/src/telemetry/prometheus_server.rs

@@ -1,17 +1,16 @@
-use crate::telemetry::metrics::{SDKAggSelector, DEFAULT_MS_BUCKETS};
+use crate::telemetry::{
+    default_resource,
+    metrics::{SDKAggSelector, DEFAULT_MS_BUCKETS},
+};
 use hyper::{
     header::CONTENT_TYPE,
     service::{make_service_fn, service_fn},
     Body, Method, Request, Response, Server,
 };
-use opentelemetry::{
-    global,
-    metrics::MetricsError,
-    sdk::{export::metrics::ExportKindSelector, metrics::controllers},
-};
-use opentelemetry_prometheus::PrometheusExporter;
+use opentelemetry::metrics::MetricsError;
+use opentelemetry_prometheus::{ExporterBuilder, PrometheusExporter};
 use prometheus::{Encoder, TextEncoder};
-use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration};
+use std::{convert::Infallible, net::SocketAddr, sync::Arc};
 
 /// Exposes prometheus metrics for scraping
 pub(super) struct PromServer {
@@ -21,7 +20,13 @@ pub(super) struct PromServer {
 
 impl PromServer {
     pub fn new(addr: SocketAddr) -> Result<Self, MetricsError> {
-        let exporter = create_exporter()?;
+        let exporter = ExporterBuilder::default()
+            .with_default_histogram_boundaries(DEFAULT_MS_BUCKETS.to_vec())
+            .with_aggregator_selector(SDKAggSelector)
+            .with_host(addr.ip().to_string())
+            .with_port(addr.port())
+            .with_resource(default_resource())
+            .try_init()?;
         Ok(Self {
             exporter: Arc::new(exporter),
             addr,
@@ -66,29 +71,3 @@ async fn metrics_req(
     };
     Ok(response)
 }
-
-/// There is no easy way to customize the aggregation selector for the prom exporter so this
-/// code is some dupe of the exporter's try_init code.
-pub fn create_exporter() -> Result<PrometheusExporter, MetricsError> {
-    let controller_builder = controllers::pull(
-        Box::new(SDKAggSelector),
-        Box::new(ExportKindSelector::Cumulative),
-    )
-    .with_cache_period(Duration::from_secs(0))
-    .with_memory(true);
-    let controller = controller_builder.build();
-
-    global::set_meter_provider(controller.provider());
-
-    // No choice bro https://github.com/open-telemetry/opentelemetry-rust/issues/633
-    #[allow(deprecated)]
-    PrometheusExporter::new(
-        prometheus::Registry::new(),
-        controller,
-        DEFAULT_MS_BUCKETS.to_vec(),
-        DEFAULT_MS_BUCKETS.to_vec(),
-        // Host and port aren't actually used for anything
-        "".to_string(),
-        0,
-    )
-}