@temporalio/core-bridge 0.17.2 → 0.18.0

package/sdk-core/core/src/lib.rs

@@ -0,0 +1,374 @@
+#![warn(missing_docs)] // error if there are missing docs
+#![allow(clippy::upper_case_acronyms)]
+
+//! This crate provides a basis for creating new Temporal SDKs without completely starting from
+//! scratch
+
+#[cfg(test)]
+#[macro_use]
+pub extern crate assert_matches;
+#[macro_use]
+extern crate tracing;
+
+mod log_export;
+mod pending_activations;
+mod pollers;
+mod protosext;
+pub mod replay;
+pub(crate) mod retry_logic;
+pub(crate) mod telemetry;
+mod worker;
+mod workflow;
+
+#[cfg(test)]
+mod core_tests;
+#[cfg(test)]
+#[macro_use]
+mod test_help;
+
+pub(crate) use temporal_sdk_core_api::errors;
+
+pub use pollers::{
+    ClientTlsConfig, RetryConfig, RetryGateway, ServerGateway, ServerGatewayApis,
+    ServerGatewayOptions, ServerGatewayOptionsBuilder, TlsConfig,
+};
+pub use telemetry::{TelemetryOptions, TelemetryOptionsBuilder};
+pub use temporal_sdk_core_api as api;
+pub use temporal_sdk_core_protos as protos;
+pub use temporal_sdk_core_protos::TaskToken;
+pub use url::Url;
+pub use worker::{WorkerConfig, WorkerConfigBuilder};
+
+use crate::{
+    telemetry::{
+        fetch_global_buffered_logs,
+        metrics::{MetricsContext, METRIC_METER},
+        telemetry_init,
+    },
+    worker::{Worker, WorkerDispatcher},
+};
+use std::{
+    ops::Deref,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+};
+use temporal_sdk_core_api::{
+    errors::{
+        CompleteActivityError, CompleteWfError, CoreInitError, PollActivityError, PollWfError,
+        WorkerRegistrationError,
+    },
+    Core, CoreLog,
+};
+use temporal_sdk_core_protos::{
+    coresdk::{
+        activity_task::ActivityTask,
+        workflow_activation::{remove_from_cache::EvictionReason, WorkflowActivation},
+        workflow_completion::WorkflowActivationCompletion,
+        ActivityHeartbeat, ActivityTaskCompletion,
+    },
+    temporal::api::history::v1::History,
+};
+
+#[cfg(test)]
+use crate::test_help::MockWorker;
+
+lazy_static::lazy_static! {
+    /// A process-wide unique string, which will be different on every startup
+    static ref PROCCESS_UNIQ_ID: String = {
+        uuid::Uuid::new_v4().to_simple().to_string()
+    };
+}
+
+/// Holds various configuration information required to call [init]
+#[derive(Debug, Clone, derive_builder::Builder)]
+#[builder(setter(into))]
+#[non_exhaustive]
+pub struct CoreInitOptions {
+    /// Options for the connection to the temporal server
+    pub gateway_opts: ServerGatewayOptions,
+    /// Options for telemetry (traces and metrics)
+    #[builder(default)]
+    pub telemetry_opts: TelemetryOptions,
+}
+
+/// Initializes an instance of the core sdk and establishes a connection to the temporal server.
+/// Expects that a tokio runtime exists.
+pub async fn init(opts: CoreInitOptions) -> Result<CoreSDK, CoreInitError> {
+    telemetry_init(&opts.telemetry_opts).map_err(CoreInitError::TelemetryInitError)?;
+    // Initialize server client
+    let server_gateway = opts.gateway_opts.connect(Some(&METRIC_METER)).await?;
+
+    Ok(CoreSDK::new(server_gateway, opts))
+}
+
+/// Initialize core using a provided gateway instance, which is typically a mock
+pub fn init_mock_gateway<SG: ServerGatewayApis + Send + Sync + 'static>(
+    opts: CoreInitOptions,
+    server_gateway: SG,
+) -> Result<CoreSDK, CoreInitError> {
+    telemetry_init(&opts.telemetry_opts).map_err(CoreInitError::TelemetryInitError)?;
+    Ok(CoreSDK::new(server_gateway, opts))
+}
+
+/// Implements the [Core] trait
+pub struct CoreSDK {
+    /// Options provided at initialization time
+    init_options: CoreInitOptions,
+    /// A client for interacting with the Temporal service.
+    server_gateway: Arc<dyn ServerGatewayApis + Send + Sync>,
+    /// Controls access to workers
+    workers: WorkerDispatcher,
+    /// Has shutdown been called and all workers drained of tasks?
+    whole_core_shutdown: AtomicBool,
+    /// Top-level metrics context
+    metrics: MetricsContext,
+}
+
+#[async_trait::async_trait]
+impl Core for CoreSDK {
+    fn register_worker(&self, config: WorkerConfig) -> Result<(), WorkerRegistrationError> {
+        info!(
+            task_queue = config.task_queue.as_str(),
+            "Registering worker"
+        );
+        let sticky_q = self.get_sticky_q_name_for_worker(&config);
+        self.workers.new_worker(
+            config,
+            sticky_q,
+            self.server_gateway.clone(),
+            self.metrics.clone(),
+        )
+    }
+
+    #[instrument(level = "debug", skip(self), fields(run_id))]
+    async fn poll_workflow_activation(
+        &self,
+        task_queue: &str,
+    ) -> Result<WorkflowActivation, PollWfError> {
+        let worker = self.worker(task_queue)?;
+        worker.next_workflow_activation().await
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    async fn poll_activity_task(
+        &self,
+        task_queue: &str,
+    ) -> Result<ActivityTask, PollActivityError> {
+        loop {
+            if self.whole_core_shutdown.load(Ordering::Relaxed) {
+                return Err(PollActivityError::ShutDown);
+            }
+            let worker = self.worker(task_queue)?;
+            match worker.activity_poll().await.transpose() {
+                Some(r) => break r,
+                None => {
+                    tokio::task::yield_now().await;
+                    continue;
+                }
+            }
+        }
+    }
+
+    #[instrument(level = "debug", skip(self, completion),
+      fields(completion=%&completion, run_id=%completion.run_id))]
+    async fn complete_workflow_activation(
+        &self,
+        completion: WorkflowActivationCompletion,
+    ) -> Result<(), CompleteWfError> {
+        let worker = self.worker(&completion.task_queue)?;
+        worker.complete_workflow_activation(completion).await
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    async fn complete_activity_task(
+        &self,
+        completion: ActivityTaskCompletion,
+    ) -> Result<(), CompleteActivityError> {
+        let task_token = TaskToken(completion.task_token);
+        let status = if let Some(s) = completion.result.and_then(|r| r.status) {
+            s
+        } else {
+            return Err(CompleteActivityError::MalformedActivityCompletion {
+                reason: "Activity completion had empty result/status field".to_owned(),
+                completion: None,
+            });
+        };
+
+        let worker = self.worker(&completion.task_queue)?;
+        worker.complete_activity(task_token, status).await
+    }
+
+    fn record_activity_heartbeat(&self, details: ActivityHeartbeat) {
+        if let Ok(w) = self.worker(&details.task_queue) {
+            w.record_heartbeat(details);
+        }
+    }
+
+    fn request_workflow_eviction(&self, task_queue: &str, run_id: &str) {
+        if let Ok(w) = self.worker(task_queue) {
+            w.request_wf_eviction(
+                run_id,
+                "Eviction explicitly requested by lang",
+                EvictionReason::LangRequested,
+            );
+        }
+    }
+
+    fn server_gateway(&self) -> Arc<dyn ServerGatewayApis + Send + Sync> {
+        self.server_gateway.clone()
+    }
+
+    async fn shutdown(&self) {
+        self.workers.shutdown_all().await;
+        self.whole_core_shutdown.store(true, Ordering::Relaxed);
+    }
+
+    async fn shutdown_worker(&self, task_queue: &str) {
+        self.workers.shutdown_one(task_queue).await;
+    }
+
+    fn fetch_buffered_logs(&self) -> Vec<CoreLog> {
+        fetch_global_buffered_logs()
+    }
+}
+
+impl CoreSDK {
+    pub(crate) fn new<SG: ServerGatewayApis + Send + Sync + 'static>(
+        server_gateway: SG,
+        init_options: CoreInitOptions,
+    ) -> Self {
+        let server_gateway = Arc::new(server_gateway);
+        let workers = WorkerDispatcher::default();
+        Self {
+            workers,
+            init_options,
+            whole_core_shutdown: AtomicBool::new(false),
+            metrics: MetricsContext::top_level(server_gateway.get_options().namespace.clone()),
+            server_gateway,
+        }
+    }
+
+    /// Register a worker for replaying a specific history. The worker should use a unique task
+    /// queue name. It will auto-shutdown as soon as the history has finished being replayed. The
+    /// provided gateway should be a mock, and this should only be used for workflow testing
+    /// purposes.
+    pub fn register_replay_worker(
+        &self,
+        config: WorkerConfig,
+        gateway: Arc<dyn ServerGatewayApis + Send + Sync>,
+        history: &History,
+    ) -> Result<(), anyhow::Error> {
+        info!(
+            task_queue = config.task_queue.as_str(),
+            "Registering replay worker"
+        );
+        // Could possibly just use mocked pollers here, but they'd need to be un-test-moded
+        let run_id = history.extract_run_id_from_start()?.to_string();
+        let last_event = history.last_event_id();
+        let tq = config.task_queue.clone();
+        let mut worker = Worker::new(config, None, gateway, self.metrics.clone());
+        worker.set_post_activate_hook(move |worker| {
+            if worker
+                .wft_manager
+                .most_recently_processed_event(&run_id)
+                .unwrap_or_default()
+                >= last_event
+            {
+                worker.initiate_shutdown();
+            }
+        });
+
+        self.workers.set_worker_for_task_queue(tq, worker)?;
+        Ok(())
+    }
+
+    /// Allow construction of workers with mocked poll responses during testing
+    #[cfg(test)]
+    pub(crate) fn reg_worker_sync(&self, worker: MockWorker) {
+        let sticky_q = self.get_sticky_q_name_for_worker(&worker.config);
+        let tq = worker.config.task_queue.clone();
+        let worker = Worker::new_with_pollers(
+            worker.config,
+            sticky_q,
+            self.server_gateway.clone(),
+            worker.wf_poller,
+            worker.act_poller,
+            self.metrics.clone(),
+        );
+        self.workers.set_worker_for_task_queue(tq, worker).unwrap();
+    }
+
+    #[cfg(test)]
+    pub(crate) fn outstanding_wfts(&self, tq: &str) -> usize {
+        self.worker(tq).unwrap().outstanding_workflow_tasks()
+    }
+    #[cfg(test)]
+    pub(crate) fn available_wft_permits(&self, tq: &str) -> usize {
+        self.worker(tq).unwrap().available_wft_permits()
+    }
+
+    fn get_sticky_q_name_for_worker(&self, config: &WorkerConfig) -> Option<String> {
+        if config.max_cached_workflows > 0 {
+            Some(format!(
+                "{}-{}-{}",
+                &self.init_options.gateway_opts.identity, &config.task_queue, *PROCCESS_UNIQ_ID
+            ))
+        } else {
+            None
+        }
+    }
+
+    fn worker(&self, tq: &str) -> Result<impl Deref<Target = Worker>, WorkerLookupErr> {
+        let worker = self.workers.get(tq);
+        if worker.is_err() && self.whole_core_shutdown.load(Ordering::Relaxed) {
+            return Err(WorkerLookupErr::Shutdown(tq.to_owned()));
+        }
+        worker
+    }
+}
+
+#[derive(Debug)]
+enum WorkerLookupErr {
+    Shutdown(String),
+    NoWorker(String),
+}
+
+impl From<WorkerLookupErr> for PollWfError {
+    fn from(e: WorkerLookupErr) -> Self {
+        match e {
+            WorkerLookupErr::Shutdown(_) => Self::ShutDown,
+            WorkerLookupErr::NoWorker(s) => Self::NoWorkerForQueue(s),
+        }
+    }
+}
+
+impl From<WorkerLookupErr> for PollActivityError {
+    fn from(e: WorkerLookupErr) -> Self {
+        match e {
+            WorkerLookupErr::Shutdown(_) => Self::ShutDown,
+            WorkerLookupErr::NoWorker(s) => Self::NoWorkerForQueue(s),
+        }
+    }
+}
+
+impl From<WorkerLookupErr> for CompleteWfError {
+    fn from(e: WorkerLookupErr) -> Self {
+        match e {
+            WorkerLookupErr::Shutdown(s) | WorkerLookupErr::NoWorker(s) => {
+                Self::NoWorkerForQueue(s)
+            }
+        }
+    }
+}
+
+impl From<WorkerLookupErr> for CompleteActivityError {
+    fn from(e: WorkerLookupErr) -> Self {
+        match e {
+            WorkerLookupErr::Shutdown(s) | WorkerLookupErr::NoWorker(s) => {
+                Self::NoWorkerForQueue(s)
+            }
+        }
+    }
+}

package/sdk-core/{src → core/src}/log_export.rs

@@ -1,31 +1,7 @@
-use log::{Level, LevelFilter, Log, Metadata, Record};
+use log::{LevelFilter, Log, Metadata, Record};
 use ringbuf::{Consumer, Producer, RingBuffer};
-use std::{
-    sync::Mutex,
-    time::{Duration, SystemTime, UNIX_EPOCH},
-};
-
-/// A log line (which ultimately came from a tracing event) exported from Core->Lang
-#[derive(Debug)]
-pub struct CoreLog {
-    /// Log message
-    pub message: String,
-    /// Time log was generated (not when it was exported to lang)
-    pub timestamp: SystemTime,
-    /// Message level
-    pub level: Level,
-    // KV pairs aren't meaningfully exposed yet to the log interface by tracing
-}
-
-impl CoreLog {
-    /// Return timestamp as ms since epoch
-    pub fn millis_since_epoch(&self) -> u128 {
-        self.timestamp
-            .duration_since(UNIX_EPOCH)
-            .unwrap_or(Duration::ZERO)
-            .as_millis()
-    }
-}
+use std::{sync::Mutex, time::SystemTime};
+use temporal_sdk_core_api::CoreLog;
 
 pub(crate) struct CoreExportLogger {
     logs_in: Mutex<Producer<CoreLog>>,

package/sdk-core/core/src/pending_activations.rs

@@ -0,0 +1,162 @@
+use parking_lot::RwLock;
+use slotmap::SlotMap;
+use std::collections::{HashMap, VecDeque};
+use temporal_sdk_core_protos::coresdk::workflow_activation::{
+    remove_from_cache::EvictionReason, RemoveFromCache,
+};
+
+/// Tracks pending activations using an internal queue, while also allowing lookup and removal of
+/// any pending activations by run ID.
+#[derive(Default)]
+pub struct PendingActivations {
+    inner: RwLock<PaInner>,
+}
+
+slotmap::new_key_type! { struct ActivationKey; }
+
+#[derive(Default)]
+struct PaInner {
+    activations: SlotMap<ActivationKey, PendingActInfo>,
+    by_run_id: HashMap<String, ActivationKey>,
+    // Holds the actual queue of activations
+    queue: VecDeque<ActivationKey>,
+}
+
+pub struct PendingActInfo {
+    pub needs_eviction: Option<RemoveFromCache>,
+    pub run_id: String,
+}
+
+impl PendingActivations {
+    /// Indicate that a run needs to be activated
+    pub fn notify_needs_activation(&self, run_id: &str) {
+        let mut inner = self.inner.write();
+
+        if inner.by_run_id.get(run_id).is_none() {
+            let key = inner.activations.insert(PendingActInfo {
+                needs_eviction: None,
+                run_id: run_id.to_string(),
+            });
+            inner.by_run_id.insert(run_id.to_string(), key);
+            inner.queue.push_back(key);
+        };
+    }
+    pub fn notify_needs_eviction(&self, run_id: &str, message: String, reason: EvictionReason) {
+        let mut inner = self.inner.write();
+
+        let evictjob = RemoveFromCache {
+            message,
+            reason: reason as i32,
+        };
+
+        if let Some(key) = inner.by_run_id.get(run_id).copied() {
+            let act = inner
+                .activations
+                .get_mut(key)
+                .expect("PA run id mapping is always in sync with slot map");
+            act.needs_eviction = Some(evictjob);
+        } else {
+            let key = inner.activations.insert(PendingActInfo {
+                needs_eviction: Some(evictjob),
+                run_id: run_id.to_string(),
+            });
+            inner.by_run_id.insert(run_id.to_string(), key);
+            inner.queue.push_back(key);
+        };
+    }
+
+    pub fn pop_first_matching(&self, predicate: impl Fn(&str) -> bool) -> Option<PendingActInfo> {
+        let mut inner = self.inner.write();
+        let mut key_queue = inner.queue.iter().copied();
+        let maybe_key = key_queue.position(|k| {
+            inner
+                .activations
+                .get(k)
+                .map_or(false, |activation| predicate(&activation.run_id))
+        });
+
+        let maybe_key = maybe_key.map(|pos| inner.queue.remove(pos).unwrap());
+        maybe_key.and_then(|key| {
+            if let Some(pa) = inner.activations.remove(key) {
+                inner.by_run_id.remove(&pa.run_id);
+                Some(pa)
+            } else {
+                // Keys no longer in the slot map are ignored, since they may have been removed
+                // by run id or anything else. Try to pop the next thing from the queue. Recurse
+                // to avoid double mutable borrow.
+                drop(inner); // Will deadlock when we recurse w/o this
+                self.pop_first_matching(predicate)
+            }
+        })
+    }
+
+    #[cfg(test)]
+    pub fn pop(&self) -> Option<PendingActInfo> {
+        self.pop_first_matching(|_| true)
+    }
+
+    pub fn has_pending(&self, run_id: &str) -> bool {
+        self.inner.read().by_run_id.contains_key(run_id)
+    }
+
+    pub fn remove_all_with_run_id(&self, run_id: &str) {
+        let mut inner = self.inner.write();
+
+        if let Some(k) = inner.by_run_id.remove(run_id) {
+            inner.activations.remove(k);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn merges_same_ids_with_evictions() {
+        let pas = PendingActivations::default();
+        let rid1 = "1";
+        let rid2 = "2";
+        pas.notify_needs_activation(rid1);
+        pas.notify_needs_eviction(rid1, "whatever".to_string(), EvictionReason::Unspecified);
+        pas.notify_needs_eviction(rid2, "whatever".to_string(), EvictionReason::Unspecified);
+        pas.notify_needs_activation(rid2);
+        assert!(pas.has_pending(rid1));
+        assert!(pas.has_pending(rid2));
+        let last = pas.pop().unwrap();
+        assert_eq!(&last.run_id, &rid1);
+        assert!(!pas.has_pending(rid1));
+        assert!(pas.has_pending(rid2));
+        // Should only be one id 2, they are all merged
+        let last = pas.pop().unwrap();
+        assert_eq!(&last.run_id, &rid2);
+        assert!(!pas.has_pending(rid2));
+        assert!(pas.pop().is_none());
+    }
+
+    #[test]
+    fn can_remove_all_with_id() {
+        let pas = PendingActivations::default();
+        let remove_me = "2";
+        pas.notify_needs_activation("1");
+        pas.notify_needs_activation(remove_me);
+        pas.notify_needs_activation("3");
+        pas.remove_all_with_run_id(remove_me);
+        assert!(!pas.has_pending(remove_me));
+        assert_eq!(&pas.pop().unwrap().run_id, "1");
+        assert_eq!(&pas.pop().unwrap().run_id, "3");
+        assert!(pas.pop().is_none());
+    }
+
+    #[test]
+    fn can_ignore_specific_runs() {
+        let pas = PendingActivations::default();
+        pas.notify_needs_activation("1");
+        pas.notify_needs_activation("2");
+        assert_eq!(
+            &pas.pop_first_matching(|rid| rid != "1").unwrap().run_id,
+            "2"
+        );
+        assert_eq!(&pas.pop().unwrap().run_id, "1");
+    }
+}

package/sdk-core/{src → core/src}/pollers/mod.rs

@@ -1,19 +1,13 @@
-mod gateway;
 mod poll_buffer;
-mod retry;
 
-#[cfg(test)]
-pub use gateway::{MockManualGateway, MockServerGatewayApis};
-
-pub use gateway::{
-    ClientTlsConfig, GatewayRef, RetryConfig, ServerGateway, ServerGatewayApis,
-    ServerGatewayOptions, ServerGatewayOptionsBuilder, TlsConfig,
-};
 pub use poll_buffer::{
     new_activity_task_buffer, new_workflow_task_buffer, PollActivityTaskBuffer,
     PollWorkflowTaskBuffer, WorkflowTaskPoller,
 };
-pub use retry::RetryGateway;
+pub use temporal_client::{
+    ClientTlsConfig, RetryConfig, RetryGateway, ServerGateway, ServerGatewayApis,
+    ServerGatewayOptions, ServerGatewayOptionsBuilder, TlsConfig,
+};
 
 use temporal_sdk_core_protos::temporal::api::workflowservice::v1::{
     PollActivityTaskQueueResponse, PollWorkflowTaskQueueResponse,
@@ -21,21 +15,9 @@ use temporal_sdk_core_protos::temporal::api::workflowservice::v1::{
 
 #[cfg(test)]
 use futures::Future;
-use tonic::Code;
 
 pub type Result<T, E = tonic::Status> = std::result::Result<T, E>;
 
-/// List of gRPC error codes that client will retry.
-pub const RETRYABLE_ERROR_CODES: [Code; 7] = [
-    Code::DataLoss,
-    Code::Internal,
-    Code::Unknown,
-    Code::ResourceExhausted,
-    Code::Aborted,
-    Code::OutOfRange,
-    Code::Unavailable,
-];
-
 /// A trait for things that poll the server. Hides complexity of concurrent polling or polling
 /// on sticky/nonsticky queues simultaneously.
 #[cfg_attr(test, mockall::automock)]

package/sdk-core/{src → core/src}/pollers/poll_buffer.rs

@@ -236,9 +236,9 @@ pub fn new_activity_task_buffer(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::pollers::MockManualGateway;
     use futures::FutureExt;
     use std::time::Duration;
+    use temporal_client::MockManualGateway;
     use tokio::{select, sync::mpsc::channel};
 
     #[tokio::test]