temporalio 0.0.0 → 0.0.2

data/bridge/sdk-core/core-api/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "temporal-sdk-core-api"
+version = "0.1.0"
+edition = "2021"
+authors = ["Spencer Judge <spencer@temporal.io>"]
+license-file = "LICENSE.txt"
+description = "Interface definitions for the Temporal Core SDK"
+homepage = "https://temporal.io/"
+repository = "https://github.com/temporalio/sdk-core"
+keywords = ["temporal", "workflow"]
+categories = ["development-tools"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1"
+derive_builder = "0.12"
+opentelemetry = "0.18"
+prost-types = "0.11"
+serde_json = "1.0"
+thiserror = "1.0"
+tonic = "0.8"
+tracing-core = "0.1"
+url = "2.3"
+
+[dependencies.temporal-sdk-core-protos]
+path = "../sdk-core-protos"
+version = "0.1"
+
+[dependencies.temporal-client]
+path = "../client"
+version = "0.1"

data/bridge/sdk-core/core-api/LICENSE.txt

@@ -0,0 +1,23 @@
+Temporal Core SDK
+
+The MIT License
+
+Copyright (c) 2021 Temporal Technologies, Inc. All Rights Reserved
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/bridge/sdk-core/core-api/src/errors.rs

@@ -0,0 +1,95 @@
+//! Error types exposed by public APIs
+
+use prost_types::TimestampError;
+use temporal_sdk_core_protos::coresdk::{
+    activity_result::ActivityExecutionResult,
+    workflow_activation::remove_from_cache::EvictionReason,
+};
+
+/// Errors thrown by [crate::Worker::poll_workflow_activation]
+#[derive(thiserror::Error, Debug)]
+pub enum PollWfError {
+    /// [crate::Worker::shutdown] was called, and there are no more replay tasks to be handled. Lang
+    /// must call [crate::Worker::complete_workflow_activation] for any remaining tasks, and then may
+    /// exit.
+    #[error("Core is shut down and there are no more workflow replay tasks")]
+    ShutDown,
+    /// Unhandled error when calling the temporal server. Core will attempt to retry any non-fatal
+    /// errors, so lang should consider this fatal.
+    #[error("Unhandled grpc error when workflow polling: {0:?}")]
+    TonicError(#[from] tonic::Status),
+    /// Unhandled error when completing a workflow during a poll -- this can happen when there is no
+    /// work for lang to perform, but the server sent us a workflow task (EX: An activity completed
+    /// even though we already cancelled it)
+    #[error("Unhandled error when auto-completing workflow task: {0:?}")]
+    AutocompleteError(#[from] CompleteWfError),
+}
+
+/// Errors thrown by [crate::Worker::poll_activity_task]
+#[derive(thiserror::Error, Debug)]
+pub enum PollActivityError {
+    /// [crate::Worker::shutdown] was called, we will no longer fetch new activity tasks. Lang must
+    /// ensure it is finished with any workflow replay, see [PollWfError::ShutDown]
+    #[error("Core is shut down")]
+    ShutDown,
+    /// Unhandled error when calling the temporal server. Core will attempt to retry any non-fatal
+    /// errors, so lang should consider this fatal.
+    #[error("Unhandled grpc error when activity polling: {0:?}")]
+    TonicError(#[from] tonic::Status),
+}
+
+/// Errors thrown by [crate::Worker::complete_workflow_activation]
+#[derive(thiserror::Error, Debug)]
+#[allow(clippy::large_enum_variant)]
+pub enum CompleteWfError {
+    /// Lang SDK sent us a malformed workflow completion. This likely means a bug in the lang sdk.
+    #[error("Lang SDK sent us a malformed workflow completion for run ({run_id}): {reason}")]
+    MalformedWorkflowCompletion {
+        /// Reason the completion was malformed
+        reason: String,
+        /// The run associated with the completion
+        run_id: String,
+    },
+}
+
+/// Errors thrown by [crate::Worker::complete_activity_task]
+#[derive(thiserror::Error, Debug)]
+pub enum CompleteActivityError {
+    /// Lang SDK sent us a malformed activity completion. This likely means a bug in the lang sdk.
+    #[error("Lang SDK sent us a malformed activity completion ({reason}): {completion:?}")]
+    MalformedActivityCompletion {
+        /// Reason the completion was malformed
+        reason: String,
+        /// The completion, which may not be included to avoid unnecessary copies.
+        completion: Option<ActivityExecutionResult>,
+    },
+}
+
+/// Errors thrown inside of workflow machines
+#[derive(thiserror::Error, Debug)]
+pub enum WFMachinesError {
+    #[error("Nondeterminism error: {0}")]
+    Nondeterminism(String),
+    #[error("Fatal error in workflow machines: {0}")]
+    Fatal(String),
+
+    #[error("Unrecoverable network error while fetching history: {0}")]
+    HistoryFetchingError(tonic::Status),
+}
+
+impl WFMachinesError {
+    pub fn evict_reason(&self) -> EvictionReason {
+        match self {
+            WFMachinesError::Nondeterminism(_) => EvictionReason::Nondeterminism,
+            WFMachinesError::Fatal(_) | WFMachinesError::HistoryFetchingError(_) => {
+                EvictionReason::Fatal
+            }
+        }
+    }
+}
+
+impl From<TimestampError> for WFMachinesError {
+    fn from(_: TimestampError) -> Self {
+        Self::Fatal("Could not decode timestamp".to_string())
+    }
+}

data/bridge/sdk-core/core-api/src/lib.rs

@@ -0,0 +1,109 @@
+pub mod errors;
+pub mod telemetry;
+pub mod worker;
+
+use crate::{
+    errors::{CompleteActivityError, CompleteWfError, PollActivityError, PollWfError},
+    worker::WorkerConfig,
+};
+use temporal_sdk_core_protos::coresdk::{
+    activity_task::ActivityTask, workflow_activation::WorkflowActivation,
+    workflow_completion::WorkflowActivationCompletion, ActivityHeartbeat, ActivityTaskCompletion,
+};
+
+/// This trait is the primary way by which language specific SDKs interact with the core SDK.
+/// It represents one worker, which has a (potentially shared) client for connecting to the service
+/// and is bound to a specific task queue.
+#[async_trait::async_trait]
+pub trait Worker: Send + Sync {
+    /// Ask the worker for some work, returning a [WorkflowActivation]. It is then the language
+    /// SDK's responsibility to call the appropriate workflow code with the provided inputs. Blocks
+    /// indefinitely until such work is available or [Worker::shutdown] is called.
+    ///
+    /// It is important to understand that all activations must be responded to. There can only
+    /// be one outstanding activation for a particular run of a workflow at any time. If an
+    /// activation is not responded to, it will cause that workflow to become stuck forever.
+    ///
+    /// Activations that contain only a `remove_from_cache` job should not cause the workflow code
+    /// to be invoked and may be responded to with an empty command list. Eviction jobs may also
+    /// appear with other jobs, but will always appear last in the job list. In this case it is
+    /// expected that the workflow code will be invoked, and the response produced as normal, but
+    /// the caller should evict the run after doing so.
+    ///
+    /// It is rarely a good idea to call poll concurrently. It handles polling the server
+    /// concurrently internally.
+    async fn poll_workflow_activation(&self) -> Result<WorkflowActivation, PollWfError>;
+
+    /// Ask the worker for some work, returning an [ActivityTask]. It is then the language SDK's
+    /// responsibility to call the appropriate activity code with the provided inputs. Blocks
+    /// indefinitely until such work is available or [Worker::shutdown] is called.
+    ///
+    /// The returned activation is guaranteed to be for the same task queue / worker which was
+    /// provided as the `task_queue` argument.
+    ///
+    /// It is rarely a good idea to call poll concurrently. It handles polling the server
+    /// concurrently internally.
+    async fn poll_activity_task(&self) -> Result<ActivityTask, PollActivityError>;
+
+    /// Tell the worker that a workflow activation has completed. May be freely called concurrently.
+    async fn complete_workflow_activation(
+        &self,
+        completion: WorkflowActivationCompletion,
+    ) -> Result<(), CompleteWfError>;
+
+    /// Tell the worker that an activity has finished executing. May be freely called concurrently.
+    async fn complete_activity_task(
+        &self,
+        completion: ActivityTaskCompletion,
+    ) -> Result<(), CompleteActivityError>;
+
+    /// Notify the Temporal service that an activity is still alive. Long running activities that
+    /// take longer than `activity_heartbeat_timeout` to finish must call this function in order to
+    /// report progress, otherwise the activity will timeout and a new attempt will be scheduled.
+    ///
+    /// The first heartbeat request will be sent immediately, subsequent rapid calls to this
+    /// function will result in heartbeat requests being aggregated and the last one received during
+    /// the aggregation period will be sent to the server, where that period is defined as half the
+    /// heartbeat timeout.
+    ///
+    /// Unlike Java/Go SDKs we do not return cancellation status as part of heartbeat response and
+    /// instead send it as a separate activity task to the lang, decoupling heartbeat and
+    /// cancellation processing.
+    ///
+    /// For now activity still need to send heartbeats if they want to receive cancellation
+    /// requests. In the future we will change this and will dispatch cancellations more
+    /// proactively. Note that this function does not block on the server call and returns
+    /// immediately. Underlying validation errors are swallowed and logged, this has been agreed to
+    /// be optimal behavior for the user as we don't want to break activity execution due to badly
+    /// configured heartbeat options.
+    fn record_activity_heartbeat(&self, details: ActivityHeartbeat);
+
+    /// Request that a workflow be evicted by its run id. This will generate a workflow activation
+    /// with the eviction job inside it to be eventually returned by
+    /// [Worker::poll_workflow_activation]. If the workflow had any existing outstanding activations,
+    /// such activations are invalidated and subsequent completions of them will do nothing and log
+    /// a warning.
+    fn request_workflow_eviction(&self, run_id: &str);
+
+    /// Return this worker's config
+    fn get_config(&self) -> &WorkerConfig;
+
+    /// Initiate shutdown. See [Worker::shutdown], this is just a sync version that starts the
+    /// process. You can then wait on `shutdown` or [Worker::finalize_shutdown].
+    fn initiate_shutdown(&self);
+
+    /// Initiates async shutdown procedure, eventually ceases all polling of the server and shuts
+    /// down this worker. [Worker::poll_workflow_activation] should be called until it
+    /// returns [PollWfError::ShutDown] to ensure that any workflows which are still undergoing
+    /// replay have an opportunity to finish. This means that the lang sdk will need to call
+    /// [Worker::complete_workflow_activation] for those workflows until they are done. At that point,
+    /// the lang SDK can end the process, or drop the [Worker] instance, which will close the
+    /// connection.
+    async fn shutdown(&self);
+
+    /// Completes shutdown and frees all resources. You should avoid simply dropping workers, as
+    /// this does not allow async tasks to report any panics that may have occurred cleanly.
+    ///
+    /// This should be called only after [Worker::shutdown] has resolved.
+    async fn finalize_shutdown(self);
+}

data/bridge/sdk-core/core-api/src/telemetry.rs

@@ -0,0 +1,147 @@
+use opentelemetry::metrics::Meter;
+use std::{
+    collections::HashMap,
+    net::SocketAddr,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+use tracing_core::Level;
+use url::Url;
+
+/// Each core runtime instance has a telemetry subsystem associated with it, this trait defines the
+/// operations that lang might want to perform on that telemetry after it's initialized.
+pub trait CoreTelemetry {
+    /// Each worker buffers logs that should be shuttled over to lang so that they may be rendered
+    /// with the user's desired logging library. Use this function to grab the most recent buffered
+    /// logs since the last time it was called. A fixed number of such logs are retained at maximum,
+    /// with the oldest being dropped when full.
+    ///
+    /// Returns the list of logs from oldest to newest. Returns an empty vec if the feature is not
+    /// configured.
+    fn fetch_buffered_logs(&self) -> Vec<CoreLog>;
+
+    /// If metrics gathering is enabled, returns the OTel meter for core telemetry, which can be
+    /// used to create metrics instruments, or passed to things that create/record metrics (ex:
+    /// clients).
+    fn get_metric_meter(&self) -> Option<&Meter>;
+}
+
+/// Telemetry configuration options. Construct with [TelemetryOptionsBuilder]
+#[derive(Debug, Clone, derive_builder::Builder)]
+#[non_exhaustive]
+pub struct TelemetryOptions {
+    /// Optional trace exporter - set as None to disable.
+    #[builder(setter(into, strip_option), default)]
+    pub tracing: Option<TraceExportConfig>,
+    /// Optional logger - set as None to disable.
+    #[builder(setter(into, strip_option), default)]
+    pub logging: Option<Logger>,
+    /// Optional metrics exporter - set as None to disable.
+    #[builder(setter(into, strip_option), default)]
+    pub metrics: Option<MetricsExporter>,
+
+    /// If set true, do not prefix metrics with `temporal_`. Will be removed eventually as
+    /// the prefix is consistent with other SDKs.
+    #[builder(default)]
+    pub no_temporal_prefix_for_metrics: bool,
+
+    /// Specifies the aggregation temporality for metric export. Defaults to cumulative.
+    #[builder(default = "MetricTemporality::Cumulative")]
+    pub metric_temporality: MetricTemporality,
+}
+
+/// Options for exporting to an OpenTelemetry Collector
+#[derive(Debug, Clone)]
+pub struct OtelCollectorOptions {
+    /// The url of the OTel collector to export telemetry and metrics to. Lang SDK should also
+    /// export to this same collector.
+    pub url: Url,
+    /// Optional set of HTTP headers to send to the Collector, e.g for authentication.
+    pub headers: HashMap<String, String>,
+    /// Optionally specify how frequently metrics should be exported. Defaults to 1 second.
+    pub metric_periodicity: Option<Duration>,
+}
+
+/// Configuration for the external export of traces
+#[derive(Debug, Clone)]
+pub struct TraceExportConfig {
+    /// An [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html) filter string.
+    pub filter: String,
+    /// Where they should go
+    pub exporter: TraceExporter,
+}
+
+/// Control where traces are exported.
+#[derive(Debug, Clone)]
+pub enum TraceExporter {
+    /// Export traces to an OpenTelemetry Collector <https://opentelemetry.io/docs/collector/>.
+    Otel(OtelCollectorOptions),
+}
+
+/// Control where metrics are exported
+#[derive(Debug, Clone)]
+pub enum MetricsExporter {
+    /// Export metrics to an OpenTelemetry Collector <https://opentelemetry.io/docs/collector/>.
+    Otel(OtelCollectorOptions),
+    /// Expose metrics directly via an embedded http server bound to the provided address.
+    Prometheus(SocketAddr),
+}
+
+/// Control where logs go
+#[derive(Debug, Clone)]
+pub enum Logger {
+    /// Log directly to console.
+    Console {
+        /// An [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html) filter string.
+        filter: String,
+    },
+    /// Forward logs to Lang - collectable with `fetch_global_buffered_logs`.
+    Forward {
+        /// An [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html) filter string.
+        filter: String,
+    },
+}
+
+/// Types of aggregation temporality for metric export.
+/// See: <https://github.com/open-telemetry/opentelemetry-specification/blob/ce50e4634efcba8da445cc23523243cb893905cb/specification/metrics/datamodel.md#temporality>
+#[derive(Debug, Clone, Copy)]
+pub enum MetricTemporality {
+    /// Successive data points repeat the starting timestamp
+    Cumulative,
+    /// Successive data points advance the starting timestamp
+    Delta,
+}
+
+impl Default for TelemetryOptions {
+    fn default() -> Self {
+        TelemetryOptionsBuilder::default().build().unwrap()
+    }
+}
+
+/// A log line (which ultimately came from a tracing event) exported from Core->Lang
+#[derive(Debug)]
+pub struct CoreLog {
+    /// The module within core this message originated from
+    pub target: String,
+    /// Log message
+    pub message: String,
+    /// Time log was generated (not when it was exported to lang)
+    pub timestamp: SystemTime,
+    /// Message level
+    pub level: Level,
+    /// Arbitrary k/v pairs (span k/vs are collapsed with event k/vs here). We could change this
+    /// to include them in `span_contexts` instead, but there's probably not much value for log
+    /// forwarding.
+    pub fields: HashMap<String, serde_json::Value>,
+    /// A list of the outermost to the innermost span names
+    pub span_contexts: Vec<String>,
+}
+
+impl CoreLog {
+    /// Return timestamp as ms since epoch
+    pub fn millis_since_epoch(&self) -> u128 {
+        self.timestamp
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or(Duration::ZERO)
+            .as_millis()
+    }
+}

data/bridge/sdk-core/core-api/src/worker.rs

@@ -0,0 +1,148 @@
+use std::time::Duration;
+
+/// Defines per-worker configuration options
+#[derive(Debug, Clone, derive_builder::Builder)]
+#[builder(setter(into), build_fn(validate = "Self::validate"))]
+#[non_exhaustive]
+pub struct WorkerConfig {
+    /// The Temporal service namespace this worker is bound to
+    pub namespace: String,
+    /// What task queue will this worker poll from? This task queue name will be used for both
+    /// workflow and activity polling.
+    pub task_queue: String,
+    /// A string that should be unique to the set of code this worker uses. IE: All the workflow,
+    /// activity, interceptor, and data converter code.
+    pub worker_build_id: String,
+    /// A human-readable string that can identify this worker. Using something like sdk version
+    /// and host name is a good default. If set, overrides the identity set (if any) on the client
+    /// used by this worker.
+    #[builder(default)]
+    pub client_identity_override: Option<String>,
+    /// If set nonzero, workflows will be cached and sticky task queues will be used, meaning that
+    /// history updates are applied incrementally to suspended instances of workflow execution.
+    /// Workflows are evicted according to a least-recently-used policy one the cache maximum is
+    /// reached. Workflows may also be explicitly evicted at any time, or as a result of errors
+    /// or failures.
+    #[builder(default = "0")]
+    pub max_cached_workflows: usize,
+    /// The maximum allowed number of workflow tasks that will ever be given to this worker at one
+    /// time. Note that one workflow task may require multiple activations - so the WFT counts as
+    /// "outstanding" until all activations it requires have been completed.
+    ///
+    /// Cannot be larger than `max_cached_workflows`.
+    #[builder(default = "100")]
+    pub max_outstanding_workflow_tasks: usize,
+    /// The maximum number of activity tasks that will ever be given to this worker concurrently
+    #[builder(default = "100")]
+    pub max_outstanding_activities: usize,
+    /// The maximum number of local activity tasks that will ever be given to this worker
+    /// concurrently
+    #[builder(default = "100")]
+    pub max_outstanding_local_activities: usize,
+    /// Maximum number of concurrent poll workflow task requests we will perform at a time on this
+    /// worker's task queue. See also [WorkerConfig::nonsticky_to_sticky_poll_ratio]. Must be at
+    /// least 1.
+    #[builder(default = "5")]
+    pub max_concurrent_wft_polls: usize,
+    /// [WorkerConfig::max_concurrent_wft_polls] * this number = the number of max pollers that will
+    /// be allowed for the nonsticky queue when sticky tasks are enabled. If both defaults are used,
+    /// the sticky queue will allow 4 max pollers while the nonsticky queue will allow one. The
+    /// minimum for either poller is 1, so if `max_concurrent_wft_polls` is 1 and sticky queues are
+    /// enabled, there will be 2 concurrent polls.
+    #[builder(default = "0.2")]
+    pub nonsticky_to_sticky_poll_ratio: f32,
+    /// Maximum number of concurrent poll activity task requests we will perform at a time on this
+    /// worker's task queue
+    #[builder(default = "5")]
+    pub max_concurrent_at_polls: usize,
+    /// If set to true this worker will only handle workflow tasks and local activities, it will not
+    /// poll for activity tasks.
+    #[builder(default = "false")]
+    pub no_remote_activities: bool,
+    /// How long a workflow task is allowed to sit on the sticky queue before it is timed out
+    /// and moved to the non-sticky queue where it may be picked up by any worker.
+    #[builder(default = "Duration::from_secs(10)")]
+    pub sticky_queue_schedule_to_start_timeout: Duration,
+
+    /// Longest interval for throttling activity heartbeats
+    #[builder(default = "Duration::from_secs(60)")]
+    pub max_heartbeat_throttle_interval: Duration,
+
+    /// Default interval for throttling activity heartbeats in case
+    /// `ActivityOptions.heartbeat_timeout` is unset.
+    /// When the timeout *is* set in the `ActivityOptions`, throttling is set to
+    /// `heartbeat_timeout * 0.8`.
+    #[builder(default = "Duration::from_secs(30)")]
+    pub default_heartbeat_throttle_interval: Duration,
+
+    /// Sets the maximum number of activities per second the task queue will dispatch, controlled
+    /// server-side. Note that this only takes effect upon an activity poll request. If multiple
+    /// workers on the same queue have different values set, they will thrash with the last poller
+    /// winning.
+    #[builder(default)]
+    pub max_task_queue_activities_per_second: Option<f64>,
+
+    /// Limits the number of activities per second that this worker will process. The worker will
+    /// not poll for new activities if by doing so it might receive and execute an activity which
+    /// would cause it to exceed this limit. Negative, zero, or NaN values will cause building
+    /// the options to fail.
+    #[builder(default)]
+    pub max_worker_activities_per_second: Option<f64>,
+
+    /// # UNDER DEVELOPMENT
+    /// If set to true this worker will opt-in to the whole-worker versioning feature.
+    /// `worker_build_id` will be used as the version.
+    /// todo: link to feature docs
+    #[builder(default = "false")]
+    pub use_worker_versioning: bool,
+
+    /// If set false (default), shutdown will not finish until all pending evictions have been
+    /// issued and replied to. If set true shutdown will be considered complete when the only
+    /// remaining work is pending evictions.
+    ///
+    /// This flag is useful during tests to avoid needing to deal with lots of uninteresting
+    /// evictions during shutdown. Alternatively, if a lang implementation finds it easy to clean
+    /// up during shutdown, setting this true saves some back-and-forth.
+    #[builder(default = "false")]
+    pub ignore_evicts_on_shutdown: bool,
+}
+
+impl WorkerConfig {
+    pub fn max_nonsticky_polls(&self) -> usize {
+        ((self.max_concurrent_wft_polls as f32 * self.nonsticky_to_sticky_poll_ratio) as usize)
+            .max(1)
+    }
+    pub fn max_sticky_polls(&self) -> usize {
+        self.max_concurrent_wft_polls
+            .saturating_sub(self.max_nonsticky_polls())
+            .max(1)
+    }
+}
+
+impl WorkerConfigBuilder {
+    fn validate(&self) -> Result<(), String> {
+        if self.max_concurrent_wft_polls == Some(0) {
+            return Err("`max_concurrent_wft_polls` must be at least 1".to_owned());
+        }
+        if self.max_concurrent_at_polls == Some(0) {
+            return Err("`max_concurrent_at_polls` must be at least 1".to_owned());
+        }
+        if self.max_cached_workflows > Some(0)
+            && self.max_outstanding_workflow_tasks > self.max_cached_workflows
+        {
+            return Err(
+                "Maximum concurrent workflow tasks cannot exceed the maximum number of cached \
+                 workflows"
+                    .to_owned(),
+            );
+        }
+        if let Some(Some(ref x)) = self.max_worker_activities_per_second {
+            if !x.is_normal() || x.is_sign_negative() {
+                return Err(
+                    "`max_worker_activities_per_second` must be positive and nonzero".to_owned(),
+                );
+            }
+        }
+        Ok(())
+    }
+}