@temporalio/core-bridge 0.19.2 → 0.20.2

package/sdk-core/core-api/src/lib.rs

@@ -2,39 +2,25 @@ pub mod errors;
 pub mod worker;
 
 use crate::{
-    errors::{
-        CompleteActivityError, CompleteWfError, PollActivityError, PollWfError,
-        WorkerRegistrationError,
-    },
+    errors::{CompleteActivityError, CompleteWfError, PollActivityError, PollWfError},
     worker::WorkerConfig,
 };
 use log::Level;
-use std::{
-    sync::Arc,
-    time::{Duration, SystemTime, UNIX_EPOCH},
-};
-use temporal_client::ServerGatewayApis;
+use opentelemetry::metrics::Meter;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use temporal_sdk_core_protos::coresdk::{
     activity_task::ActivityTask, workflow_activation::WorkflowActivation,
     workflow_completion::WorkflowActivationCompletion, ActivityHeartbeat, ActivityTaskCompletion,
 };
 
-/// This trait is the primary way by which language specific SDKs interact with the core SDK. It is
-/// expected that only one instance of an implementation will exist for the lifetime of the
-/// worker(s) using it.
+/// This trait is the primary way by which language specific SDKs interact with the core SDK.
+/// It represents one worker, which has a (potentially shared) client for connecting to the service
+/// and is bound to a specific task queue.
 #[async_trait::async_trait]
-pub trait Core: Send + Sync {
-    /// Register a worker with core. Workers poll on a specific task queue, and when calling core's
-    /// poll functions, you must provide a task queue name. If there was already a worker registered
-    /// with the same task queue name, it will be shut down and a new one will be created.
-    fn register_worker(&self, config: WorkerConfig) -> Result<(), WorkerRegistrationError>;
-
-    /// Ask the core for some work, returning a [WorkflowActivation]. It is then the language SDK's
-    /// responsibility to call the appropriate workflow code with the provided inputs. Blocks
-    /// indefinitely until such work is available or [Core::shutdown] is called.
-    ///
-    /// The returned activation is guaranteed to be for the same task queue / worker which was
-    /// provided as the `task_queue` argument.
+pub trait Worker: Send + Sync {
+    /// Ask the worker for some work, returning a [WorkflowActivation]. It is then the language
+    /// SDK's responsibility to call the appropriate workflow code with the provided inputs. Blocks
+    /// indefinitely until such work is available or [Worker::shutdown] is called.
     ///
     /// It is important to understand that all activations must be responded to. There can only
     /// be one outstanding activation for a particular run of a workflow at any time. If an
@@ -48,49 +34,41 @@ pub trait Core: Send + Sync {
     ///
     /// It is rarely a good idea to call poll concurrently. It handles polling the server
     /// concurrently internally.
-    ///
-    /// TODO: Examples
-    async fn poll_workflow_activation(
-        &self,
-        task_queue: &str,
-    ) -> Result<WorkflowActivation, PollWfError>;
+    async fn poll_workflow_activation(&self) -> Result<WorkflowActivation, PollWfError>;
 
-    /// Ask the core for some work, returning an [ActivityTask]. It is then the language SDK's
+    /// Ask the worker for some work, returning an [ActivityTask]. It is then the language SDK's
     /// responsibility to call the appropriate activity code with the provided inputs. Blocks
-    /// indefinitely until such work is available or [Core::shutdown] is called.
+    /// indefinitely until such work is available or [Worker::shutdown] is called.
     ///
     /// The returned activation is guaranteed to be for the same task queue / worker which was
     /// provided as the `task_queue` argument.
     ///
     /// It is rarely a good idea to call poll concurrently. It handles polling the server
     /// concurrently internally.
-    ///
-    /// TODO: Examples
-    async fn poll_activity_task(&self, task_queue: &str)
-        -> Result<ActivityTask, PollActivityError>;
+    async fn poll_activity_task(&self) -> Result<ActivityTask, PollActivityError>;
 
-    /// Tell the core that a workflow activation has completed. May be freely called concurrently.
+    /// Tell the worker that a workflow activation has completed. May be freely called concurrently.
     async fn complete_workflow_activation(
         &self,
         completion: WorkflowActivationCompletion,
     ) -> Result<(), CompleteWfError>;
 
-    /// Tell the core that an activity has finished executing. May be freely called concurrently.
+    /// Tell the worker that an activity has finished executing. May be freely called concurrently.
     async fn complete_activity_task(
         &self,
         completion: ActivityTaskCompletion,
     ) -> Result<(), CompleteActivityError>;
 
-    /// Notify workflow that an activity is still alive. Long running activities that take longer
-    /// than `activity_heartbeat_timeout` to finish must call this function in order to report
-    /// progress, otherwise the activity will timeout and a new attempt will be scheduled.
+    /// Notify the Temporal service that an activity is still alive. Long running activities that
+    /// take longer than `activity_heartbeat_timeout` to finish must call this function in order to
+    /// report progress, otherwise the activity will timeout and a new attempt will be scheduled.
     ///
     /// The first heartbeat request will be sent immediately, subsequent rapid calls to this
     /// function will result in heartbeat requests being aggregated and the last one received during
     /// the aggregation period will be sent to the server, where that period is defined as half the
     /// heartbeat timeout.
     ///
-    /// Unlike java/go SDKs we do not return cancellation status as part of heartbeat response and
+    /// Unlike Java/Go SDKs we do not return cancellation status as part of heartbeat response and
     /// instead send it as a separate activity task to the lang, decoupling heartbeat and
     /// cancellation processing.
     ///
@@ -104,27 +82,36 @@ pub trait Core: Send + Sync {
 
     /// Request that a workflow be evicted by its run id. This will generate a workflow activation
     /// with the eviction job inside it to be eventually returned by
-    /// [Core::poll_workflow_activation]. If the workflow had any existing outstanding activations,
+    /// [Worker::poll_workflow_activation]. If the workflow had any existing outstanding activations,
     /// such activations are invalidated and subsequent completions of them will do nothing and log
     /// a warning.
-    fn request_workflow_eviction(&self, task_queue: &str, run_id: &str);
+    fn request_workflow_eviction(&self, run_id: &str);
+
+    /// Return this worker's config
+    fn get_config(&self) -> &WorkerConfig;
 
-    /// Returns core's instance of the [ServerGatewayApis] implementor it is using.
-    fn server_gateway(&self) -> Arc<dyn ServerGatewayApis + Send + Sync>;
+    /// TODO: Will be replaced/fixed/whatever by shutdown refactoring
+    fn initiate_shutdown(&self);
 
     /// Initiates async shutdown procedure, eventually ceases all polling of the server and shuts
-    /// down all registered workers. [Core::poll_workflow_activation] should be called until it
+    /// down this worker. [Worker::poll_workflow_activation] should be called until it
     /// returns [PollWfError::ShutDown] to ensure that any workflows which are still undergoing
     /// replay have an opportunity to finish. This means that the lang sdk will need to call
-    /// [Core::complete_workflow_activation] for those workflows until they are done. At that point,
-    /// the lang SDK can end the process, or drop the [Core] instance, which will close the
+    /// [Worker::complete_workflow_activation] for those workflows until they are done. At that point,
+    /// the lang SDK can end the process, or drop the [Worker] instance, which will close the
     /// connection.
     async fn shutdown(&self);
 
-    /// Shut down a specific worker. Will cease all polling on the task queue and future attempts
-    /// to poll that queue will return [PollWfError::NoWorkerForQueue].
-    async fn shutdown_worker(&self, task_queue: &str);
+    /// Completes shutdown and frees all resources. You should avoid simply dropping workers, as
+    /// this does not allow async tasks to report any panics that may have occurred cleanly.
+    ///
+    /// This should be called only after [Worker::shutdown] has resolved.
+    async fn finalize_shutdown(self);
+}
 
+/// Should be backed by a process-wide singleton who is responsible for telemetry and logging
+/// management.
+pub trait CoreTelemetry {
     /// Core buffers logs that should be shuttled over to lang so that they may be rendered with
     /// the user's desired logging library. Use this function to grab the most recent buffered logs
     /// since the last time it was called. A fixed number of such logs are retained at maximum, with
@@ -133,6 +120,11 @@ pub trait Core: Send + Sync {
     /// Returns the list of logs from oldest to newest. Returns an empty vec if the feature is not
     /// configured.
     fn fetch_buffered_logs(&self) -> Vec<CoreLog>;
+
+    /// If metrics gathering is enabled, returns the OTel meter for core telemetry, which can be
+    /// used to create metrics instruments, or passed to things that create/record metrics (ex:
+    /// clients).
+    fn get_metric_meter(&self) -> Option<&Meter>;
 }
 
 /// A log line (which ultimately came from a tracing event) exported from Core->Lang

package/sdk-core/core-api/src/worker.rs

@@ -4,8 +4,9 @@ use std::time::Duration;
 #[derive(Debug, Clone, derive_builder::Builder)]
 #[builder(setter(into), build_fn(validate = "Self::validate"))]
 #[non_exhaustive]
-// TODO: per-second queue limits
 pub struct WorkerConfig {
+    /// The Temporal service namespace this worker is bound to
+    pub namespace: String,
     /// What task queue will this worker poll from? This task queue name will be used for both
     /// workflow and activity polling.
     pub task_queue: String,
@@ -45,7 +46,7 @@ pub struct WorkerConfig {
     #[builder(default = "5")]
     pub max_concurrent_at_polls: usize,
     /// If set to true this worker will only handle workflow tasks and local activities, it will not
-    /// poll for activity tasks. This option exists because
+    /// poll for activity tasks.
     #[builder(default = "false")]
     pub no_remote_activities: bool,
     /// How long a workflow task is allowed to sit on the sticky queue before it is timed out
@@ -63,6 +64,13 @@ pub struct WorkerConfig {
     /// `heartbeat_timeout * 0.8`.
     #[builder(default = "Duration::from_secs(30)")]
     pub default_heartbeat_throttle_interval: Duration,
+
+    /// Sets the maximum number of activities per second the task queue will dispatch, controlled
+    /// server-side. Note that this only takes effect upon an activity poll request. If multiple
+    /// workers on the same queue have different values set, they will thrash with the last poller
+    /// winning.
+    #[builder(setter(strip_option), default)]
+    pub max_task_queue_activities_per_second: Option<f64>,
 }
 
 impl WorkerConfig {

package/sdk-core/etc/deps.svg

@@ -1,156 +1,187 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<!-- Generated by graphviz version 2.50.0 (0)
+<!-- Generated by graphviz version 2.43.0 (0)
  -->
-<!-- Pages: 1 -->
-<svg width="421pt" height="404pt"
- viewBox="0.00 0.00 420.50 404.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 400)">
-<polygon fill="white" stroke="transparent" points="-4,4 -4,-400 416.5,-400 416.5,4 -4,4"/>
+<!-- Title: %3 Pages: 1 -->
+<svg width="525pt" height="476pt"
+ viewBox="0.00 0.00 524.50 476.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 472)">
+<title>%3</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-472 520.5,-472 520.5,4 -4,4"/>
 <!-- 0 -->
 <g id="node1" class="node">
 <title>0</title>
-<polygon fill="none" stroke="black" points="248,-396 132,-396 132,-360 248,-360 248,-396"/>
-<text text-anchor="middle" x="190" y="-374.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk&#45;core</text>
+<polygon fill="none" stroke="black" points="173,-468 0,-468 0,-432 173,-432 173,-468"/>
+<text text-anchor="middle" x="86.5" y="-446.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk&#45;core&#45;bridge&#45;ffi</text>
 </g>
 <!-- 1 -->
 <g id="node2" class="node">
 <title>1</title>
-<polygon fill="none" stroke="black" points="58,-324 0,-324 0,-288 58,-288 58,-324"/>
-<text text-anchor="middle" x="29" y="-302.3" font-family="Times,serif" font-size="14.00">rustfsm</text>
+<polygon fill="none" stroke="black" points="369.5,-396 253.5,-396 253.5,-360 369.5,-360 369.5,-396"/>
+<text text-anchor="middle" x="311.5" y="-374.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk&#45;core</text>
 </g>
 <!-- 0&#45;&gt;1 -->
 <g id="edge1" class="edge">
 <title>0&#45;&gt;1</title>
-<path fill="none" stroke="black" d="M150.61,-359.88C125.27,-348.86 92.5,-334.61 67.36,-323.68"/>
-<polygon fill="black" stroke="black" points="68.7,-320.44 58.13,-319.67 65.91,-326.86 68.7,-320.44"/>
-</g>
-<!-- 4 -->
-<g id="node3" class="node">
-<title>4</title>
-<polygon fill="none" stroke="black" points="214,-108 114,-108 114,-72 214,-72 214,-108"/>
-<text text-anchor="middle" x="164" y="-86.3" font-family="Times,serif" font-size="14.00">temporal&#45;client</text>
-</g>
-<!-- 0&#45;&gt;4 -->
-<g id="edge2" class="edge">
-<title>0&#45;&gt;4</title>
-<path fill="none" stroke="black" d="M181.7,-359.96C171.44,-325.92 157.74,-247.02 153,-180 151.87,-164.04 152.7,-160 153,-144 153.16,-135.58 153.46,-126.42 154.19,-118.12"/>
-<polygon fill="black" stroke="black" points="157.67,-118.52 155.37,-108.18 150.71,-117.7 157.67,-118.52"/>
-</g>
-<!-- 0&#45;&gt;4 -->
-<g id="edge3" class="edge">
-<title>0&#45;&gt;4</title>
-<path fill="none" stroke="blue" d="M191.8,-359.96C189.44,-325.92 175.74,-247.02 171,-180 169.87,-164.04 170.7,-160 171,-144 171.16,-135.68 171.45,-126.63 171.37,-118.4"/>
-<polygon fill="blue" stroke="blue" points="174.86,-118.05 171.01,-108.18 167.87,-118.3 174.86,-118.05"/>
-</g>
-<!-- 5 -->
-<g id="node4" class="node">
-<title>5</title>
-<polygon fill="none" stroke="black" points="319.5,-36 164.5,-36 164.5,0 319.5,0 319.5,-36"/>
-<text text-anchor="middle" x="242" y="-14.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk&#45;core&#45;protos</text>
-</g>
-<!-- 0&#45;&gt;5 -->
-<g id="edge6" class="edge">
-<title>0&#45;&gt;5</title>
-<path fill="none" stroke="black" d="M165.88,-359.82C135.16,-335.73 86,-288.78 86,-235 86,-235 86,-235 86,-161 86,-120.55 78.76,-102.78 105,-72 118.09,-56.64 136.18,-45.72 154.93,-37.95"/>
-<polygon fill="black" stroke="black" points="156.39,-41.14 164.46,-34.27 153.87,-34.6 156.39,-41.14"/>
+<path fill="none" stroke="black" d="M141.25,-431.97C173.1,-422.06 213.42,-409.51 246.83,-399.12"/>
+<polygon fill="black" stroke="black" points="247.99,-402.42 256.5,-396.11 245.91,-395.74 247.99,-402.42"/>
 </g>
 <!-- 6 -->
 <g id="node5" class="node">
 <title>6</title>
-<polygon fill="none" stroke="black" points="307.5,-180 170.5,-180 170.5,-144 307.5,-144 307.5,-180"/>
-<text text-anchor="middle" x="239" y="-158.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk&#45;core&#45;api</text>
+<polygon fill="none" stroke="black" points="401,-36 246,-36 246,0 401,0 401,-36"/>
+<text text-anchor="middle" x="323.5" y="-14.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk&#45;core&#45;protos</text>
 </g>
 <!-- 0&#45;&gt;6 -->
-<g id="edge5" class="edge">
+<g id="edge3" class="edge">
 <title>0&#45;&gt;6</title>
-<path fill="none" stroke="black" d="M193.8,-359.93C197.8,-342.01 204.29,-313.01 210,-288 217.68,-254.36 226.68,-215.68 232.6,-190.36"/>
-<polygon fill="black" stroke="black" points="236.06,-190.92 234.93,-180.38 229.24,-189.32 236.06,-190.92"/>
+<path fill="none" stroke="black" d="M78.81,-431.91C67.71,-405.62 48.5,-353.45 48.5,-307 48.5,-307 48.5,-307 48.5,-161 48.5,-75.33 155.16,-41.29 235.63,-27.8"/>
+<polygon fill="black" stroke="black" points="236.47,-31.21 245.8,-26.18 235.37,-24.3 236.47,-31.21"/>
 </g>
 <!-- 7 -->
 <g id="node6" class="node">
 <title>7</title>
-<polygon fill="none" stroke="black" points="412.5,-252 323.5,-252 323.5,-216 412.5,-216 412.5,-252"/>
-<text text-anchor="middle" x="368" y="-230.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk</text>
+<polygon fill="none" stroke="black" points="266,-180 129,-180 129,-144 266,-144 266,-180"/>
+<text text-anchor="middle" x="197.5" y="-158.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk&#45;core&#45;api</text>
 </g>
 <!-- 0&#45;&gt;7 -->
-<g id="edge4" class="edge">
+<g id="edge2" class="edge">
 <title>0&#45;&gt;7</title>
-<path fill="none" stroke="blue" d="M224.19,-359.83C242.9,-350.78 265.76,-338.54 284,-324 307.09,-305.6 328.26,-279.21 343.85,-259.94"/>
-<polygon fill="blue" stroke="blue" points="346.58,-262.13 350.18,-252.17 341.15,-257.71 346.58,-262.13"/>
+<path fill="none" stroke="black" d="M85.71,-431.86C84.92,-401.65 85.97,-337.48 105.5,-288 120.88,-249.04 151.5,-211.31 173.18,-187.68"/>
+<polygon fill="black" stroke="black" points="175.88,-189.92 180.16,-180.23 170.77,-185.14 175.88,-189.92"/>
+</g>
+<!-- 2 -->
+<g id="node3" class="node">
+<title>2</title>
+<polygon fill="none" stroke="black" points="172.5,-324 114.5,-324 114.5,-288 172.5,-288 172.5,-324"/>
+<text text-anchor="middle" x="143.5" y="-302.3" font-family="Times,serif" font-size="14.00">rustfsm</text>
+</g>
+<!-- 1&#45;&gt;2 -->
+<g id="edge4" class="edge">
+<title>1&#45;&gt;2</title>
+<path fill="none" stroke="black" d="M270.4,-359.88C243.41,-348.63 208.34,-334.02 181.91,-323"/>
+<polygon fill="black" stroke="black" points="183.17,-319.74 172.59,-319.12 180.47,-326.2 183.17,-319.74"/>
+</g>
+<!-- 5 -->
+<g id="node4" class="node">
+<title>5</title>
+<polygon fill="none" stroke="black" points="412.5,-108 312.5,-108 312.5,-72 412.5,-72 412.5,-108"/>
+<text text-anchor="middle" x="362.5" y="-86.3" font-family="Times,serif" font-size="14.00">temporal&#45;client</text>
+</g>
+<!-- 1&#45;&gt;5 -->
+<g id="edge5" class="edge">
+<title>1&#45;&gt;5</title>
+<path fill="none" stroke="black" d="M369.63,-361.71C391.38,-353.54 414.82,-341.45 431.5,-324 455.23,-299.18 453.17,-285.75 459.5,-252 462.45,-236.27 464.68,-231.14 459.5,-216 445.69,-175.67 412.97,-138.29 389.35,-115.12"/>
+<polygon fill="black" stroke="black" points="391.67,-112.5 382.03,-108.11 386.83,-117.55 391.67,-112.5"/>
+</g>
+<!-- 1&#45;&gt;6 -->
+<g id="edge8" class="edge">
+<title>1&#45;&gt;6</title>
+<path fill="none" stroke="black" d="M369.7,-363.34C430.3,-345.24 516.5,-306.87 516.5,-235 516.5,-235 516.5,-235 516.5,-161 516.5,-98.28 449.55,-60.18 394.51,-39.46"/>
+<polygon fill="black" stroke="black" points="395.69,-36.16 385.1,-36.04 393.3,-42.74 395.69,-36.16"/>
+</g>
+<!-- 1&#45;&gt;7 -->
+<g id="edge7" class="edge">
+<title>1&#45;&gt;7</title>
+<path fill="none" stroke="black" d="M264.03,-359.96C247.39,-351.64 230.2,-339.91 219.5,-324 192.37,-283.65 191.8,-224.66 194.32,-190.41"/>
+<polygon fill="black" stroke="black" points="197.83,-190.49 195.22,-180.22 190.86,-189.87 197.83,-190.49"/>
 </g>
 <!-- 8 -->
 <g id="node7" class="node">
 <title>8</title>
-<polygon fill="none" stroke="black" points="283.5,-324 218.5,-324 218.5,-288 283.5,-288 283.5,-324"/>
-<text text-anchor="middle" x="251" y="-302.3" font-family="Times,serif" font-size="14.00">test_utils</text>
+<polygon fill="none" stroke="black" points="450,-252 361,-252 361,-216 450,-216 450,-252"/>
+<text text-anchor="middle" x="405.5" y="-230.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk</text>
 </g>
-<!-- 0&#45;&gt;8 -->
-<g id="edge7" class="edge">
-<title>0&#45;&gt;8</title>
-<path fill="none" stroke="blue" d="M199.16,-359.7C205.68,-351.22 214.51,-340.86 223.1,-331.58"/>
-<polygon fill="blue" stroke="blue" points="225.85,-333.77 230.2,-324.1 220.78,-328.95 225.85,-333.77"/>
+<!-- 1&#45;&gt;8 -->
+<g id="edge6" class="edge">
+<title>1&#45;&gt;8</title>
+<path fill="none" stroke="blue" d="M354.05,-359.78C369.82,-351.61 386.07,-340.02 395.5,-324 406.38,-305.52 405.76,-280.93 404.34,-262.27"/>
+<polygon fill="blue" stroke="blue" points="407.81,-261.75 403.49,-252.07 400.83,-262.32 407.81,-261.75"/>
 </g>
-<!-- 4&#45;&gt;5 -->
-<g id="edge8" class="edge">
-<title>4&#45;&gt;5</title>
-<path fill="none" stroke="black" d="M183.28,-71.7C192.92,-63.05 204.73,-52.45 215.23,-43.03"/>
-<polygon fill="black" stroke="black" points="217.84,-45.39 222.94,-36.1 213.16,-40.18 217.84,-45.39"/>
+<!-- 9 -->
+<g id="node8" class="node">
+<title>9</title>
+<polygon fill="none" stroke="black" points="395,-324 228,-324 228,-288 395,-288 395,-324"/>
+<text text-anchor="middle" x="311.5" y="-302.3" font-family="Times,serif" font-size="14.00">temporal&#45;sdk&#45;core&#45;test&#45;utils</text>
 </g>
-<!-- 6&#45;&gt;4 -->
+<!-- 1&#45;&gt;9 -->
 <g id="edge9" class="edge">
-<title>6&#45;&gt;4</title>
-<path fill="none" stroke="black" d="M220.46,-143.7C211.2,-135.05 199.84,-124.45 189.74,-115.03"/>
-<polygon fill="black" stroke="black" points="192.02,-112.37 182.33,-108.1 187.25,-117.49 192.02,-112.37"/>
+<title>1&#45;&gt;9</title>
+<path fill="none" stroke="blue" d="M305.58,-359.7C304.79,-351.98 304.56,-342.71 304.9,-334.11"/>
+<polygon fill="blue" stroke="blue" points="308.4,-334.32 305.6,-324.1 301.42,-333.84 308.4,-334.32"/>
 </g>
-<!-- 6&#45;&gt;5 -->
+<!-- 5&#45;&gt;6 -->
 <g id="edge10" class="edge">
-<title>6&#45;&gt;5</title>
-<path fill="none" stroke="black" d="M239.36,-143.87C239.87,-119.67 240.81,-75.21 241.42,-46.39"/>
-<polygon fill="black" stroke="black" points="244.92,-46.26 241.64,-36.19 237.93,-46.11 244.92,-46.26"/>
-</g>
-<!-- 7&#45;&gt;0 -->
-<g id="edge11" class="edge">
-<title>7&#45;&gt;0</title>
-<path fill="none" stroke="black" d="M363.18,-252.17C352.26,-271.5 328.39,-302.97 302,-324 286.4,-336.43 267.42,-347.19 249.5,-355.7"/>
-<polygon fill="black" stroke="black" points="247.88,-352.6 240.25,-359.95 250.8,-358.96 247.88,-352.6"/>
+<title>5&#45;&gt;6</title>
+<path fill="none" stroke="black" d="M352.86,-71.7C348.37,-63.64 342.94,-53.89 337.98,-44.98"/>
+<polygon fill="black" stroke="black" points="340.95,-43.14 333.03,-36.1 334.84,-46.54 340.95,-43.14"/>
 </g>
 <!-- 7&#45;&gt;5 -->
-<g id="edge13" class="edge">
+<g id="edge11" class="edge">
 <title>7&#45;&gt;5</title>
-<path fill="none" stroke="black" d="M371.05,-215.88C375.69,-184.29 380.99,-116.06 350,-72 340.45,-58.42 326.41,-48.16 311.6,-40.47"/>
-<polygon fill="black" stroke="black" points="312.84,-37.19 302.31,-36.01 309.81,-43.5 312.84,-37.19"/>
+<path fill="none" stroke="black" d="M237.86,-143.88C260.52,-134.26 288.97,-122.19 312.97,-112.01"/>
+<polygon fill="black" stroke="black" points="314.43,-115.2 322.27,-108.07 311.69,-108.75 314.43,-115.2"/>
 </g>
 <!-- 7&#45;&gt;6 -->
 <g id="edge12" class="edge">
 <title>7&#45;&gt;6</title>
-<path fill="none" stroke="black" d="M336.44,-215.88C319.34,-206.6 298.02,-195.03 279.68,-185.08"/>
-<polygon fill="black" stroke="black" points="281.14,-181.88 270.68,-180.19 277.8,-188.04 281.14,-181.88"/>
+<path fill="none" stroke="black" d="M210.74,-143.85C225.01,-125.62 248.63,-96.14 270.5,-72 279.24,-62.36 289.24,-52.16 298.18,-43.31"/>
+<polygon fill="black" stroke="black" points="300.7,-45.74 305.39,-36.24 295.8,-40.75 300.7,-45.74"/>
 </g>
-<!-- 8&#45;&gt;0 -->
-<g id="edge15" class="edge">
-<title>8&#45;&gt;0</title>
-<path fill="none" stroke="black" d="M241.99,-324.1C235.51,-332.55 226.69,-342.9 218.1,-352.2"/>
-<polygon fill="black" stroke="black" points="215.33,-350.03 210.99,-359.7 220.41,-354.85 215.33,-350.03"/>
+<!-- 8&#45;&gt;1 -->
+<g id="edge14" class="edge">
+<title>8&#45;&gt;1</title>
+<path fill="none" stroke="black" d="M416.59,-252.07C423.07,-271.04 426.51,-301.89 413.5,-324 405.57,-337.47 392.82,-347.8 378.95,-355.62"/>
+<polygon fill="black" stroke="black" points="376.99,-352.69 369.71,-360.39 380.2,-358.91 376.99,-352.69"/>
 </g>
 <!-- 8&#45;&gt;5 -->
-<g id="edge17" class="edge">
+<g id="edge13" class="edge">
 <title>8&#45;&gt;5</title>
-<path fill="none" stroke="black" d="M266.47,-287.62C290.36,-258.6 332,-198.15 317,-144 306.53,-106.2 281.25,-68.36 262.93,-44.38"/>
-<polygon fill="black" stroke="black" points="265.45,-41.92 256.54,-36.2 259.94,-46.23 265.45,-41.92"/>
+<path fill="none" stroke="black" d="M400.31,-215.87C392.95,-191.56 379.4,-146.82 370.68,-118.01"/>
+<polygon fill="black" stroke="black" points="373.95,-116.75 367.71,-108.19 367.25,-118.77 373.95,-116.75"/>
 </g>
 <!-- 8&#45;&gt;6 -->
 <g id="edge16" class="edge">
 <title>8&#45;&gt;6</title>
-<path fill="none" stroke="black" d="M249.55,-287.87C247.51,-263.67 243.75,-219.21 241.31,-190.39"/>
-<polygon fill="black" stroke="black" points="244.78,-189.86 240.45,-180.19 237.81,-190.45 244.78,-189.86"/>
+<path fill="none" stroke="black" d="M413.59,-215.56C426.78,-184.32 448.83,-117.84 421.5,-72 413.54,-58.64 400.91,-48.45 387.34,-40.75"/>
+<polygon fill="black" stroke="black" points="388.8,-37.56 378.32,-36.04 385.56,-43.77 388.8,-37.56"/>
 </g>
 <!-- 8&#45;&gt;7 -->
-<g id="edge14" class="edge">
+<g id="edge15" class="edge">
 <title>8&#45;&gt;7</title>
-<path fill="none" stroke="black" d="M279.62,-287.88C294.92,-278.72 313.94,-267.34 330.42,-257.48"/>
-<polygon fill="black" stroke="black" points="332.48,-260.33 339.27,-252.19 328.89,-254.32 332.48,-260.33"/>
+<path fill="none" stroke="black" d="M360.88,-217.98C330.78,-207.86 290.71,-194.37 257.79,-183.29"/>
+<polygon fill="black" stroke="black" points="258.86,-179.96 248.26,-180.08 256.62,-186.59 258.86,-179.96"/>
+</g>
+<!-- 9&#45;&gt;1 -->
+<g id="edge19" class="edge">
+<title>9&#45;&gt;1</title>
+<path fill="none" stroke="black" d="M317.4,-324.1C318.2,-331.79 318.44,-341.05 318.1,-349.67"/>
+<polygon fill="black" stroke="black" points="314.61,-349.48 317.42,-359.7 321.59,-349.96 314.61,-349.48"/>
+</g>
+<!-- 9&#45;&gt;5 -->
+<g id="edge17" class="edge">
+<title>9&#45;&gt;5</title>
+<path fill="none" stroke="black" d="M315.59,-287.85C324.43,-250.75 345.39,-162.81 356.04,-118.1"/>
+<polygon fill="black" stroke="black" points="359.48,-118.77 358.39,-108.23 352.67,-117.15 359.48,-118.77"/>
+</g>
+<!-- 9&#45;&gt;6 -->
+<g id="edge21" class="edge">
+<title>9&#45;&gt;6</title>
+<path fill="none" stroke="black" d="M308.43,-287.86C302.03,-249.05 288.97,-151.64 303.5,-72 305.11,-63.2 308.1,-53.95 311.29,-45.7"/>
+<polygon fill="black" stroke="black" points="314.62,-46.79 315.19,-36.21 308.15,-44.12 314.62,-46.79"/>
+</g>
+<!-- 9&#45;&gt;7 -->
+<g id="edge20" class="edge">
+<title>9&#45;&gt;7</title>
+<path fill="none" stroke="black" d="M297.75,-287.87C277.81,-263.03 240.73,-216.85 217.68,-188.14"/>
+<polygon fill="black" stroke="black" points="220.29,-185.8 211.3,-180.19 214.83,-190.18 220.29,-185.8"/>
+</g>
+<!-- 9&#45;&gt;8 -->
+<g id="edge18" class="edge">
+<title>9&#45;&gt;8</title>
+<path fill="none" stroke="black" d="M334.74,-287.7C346.69,-278.8 361.42,-267.82 374.35,-258.2"/>
+<polygon fill="black" stroke="black" points="376.6,-260.88 382.53,-252.1 372.42,-255.27 376.6,-260.88"/>
 </g>
 </g>
 </svg>

package/sdk-core/protos/api_upstream/temporal/api/command/v1/message.proto

@@ -48,17 +48,21 @@ message ScheduleActivityTaskCommandAttributes {
     temporal.api.taskqueue.v1.TaskQueue task_queue = 4;
     temporal.api.common.v1.Header header = 5;
     temporal.api.common.v1.Payloads input = 6;
-    // Indicates how long the caller is willing to wait for activity completion. Limits how long
-    // retries will be attempted. Either this or `start_to_close_timeout` must be specified. When
-    // not specified, defaults to the workflow execution timeout.
+    // Indicates how long the caller is willing to wait for activity completion. The "schedule" time
+    // is when the activity is initially scheduled, not when the most recent retry is scheduled.
+    // Limits how long retries will be attempted. Either this or `start_to_close_timeout` must be
+    // specified. When not specified, defaults to the workflow execution timeout.
     //
     // (-- api-linter: core::0140::prepositions=disabled
     //     aip.dev/not-precedent: "to" is used to indicate interval. --)
     google.protobuf.Duration schedule_to_close_timeout = 7 [(gogoproto.stdduration) = true];
-    // Limits the time an activity task can stay in a task queue before a worker picks it up. This
-    // timeout is always non retryable, as all a retry would achieve is to put it back into the same
-    // queue. Defaults to `schedule_to_close_timeout` or workflow execution timeout if that is not
-    // specified.
+    // Limits the time an activity task can stay in a task queue before a worker picks it up. The
+    // "schedule" time is when the most recent retry is scheduled. This timeout should usually not
+    // be set: it's useful in specific scenarios like worker-specific task queues. This timeout is
+    // always non retryable, as all a retry would achieve is to put it back into the same queue.
+    // Defaults to `schedule_to_close_timeout` or workflow execution timeout if that is not
+    // specified. More info:
+    // https://docs.temporal.io/docs/content/what-is-a-schedule-to-start-timeout/
     //
     // (-- api-linter: core::0140::prepositions=disabled
     //     aip.dev/not-precedent: "to" is used to indicate interval. --)

package/sdk-core/protos/api_upstream/temporal/api/enums/v1/failed_cause.proto

@@ -82,3 +82,13 @@ enum SignalExternalWorkflowExecutionFailedCause {
     SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_UNSPECIFIED = 0;
     SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_EXTERNAL_WORKFLOW_EXECUTION_NOT_FOUND = 1;
 }
+
+enum ResourceExhaustedCause {
+    RESOURCE_EXHAUSTED_CAUSE_UNSPECIFIED = 0;
+    // Caller exceeds request per second limit.
+    RESOURCE_EXHAUSTED_CAUSE_RPS_LIMIT = 1;
+    // Caller exceeds max concurrent request limit.
+    RESOURCE_EXHAUSTED_CAUSE_CONCURRENT_LIMIT = 2;
+    // System overloaded.
+    RESOURCE_EXHAUSTED_CAUSE_SYSTEM_OVERLOADED = 3;
+}

package/sdk-core/protos/api_upstream/temporal/api/enums/v1/namespace.proto

@@ -36,7 +36,6 @@ enum NamespaceState {
     NAMESPACE_STATE_REGISTERED = 1;
     NAMESPACE_STATE_DEPRECATED = 2;
     NAMESPACE_STATE_DELETED = 3;
-    NAMESPACE_STATE_HANDOVER = 4;
 }
 
 enum ArchivalState {
@@ -44,3 +43,9 @@ enum ArchivalState {
     ARCHIVAL_STATE_DISABLED = 1;
     ARCHIVAL_STATE_ENABLED = 2;
 }
+
+enum ReplicationState {
+    REPLICATION_STATE_UNSPECIFIED = 0;
+    REPLICATION_STATE_NORMAL = 1;
+    REPLICATION_STATE_HANDOVER = 2;
+}

package/sdk-core/protos/api_upstream/temporal/api/enums/v1/workflow.proto

@@ -87,6 +87,12 @@ enum PendingActivityState {
     PENDING_ACTIVITY_STATE_CANCEL_REQUESTED = 3;
 }
 
+enum PendingWorkflowTaskState {
+    PENDING_WORKFLOW_TASK_STATE_UNSPECIFIED = 0;
+    PENDING_WORKFLOW_TASK_STATE_SCHEDULED = 1;
+    PENDING_WORKFLOW_TASK_STATE_STARTED = 2;
+}
+
 enum HistoryEventFilterType {
     HISTORY_EVENT_FILTER_TYPE_UNSPECIFIED = 0;
     HISTORY_EVENT_FILTER_TYPE_ALL_EVENT = 1;

package/sdk-core/protos/api_upstream/temporal/api/errordetails/v1/message.proto

@@ -33,6 +33,8 @@ option java_outer_classname = "MessageProto";
 option ruby_package = "Temporal::Api::ErrorDetails::V1";
 option csharp_namespace = "Temporal.Api.ErrorDetails.V1";
 
+import "temporal/api/enums/v1/failed_cause.proto";
+
 message NotFoundFailure {
     string current_cluster = 1;
     string active_cluster = 2;
@@ -72,3 +74,7 @@ message QueryFailedFailure {
 message PermissionDeniedFailure {
     string reason = 1;
 }
+
+message ResourceExhaustedFailure {
+    temporal.api.enums.v1.ResourceExhaustedCause cause = 1;
+}

package/sdk-core/protos/api_upstream/temporal/api/history/v1/message.proto

@@ -81,7 +81,8 @@ message WorkflowExecutionStartedEventAttributes {
     google.protobuf.Timestamp workflow_execution_expiration_time = 19 [(gogoproto.stdtime) = true];
     // If this workflow runs on a cron schedule, it will appear here
     string cron_schedule = 20;
-    // TODO: What is this? Appears unused.
+    // For a cron workflow, this contains the amount of time between when this iteration of
+    // the cron workflow was scheduled and when it should run next per its cron_schedule.
     google.protobuf.Duration first_workflow_task_backoff = 21 [(gogoproto.stdduration) = true];
     temporal.api.common.v1.Memo memo = 22;
     temporal.api.common.v1.SearchAttributes search_attributes = 23;