@temporalio/core-bridge 1.6.0 → 1.7.1

package/sdk-core/core/src/worker/workflow/run_cache.rs

@@ -2,19 +2,21 @@ use crate::{
     telemetry::metrics::workflow_type,
     worker::workflow::{
         managed_run::{ManagedRun, RunUpdateAct},
-        HistoryUpdate, LocalActivityRequestSink, PermittedWFT,
+        HistoryUpdate, LocalActivityRequestSink, PermittedWFT, RunBasics,
     },
     MetricsContext,
 };
 use lru::LruCache;
-use std::{mem, num::NonZeroUsize, sync::Arc};
+use std::{mem, num::NonZeroUsize, rc::Rc};
+use temporal_sdk_core_protos::temporal::api::workflowservice::v1::get_system_info_response;
 
 pub(super) struct RunCache {
     max: usize,
     namespace: String,
+    server_capabilities: get_system_info_response::Capabilities,
     /// Run id -> Data
     runs: LruCache<String, ManagedRun>,
-    local_activity_request_sink: Arc<dyn LocalActivityRequestSink>,
+    local_activity_request_sink: Rc<dyn LocalActivityRequestSink>,
 
     metrics: MetricsContext,
 }
@@ -23,7 +25,8 @@ impl RunCache {
     pub fn new(
         max_cache_size: usize,
         namespace: String,
-        local_activity_request_sink: Arc<dyn LocalActivityRequestSink>,
+        server_capabilities: get_system_info_response::Capabilities,
+        local_activity_request_sink: impl LocalActivityRequestSink,
         metrics: MetricsContext,
     ) -> Self {
         // The cache needs room for at least one run, otherwise we couldn't do anything. In
@@ -36,10 +39,11 @@ impl RunCache {
         Self {
             max: max_cache_size,
             namespace,
+            server_capabilities,
             runs: LruCache::new(
                 NonZeroUsize::new(lru_size).expect("LRU size is guaranteed positive"),
             ),
-            local_activity_request_sink,
+            local_activity_request_sink: Rc::new(local_activity_request_sink),
             metrics,
         }
     }
@@ -63,13 +67,16 @@ impl RunCache {
         // with the update.
         let history_update = mem::replace(&mut pwft.work.update, HistoryUpdate::dummy());
         let mut mrh = ManagedRun::new(
-            history_update,
-            self.namespace.clone(),
-            pwft.work.execution.workflow_id.clone(),
-            pwft.work.workflow_type.clone(),
-            pwft.work.execution.run_id.clone(),
+            RunBasics {
+                namespace: self.namespace.clone(),
+                workflow_id: pwft.work.execution.workflow_id.clone(),
+                workflow_type: pwft.work.workflow_type.clone(),
+                run_id: pwft.work.execution.run_id.clone(),
+                history: history_update,
+                metrics,
+                capabilities: &self.server_capabilities,
+            },
             self.local_activity_request_sink.clone(),
-            metrics,
         );
         let run_id = run_id.to_string();
         let rur = mrh.incoming_wft(pwft);

package/sdk-core/core/src/worker/workflow/wft_extraction.rs

@@ -12,6 +12,7 @@ use crate::{
 use futures::Stream;
 use futures_util::{stream, stream::PollNext, FutureExt, StreamExt};
 use std::{future, sync::Arc};
+use temporal_sdk_core_protos::TaskToken;
 use tracing::Span;
 
 /// Transforms incoming validated WFTs and history fetching requests into [PermittedWFT]s ready
@@ -30,6 +31,7 @@ pub(super) enum WFTExtractorOutput {
     FailedFetch {
         run_id: String,
         err: tonic::Status,
+        auto_reply_fail_tt: Option<TaskToken>,
     },
     PollerDead,
 }
@@ -62,13 +64,18 @@ impl WFTExtractor {
                     match wft {
                         Ok(wft) => {
                             let run_id = wft.workflow_execution.run_id.clone();
+                            let tt = wft.task_token.clone();
                             Ok(match HistoryPaginator::from_poll(wft, client).await {
                                 Ok((pag, prep)) => WFTExtractorOutput::NewWFT(PermittedWFT {
                                     work: prep,
-                                    permit,
+                                    permit: permit.into_used(),
                                     paginator: pag,
                                 }),
-                                Err(err) => WFTExtractorOutput::FailedFetch { run_id, err },
+                                Err(err) => WFTExtractorOutput::FailedFetch {
+                                    run_id,
+                                    err,
+                                    auto_reply_fail_tt: Some(tt),
+                                },
                             })
                         }
                         Err(e) => Err(e),
@@ -96,7 +103,11 @@ impl WFTExtractor {
                             let run_id = req.original_wft.work.execution.run_id.clone();
                             match HistoryPaginator::from_fetchreq(req, client).await {
                                 Ok(r) => WFTExtractorOutput::FetchResult(r, rc),
-                                Err(err) => WFTExtractorOutput::FailedFetch { run_id, err },
+                                Err(err) => WFTExtractorOutput::FailedFetch {
+                                    run_id,
+                                    err,
+                                    auto_reply_fail_tt: None,
+                                },
                             }
                         }
                         HistoryFetchReq::NextPage(mut req, rc) => {
@@ -110,6 +121,7 @@ impl WFTExtractor {
                                 Err(err) => WFTExtractorOutput::FailedFetch {
                                     run_id: req.paginator.run_id,
                                     err,
+                                    auto_reply_fail_tt: None,
                                 },
                             }
                         }

package/sdk-core/core/src/worker/workflow/workflow_stream/saved_wf_inputs.rs

@@ -1,6 +1,7 @@
 use crate::{
     telemetry::metrics::MetricsContext,
     worker::{
+        client::mocks::DEFAULT_TEST_CAPABILITIES,
         workflow::{
             workflow_stream::{WFStream, WFStreamInput},
             LAReqSink, LocalActivityRequestSink,
@@ -45,6 +46,7 @@ pub async fn replay_wf_state_inputs(mut config: WorkerConfig, inputs: impl Strea
         &mut config,
         MetricsContext::no_op(),
         CancellationToken::new(),
+        DEFAULT_TEST_CAPABILITIES.clone(),
     );
     let sink = ReadingFromFileLaReqSink {
         resolutions: la_resp_q,

package/sdk-core/core/src/worker/workflow/workflow_stream.rs

@@ -19,10 +19,7 @@ use crate::{
 use futures::{stream, stream::PollNext, Stream, StreamExt};
 use std::{collections::VecDeque, fmt::Debug, future, sync::Arc};
 use temporal_sdk_core_api::errors::PollWfError;
-use temporal_sdk_core_protos::{
-    coresdk::workflow_activation::remove_from_cache::EvictionReason,
-    temporal::api::enums::v1::WorkflowTaskFailedCause,
-};
+use temporal_sdk_core_protos::coresdk::workflow_activation::remove_from_cache::EvictionReason;
 use tokio_util::sync::CancellationToken;
 use tracing::{Level, Span};
 
@@ -100,7 +97,8 @@ impl WFStream {
             runs: RunCache::new(
                 basics.max_cached_workflows,
                 basics.namespace.clone(),
-                Arc::new(local_activity_request_sink),
+                basics.server_capabilities.clone(),
+                local_activity_request_sink,
                 basics.metrics.clone(),
             ),
             shutdown_token: basics.shutdown_token,
@@ -165,11 +163,16 @@ impl WFStream {
                             }
                         }
                     }
-                    WFStreamInput::FailedFetch { run_id, err } => state
+                    WFStreamInput::FailedFetch {
+                        run_id,
+                        err,
+                        auto_reply_fail_tt,
+                    } => state
                         .request_eviction(RequestEvictMsg {
                             run_id,
                             message: format!("Fetching history failed: {err:?}"),
                             reason: EvictionReason::PaginationOrHistoryFetch,
+                            auto_reply_fail_tt,
                         })
                         .into_run_update_resp(),
                     WFStreamInput::PollerDead => {
@@ -282,21 +285,23 @@ impl WFStream {
         };
         let mut acts: Vec<_> = match complete {
             NewOrFetchedComplete::New(complete) => match complete.completion {
-                ValidatedCompletion::Success { commands, .. } => {
-                    match rh.successful_completion(commands, complete.response_tx) {
-                        Ok(acts) => acts,
-                        Err(npr) => {
-                            self.runs_needing_fetching
-                                .push_back(HistoryFetchReq::NextPage(
-                                    npr,
-                                    self.history_fetch_refcounter.clone(),
-                                ));
-                            None
-                        }
+                ValidatedCompletion::Success {
+                    commands,
+                    used_flags,
+                    ..
+                } => match rh.successful_completion(commands, used_flags, complete.response_tx) {
+                    Ok(acts) => acts,
+                    Err(npr) => {
+                        self.runs_needing_fetching
+                            .push_back(HistoryFetchReq::NextPage(
+                                npr,
+                                self.history_fetch_refcounter.clone(),
+                            ));
+                        None
                     }
-                }
+                },
                 ValidatedCompletion::Fail { failure, .. } => rh.failed_completion(
-                    WorkflowTaskFailedCause::Unspecified,
+                    failure.force_cause(),
                     EvictionReason::LangFail,
                     failure,
                     complete.response_tx,
@@ -333,37 +338,39 @@ impl WFStream {
 
         // If we reported to server, we always want to mark it complete.
         let maybe_t = self.complete_wft(run_id, report.wft_report_status);
-        // Delete the activation
-        let activation = self
-            .runs
-            .get_mut(run_id)
-            .and_then(|rh| rh.delete_activation());
-
-        // Evict the run if the activation contained an eviction
-        let mut applied_buffered_poll_for_this_run = false;
-        if activation.map(|a| a.has_eviction()).unwrap_or_default() {
-            debug!(run_id=%run_id, "Evicting run");
-
-            if let Some(mut rh) = self.runs.remove(run_id) {
-                if let Some(buff) = rh.take_buffered_wft() {
-                    // Don't try to apply a buffered poll for this run if we just got a new WFT
-                    // from completing, because by definition that buffered poll is now an
-                    // out-of-date WFT.
-                    if wft_from_complete.is_none() {
-                        res = self.instantiate_or_update(buff);
-                        applied_buffered_poll_for_this_run = true;
+        // Delete the activation, but only if the report came from lang, or we know the outstanding
+        // activation is expected to be completed internally.
+        if let Some(activation) = self.runs.get_mut(run_id).and_then(|rh| {
+            rh.delete_activation(|act| {
+                !report.is_autocomplete || matches!(act, OutstandingActivation::Autocomplete)
+            })
+        }) {
+            // Evict the run if the activation contained an eviction
+            let mut applied_buffered_poll_for_this_run = false;
+            if activation.has_eviction() {
+                debug!(run_id=%run_id, "Evicting run");
+
+                if let Some(mut rh) = self.runs.remove(run_id) {
+                    if let Some(buff) = rh.take_buffered_wft() {
+                        // Don't try to apply a buffered poll for this run if we just got a new WFT
+                        // from completing, because by definition that buffered poll is now an
+                        // out-of-date WFT.
+                        if wft_from_complete.is_none() {
+                            res = self.instantiate_or_update(buff);
+                            applied_buffered_poll_for_this_run = true;
+                        }
                     }
                 }
-            }
 
-            // Attempt to apply a buffered poll for some *other* run, if we didn't have a wft
-            // from complete or a buffered poll for *this* run.
-            if wft_from_complete.is_none() && !applied_buffered_poll_for_this_run {
-                if let Some(buff) = self.buffered_polls_need_cache_slot.pop_front() {
-                    res = self.instantiate_or_update(buff);
+                // Attempt to apply a buffered poll for some *other* run, if we didn't have a wft
+                // from complete or a buffered poll for *this* run.
+                if wft_from_complete.is_none() && !applied_buffered_poll_for_this_run {
+                    if let Some(buff) = self.buffered_polls_need_cache_slot.pop_front() {
+                        res = self.instantiate_or_update(buff);
+                    }
                 }
-            }
-        };
+            };
+        }
 
         if let Some((wft, pag)) = wft_from_complete {
             debug!(run_id=%wft.execution.run_id, "New WFT from completion");
@@ -425,6 +432,7 @@ impl WFStream {
                 run_id,
                 message: "Workflow cache full".to_string(),
                 reason: EvictionReason::CacheFull,
+                auto_reply_fail_tt: None,
             })
         } else {
             // This branch shouldn't really be possible
@@ -513,6 +521,7 @@ impl WFStream {
                     run_id,
                     message: "Workflow cache full".to_string(),
                     reason: EvictionReason::CacheFull,
+                    auto_reply_fail_tt: None,
                 })
                 .into_run_update_resp(),
             );
@@ -581,6 +590,7 @@ enum WFStreamInput {
             serde(with = "tonic_status_serde::SerdeStatus")
         )]
         err: tonic::Status,
+        auto_reply_fail_tt: Option<TaskToken>,
     },
 }
 
@@ -652,6 +662,7 @@ enum ExternalPollerInputs {
     FailedFetch {
         run_id: String,
         err: tonic::Status,
+        auto_reply_fail_tt: Option<TaskToken>,
     },
 }
 impl From<ExternalPollerInputs> for WFStreamInput {
@@ -661,9 +672,15 @@ impl From<ExternalPollerInputs> for WFStreamInput {
             ExternalPollerInputs::PollerDead => WFStreamInput::PollerDead,
             ExternalPollerInputs::PollerError(e) => WFStreamInput::PollerError(e),
             ExternalPollerInputs::FetchedUpdate(wft) => WFStreamInput::NewWft(wft),
-            ExternalPollerInputs::FailedFetch { run_id, err } => {
-                WFStreamInput::FailedFetch { run_id, err }
-            }
+            ExternalPollerInputs::FailedFetch {
+                run_id,
+                err,
+                auto_reply_fail_tt,
+            } => WFStreamInput::FailedFetch {
+                run_id,
+                err,
+                auto_reply_fail_tt,
+            },
             ExternalPollerInputs::NextPage {
                 paginator,
                 update,
@@ -692,9 +709,15 @@ impl From<Result<WFTExtractorOutput, tonic::Status>> for ExternalPollerInputs {
                 update,
                 span,
             },
-            Ok(WFTExtractorOutput::FailedFetch { run_id, err }) => {
-                ExternalPollerInputs::FailedFetch { run_id, err }
-            }
+            Ok(WFTExtractorOutput::FailedFetch {
+                run_id,
+                err,
+                auto_reply_fail_tt,
+            }) => ExternalPollerInputs::FailedFetch {
+                run_id,
+                err,
+                auto_reply_fail_tt,
+            },
             Ok(WFTExtractorOutput::PollerDead) => ExternalPollerInputs::PollerDead,
             Err(e) => ExternalPollerInputs::PollerError(e),
         }

package/sdk-core/core-api/Cargo.toml

@@ -15,7 +15,6 @@ categories = ["development-tools"]
 [dependencies]
 async-trait = "0.1"
 derive_builder = "0.12"
-opentelemetry = "0.18"
 prost-types = "0.11"
 serde = { version = "1.0", default_features = false, features = ["derive"] }
 serde_json = "1.0"

package/sdk-core/core-api/src/lib.rs

@@ -97,17 +97,23 @@ pub trait Worker: Send + Sync {
     fn initiate_shutdown(&self);
 
     /// Initiates async shutdown procedure, eventually ceases all polling of the server and shuts
-    /// down this worker. [Worker::poll_workflow_activation] should be called until it
-    /// returns [PollWfError::ShutDown] to ensure that any workflows which are still undergoing
-    /// replay have an opportunity to finish. This means that the lang sdk will need to call
-    /// [Worker::complete_workflow_activation] for those workflows until they are done. At that point,
-    /// the lang SDK can end the process, or drop the [Worker] instance, which will close the
-    /// connection.
+    /// down this worker. [Worker::poll_workflow_activation] and [Worker::poll_activity_task] should
+    /// be called until both return a `ShutDown` error to ensure that all outstanding work is
+    /// complete. This means that the lang sdk will need to call
+    /// [Worker::complete_workflow_activation] and [Worker::complete_activity_task] for those
+    /// workflows & activities until they are done. At that point, the lang SDK can end the process,
+    /// or drop the [Worker] instance via [Worker::finalize_shutdown], which will close the
+    /// connection and free resources. If you have set [WorkerConfig::no_remote_activities], you may
+    /// skip calling [Worker::poll_activity_task].
+    ///
+    /// Lang implementations should use [Worker::initiate_shutdown] followed by
+    /// [Worker::finalize_shutdown].
     async fn shutdown(&self);
 
     /// Completes shutdown and frees all resources. You should avoid simply dropping workers, as
     /// this does not allow async tasks to report any panics that may have occurred cleanly.
     ///
-    /// This should be called only after [Worker::shutdown] has resolved.
+    /// This should be called only after [Worker::shutdown] has resolved and/or both polling
+    /// functions have returned `ShutDown` errors.
     async fn finalize_shutdown(self);
 }

package/sdk-core/core-api/src/telemetry.rs

@@ -1,4 +1,3 @@
-use opentelemetry::metrics::Meter;
 use std::{
     collections::HashMap,
     net::SocketAddr,
@@ -18,11 +17,6 @@ pub trait CoreTelemetry {
     /// Returns the list of logs from oldest to newest. Returns an empty vec if the feature is not
     /// configured.
     fn fetch_buffered_logs(&self) -> Vec<CoreLog>;
-
-    /// If metrics gathering is enabled, returns the OTel meter for core telemetry, which can be
-    /// used to create metrics instruments, or passed to things that create/record metrics (ex:
-    /// clients).
-    fn get_metric_meter(&self) -> Option<&Meter>;
 }
 
 /// Telemetry configuration options. Construct with [TelemetryOptionsBuilder]
@@ -47,6 +41,10 @@ pub struct TelemetryOptions {
     /// Specifies the aggregation temporality for metric export. Defaults to cumulative.
     #[builder(default = "MetricTemporality::Cumulative")]
     pub metric_temporality: MetricTemporality,
+
+    // A map of tags to be applied to all metrics
+    #[builder(default)]
+    pub global_tags: HashMap<String, String>,
 }
 
 /// Options for exporting to an OpenTelemetry Collector

package/sdk-core/core-api/src/worker.rs

@@ -118,6 +118,11 @@ pub struct WorkerConfig {
     #[builder(default)]
     #[serde(skip)]
     pub wf_state_inputs: Option<UnboundedSender<Vec<u8>>>,
+
+    /// If set, core will issue cancels for all outstanding activities after shutdown has been
+    /// initiated and this amount of time has elapsed.
+    #[builder(default)]
+    pub graceful_shutdown_period: Option<Duration>,
 }
 
 impl WorkerConfig {