@temporalio/core-bridge 1.7.0 → 1.7.2

package/sdk-core/core/src/worker/activities.rs

@@ -37,7 +37,10 @@ use governor::{Quota, RateLimiter};
 use std::{
     convert::TryInto,
     future,
-    sync::Arc,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
     time::{Duration, Instant},
 };
 use temporal_sdk_core_protos::{
@@ -117,8 +120,8 @@ impl RemoteInFlightActInfo {
 }
 
 pub(crate) struct WorkerActivityTasks {
-    /// Token used to signal the server task poller that shutdown is beginning
-    poller_shutdown_token: CancellationToken,
+    /// Token which is cancelled once shutdown is beginning
+    shutdown_initiated_token: CancellationToken,
     /// Centralizes management of heartbeat issuing / throttling
     heartbeat_manager: ActivityHeartbeatManager,
     /// Combined stream for any ActivityTask producing source (polls, eager activities,
@@ -169,7 +172,7 @@ impl WorkerActivityTasks {
             metrics.with_new_attrs([activity_worker_type()]),
             MetricsContext::available_task_slots,
         ));
-        let poller_shutdown_token = CancellationToken::new();
+        let shutdown_initiated_token = CancellationToken::new();
         let rate_limiter = max_worker_act_per_sec.and_then(|ps| {
             Quota::with_period(Duration::from_secs_f64(ps.recip())).map(RateLimiter::direct)
         });
@@ -179,7 +182,7 @@ impl WorkerActivityTasks {
             semaphore.clone(),
             rate_limiter,
             metrics.clone(),
-            poller_shutdown_token.clone(),
+            shutdown_initiated_token.clone(),
         );
         let (eager_activities_tx, eager_activities_rx) = unbounded_channel();
         let eager_activities_semaphore = ClosableMeteredSemaphore::new_arc(semaphore);
@@ -199,22 +202,21 @@ impl WorkerActivityTasks {
             starts_stream.map(ActivityTaskSource::from),
             |_: &mut ()| PollNext::Left,
         );
-        // Create a task stream composed of (in poll preference order):
-        //  cancels_stream ------------------------------+--- activity_task_stream
-        //  eager_activities_rx ---+--- starts_stream ---|
-        //  server_poll_stream  ---|
-        let activity_task_stream = Self::merge_source_streams(
+
+        let activity_task_stream = ActivityTaskStream {
             source_stream,
-            outstanding_activity_tasks.clone(),
+            outstanding_tasks: outstanding_activity_tasks.clone(),
             start_tasks_stream_complete,
-            complete_notify.clone(),
-            graceful_shutdown,
+            complete_notify: complete_notify.clone(),
+            grace_period: graceful_shutdown,
             cancels_tx,
-            metrics.clone(),
-        );
+            shutdown_initiated_token: shutdown_initiated_token.clone(),
+            metrics: metrics.clone(),
+        }
+        .streamify();
 
         Self {
-            poller_shutdown_token,
+            shutdown_initiated_token,
             eager_activities_tx,
             heartbeat_manager,
             activity_task_stream: Mutex::new(activity_task_stream.boxed()),
@@ -263,9 +265,7 @@ impl WorkerActivityTasks {
 
         // Prefer eager activities over polling the server
         stream::select_with_strategy(non_poll_stream, poller_stream, |_: &mut ()| PollNext::Left)
-            .map(|res| res.map_err(|err| err.into()))
-            // This map, chain, filter_map sequence is here to cancel the token when this stream ends.
-            .map(Some)
+            .map(|res| Some(res.map_err(Into::into)))
             .chain(futures::stream::once(async move {
                 on_complete_token.cancel();
                 None
@@ -273,98 +273,8 @@ impl WorkerActivityTasks {
             .filter_map(future::ready)
     }
 
-    /// Builds an [ActivityTask] stream for both cancellation tasks from cancels delivered from
-    /// heartbeats as well as new activity starts
-    fn merge_source_streams(
-        source_stream: impl Stream<Item = ActivityTaskSource>,
-        outstanding_tasks: Arc<DashMap<TaskToken, RemoteInFlightActInfo>>,
-        start_tasks_stream_complete: CancellationToken,
-        complete_notify: Arc<Notify>,
-        grace_period: Option<Duration>,
-        cancels_tx: UnboundedSender<PendingActivityCancel>,
-        metrics: MetricsContext,
-    ) -> impl Stream<Item = Result<ActivityTask, PollActivityError>> {
-        let outstanding_tasks_clone = outstanding_tasks.clone();
-        source_stream
-            .filter_map(move |source| {
-                let outstanding_tasks = outstanding_tasks.clone();
-                let metrics = metrics.clone();
-                async move {
-                    match source {
-                        ActivityTaskSource::PendingCancel(next_pc) => {
-                            // It's possible that activity has been completed and we no longer have
-                            // an outstanding activity task. This is fine because it means that we
-                            // no longer need to cancel this activity, so we'll just ignore such
-                            // orphaned cancellations.
-                            if let Some(mut details) =
-                                outstanding_tasks.get_mut(&next_pc.task_token)
-                            {
-                                if details.issued_cancel_to_lang.is_some() {
-                                    // Don't double-issue cancellations
-                                    return None;
-                                }
-
-                                details.issued_cancel_to_lang = Some(next_pc.reason);
-                                if next_pc.reason == ActivityCancelReason::NotFound {
-                                    details.known_not_found = true;
-                                }
-                                Some(Ok(ActivityTask::cancel_from_ids(
-                                    next_pc.task_token.0,
-                                    next_pc.reason,
-                                )))
-                            } else {
-                                debug!(task_token = ?next_pc.task_token,
-                                   "Unknown activity task when issuing cancel");
-                                // If we can't find the activity here, it's already been completed,
-                                // in which case issuing a cancel again is pointless.
-                                None
-                            }
-                        }
-                        ActivityTaskSource::PendingStart(res) => {
-                            Some(res.map(|(task, is_eager)| {
-                                Self::about_to_issue_task(
-                                    outstanding_tasks,
-                                    task,
-                                    is_eager,
-                                    metrics,
-                                )
-                            }))
-                        }
-                    }
-                }
-            })
-            .take_until(async move {
-                start_tasks_stream_complete.cancelled().await;
-                // Issue cancels for any still-living act tasks after the grace period
-                let (grace_killer, stop_grace) = futures_util::future::abortable(async {
-                    if let Some(gp) = grace_period {
-                        // Make sure we've waited at least the grace period. This way if waiting for
-                        // starts to finish took a while, we subtract that from the grace period.
-                        tokio::time::sleep(gp).await;
-                        for mapref in outstanding_tasks_clone.iter() {
-                            let _ = cancels_tx.send(PendingActivityCancel::new(
-                                mapref.key().clone(),
-                                ActivityCancelReason::WorkerShutdown,
-                            ));
-                        }
-                    }
-                });
-                join!(
-                    async {
-                        while !outstanding_tasks_clone.is_empty() {
-                            complete_notify.notified().await
-                        }
-                        // If we were waiting for the grace period but everything already finished,
-                        // we don't need to keep waiting.
-                        stop_grace.abort();
-                    },
-                    grace_killer
-                )
-            })
-    }
-
     pub(crate) fn initiate_shutdown(&self) {
-        self.poller_shutdown_token.cancel();
+        self.shutdown_initiated_token.cancel();
         self.eager_activities_semaphore.close();
     }
 
@@ -518,42 +428,142 @@ impl WorkerActivityTasks {
         }
     }
 
-    /// Called when there is a new [ActivityTask] about to be bubbled up out of the poller
-    fn about_to_issue_task(
-        outstanding_tasks: Arc<DashMap<TaskToken, RemoteInFlightActInfo>>,
-        task: PermittedTqResp,
-        is_eager: bool,
-        metrics: MetricsContext,
-    ) -> ActivityTask {
-        if let Some(ref act_type) = task.resp.activity_type {
-            if let Some(ref wf_type) = task.resp.workflow_type {
-                metrics
-                    .with_new_attrs([
-                        activity_type(act_type.name.clone()),
-                        workflow_type(wf_type.name.clone()),
-                        eager(is_eager),
-                    ])
-                    .act_task_received();
-            }
-        }
-        // There could be an else statement here but since the response should always contain both
-        // activity_type and workflow_type, we won't bother.
+    #[cfg(test)]
+    pub(crate) fn remaining_activity_capacity(&self) -> usize {
+        self.eager_activities_semaphore.available_permits()
+    }
+}
 
-        if let Some(dur) = task.resp.sched_to_start() {
-            metrics.act_sched_to_start_latency(dur);
-        };
+struct ActivityTaskStream<SrcStrm> {
+    source_stream: SrcStrm,
+    outstanding_tasks: Arc<DashMap<TaskToken, RemoteInFlightActInfo>>,
+    start_tasks_stream_complete: CancellationToken,
+    complete_notify: Arc<Notify>,
+    grace_period: Option<Duration>,
+    cancels_tx: UnboundedSender<PendingActivityCancel>,
+    /// Token which is cancelled once shutdown is beginning
+    shutdown_initiated_token: CancellationToken,
+    metrics: MetricsContext,
+}
 
-        outstanding_tasks.insert(
-            task.resp.task_token.clone().into(),
-            RemoteInFlightActInfo::new(&task.resp, task.permit.into_used()),
-        );
+impl<SrcStrm> ActivityTaskStream<SrcStrm>
+where
+    SrcStrm: Stream<Item = ActivityTaskSource>,
+{
+    /// Create a task stream composed of (in poll preference order):
+    ///  cancels_stream ------------------------------+--- activity_task_stream
+    ///  eager_activities_rx ---+--- starts_stream ---|
+    ///  server_poll_stream  ---|
+    fn streamify(self) -> impl Stream<Item = Result<ActivityTask, PollActivityError>> {
+        let outstanding_tasks_clone = self.outstanding_tasks.clone();
+        let should_issue_immediate_cancel = Arc::new(AtomicBool::new(false));
+        let should_issue_immediate_cancel_clone = should_issue_immediate_cancel.clone();
+        let cancels_tx = self.cancels_tx.clone();
+        self.source_stream
+            .filter_map(move |source| {
+                let res = match source {
+                    ActivityTaskSource::PendingCancel(next_pc) => {
+                        // It's possible that activity has been completed and we no longer have
+                        // an outstanding activity task. This is fine because it means that we
+                        // no longer need to cancel this activity, so we'll just ignore such
+                        // orphaned cancellations.
+                        if let Some(mut details) =
+                            self.outstanding_tasks.get_mut(&next_pc.task_token)
+                        {
+                            if details.issued_cancel_to_lang.is_some() {
+                                // Don't double-issue cancellations
+                                None
+                            } else {
+                                details.issued_cancel_to_lang = Some(next_pc.reason);
+                                if next_pc.reason == ActivityCancelReason::NotFound {
+                                    details.known_not_found = true;
+                                }
+                                Some(Ok(ActivityTask::cancel_from_ids(
+                                    next_pc.task_token.0,
+                                    next_pc.reason,
+                                )))
+                            }
+                        } else {
+                            debug!(task_token = ?next_pc.task_token,
+                                   "Unknown activity task when issuing cancel");
+                            // If we can't find the activity here, it's already been completed,
+                            // in which case issuing a cancel again is pointless.
+                            None
+                        }
+                    }
+                    ActivityTaskSource::PendingStart(res) => {
+                        Some(res.map(|(task, is_eager)| {
+                            if let Some(ref act_type) = task.resp.activity_type {
+                                if let Some(ref wf_type) = task.resp.workflow_type {
+                                    self.metrics
+                                        .with_new_attrs([
+                                            activity_type(act_type.name.clone()),
+                                            workflow_type(wf_type.name.clone()),
+                                            eager(is_eager),
+                                        ])
+                                        .act_task_received();
+                                }
+                            }
+                            // There could be an else statement here but since the response
+                            // should always contain both activity_type and workflow_type, we
+                            // won't bother.
 
-        ActivityTask::start_from_poll_resp(task.resp)
-    }
+                            if let Some(dur) = task.resp.sched_to_start() {
+                                self.metrics.act_sched_to_start_latency(dur);
+                            };
 
-    #[cfg(test)]
-    pub(crate) fn remaining_activity_capacity(&self) -> usize {
-        self.eager_activities_semaphore.available_permits()
+                            let tt: TaskToken = task.resp.task_token.clone().into();
+                            self.outstanding_tasks.insert(
+                                tt.clone(),
+                                RemoteInFlightActInfo::new(&task.resp, task.permit.into_used()),
+                            );
+                            // If we have already waited the grace period and issued cancels,
+                            // this will have been set true, indicating anything that happened
+                            // to be buffered/in-flight/etc should get an immediate cancel. This
+                            // is to allow the user to potentially decide to ignore cancels and
+                            // do work on polls that got received during shutdown.
+                            if should_issue_immediate_cancel.load(Ordering::Acquire) {
+                                let _ = cancels_tx.send(PendingActivityCancel::new(
+                                    tt,
+                                    ActivityCancelReason::WorkerShutdown,
+                                ));
+                            }
+
+                            ActivityTask::start_from_poll_resp(task.resp)
+                        }))
+                    }
+                };
+                async move { res }
+            })
+            .take_until(async move {
+                // Once we've been told to begin cancelling, wait the grace period and then start
+                // cancelling anything outstanding.
+                let (grace_killer, stop_grace) = futures_util::future::abortable(async {
+                    if let Some(gp) = self.grace_period {
+                        self.shutdown_initiated_token.cancelled().await;
+                        tokio::time::sleep(gp).await;
+                        should_issue_immediate_cancel_clone.store(true, Ordering::Release);
+                        for mapref in outstanding_tasks_clone.iter() {
+                            let _ = self.cancels_tx.send(PendingActivityCancel::new(
+                                mapref.key().clone(),
+                                ActivityCancelReason::WorkerShutdown,
+                            ));
+                        }
+                    }
+                });
+                join!(
+                    async {
+                        self.start_tasks_stream_complete.cancelled().await;
+                        while !outstanding_tasks_clone.is_empty() {
+                            self.complete_notify.notified().await
+                        }
+                        // If we were waiting for the grace period but everything already finished,
+                        // we don't need to keep waiting.
+                        stop_grace.abort();
+                    },
+                    grace_killer
+                )
+            })
     }
 }
 

package/sdk-core/core/src/worker/client.rs

@@ -7,12 +7,15 @@ use temporal_sdk_core_protos::{
     coresdk::workflow_commands::QueryResult,
     temporal::api::{
         command::v1::Command,
-        common::v1::{MeteringMetadata, Payloads, WorkflowExecution},
+        common::v1::{
+            MeteringMetadata, Payloads, WorkerVersionCapabilities, WorkerVersionStamp,
+            WorkflowExecution,
+        },
         enums::v1::{TaskQueueKind, WorkflowTaskFailedCause},
         failure::v1::Failure,
         query::v1::WorkflowQueryResult,
         sdk::v1::WorkflowTaskCompletedMetadata,
-        taskqueue::v1::{StickyExecutionAttributes, TaskQueue, TaskQueueMetadata, VersionId},
+        taskqueue::v1::{StickyExecutionAttributes, TaskQueue, TaskQueueMetadata},
         workflowservice::v1::{get_system_info_response::Capabilities, *},
     },
     TaskToken,
@@ -138,8 +141,8 @@ impl WorkerClient for WorkerClientBag {
             } else {
                 self.worker_build_id.clone()
             },
-            worker_versioning_id: Some(VersionId {
-                worker_build_id: self.versioning_build_id(),
+            worker_version_capabilities: Some(WorkerVersionCapabilities {
+                build_id: self.versioning_build_id(),
             }),
         };
 
@@ -166,8 +169,8 @@ impl WorkerClient for WorkerClientBag {
             task_queue_metadata: max_tasks_per_sec.map(|tps| TaskQueueMetadata {
                 max_tasks_per_second: Some(tps),
             }),
-            worker_versioning_id: Some(VersionId {
-                worker_build_id: self.versioning_build_id(),
+            worker_version_capabilities: Some(WorkerVersionCapabilities {
+                build_id: self.versioning_build_id(),
             }),
         };
 
@@ -190,8 +193,9 @@ impl WorkerClient for WorkerClientBag {
             sticky_attributes: request.sticky_attributes,
             return_new_workflow_task: request.return_new_workflow_task,
             force_create_new_workflow_task: request.force_create_new_workflow_task,
-            worker_versioning_id: Some(VersionId {
-                worker_build_id: self.versioning_build_id(),
+            worker_version_stamp: Some(WorkerVersionStamp {
+                build_id: self.versioning_build_id(),
+                bundle_id: "".to_string(),
             }),
             messages: vec![],
             binary_checksum: self.worker_build_id.clone(),

package/sdk-core/core/src/worker/mod.rs

@@ -146,6 +146,13 @@ impl WorkerTrait for Worker {
 
     /// Begins the shutdown process, tells pollers they should stop. Is idempotent.
     fn initiate_shutdown(&self) {
+        if !self.shutdown_token.is_cancelled() {
+            info!(
+                task_queue=%self.config.task_queue,
+                namespace=%self.config.namespace,
+                "Initiated shutdown",
+            );
+        }
         self.shutdown_token.cancel();
         // First, we want to stop polling of both activity and workflow tasks
         if let Some(atm) = self.at_task_mgr.as_ref() {
@@ -157,11 +164,6 @@ impl WorkerTrait for Worker {
         if !self.workflows.ever_polled() {
             self.local_act_mgr.workflows_have_shutdown();
         }
-        info!(
-            task_queue=%self.config.task_queue,
-            namespace=%self.config.namespace,
-            "Initiated shutdown",
-        );
     }
 
     async fn shutdown(&self) {
@@ -538,6 +540,7 @@ impl Worker {
         self.workflows
             .activation_completed(
                 completion,
+                false,
                 self.post_activate_hook
                     .as_ref()
                     .map(|h| |data: PostActivateHookData| h(self, data)),

package/sdk-core/core/src/worker/workflow/history_update.rs

@@ -84,6 +84,7 @@ pub struct HistoryPaginator {
     pub(crate) run_id: String,
     pub(crate) previous_wft_started_id: i64,
     pub(crate) wft_started_event_id: i64,
+    id_of_last_event_in_last_extracted_update: Option<i64>,
 
     #[cfg_attr(feature = "save_wf_inputs", serde(skip))]
     client: Arc<dyn WorkerClient>,
@@ -175,6 +176,7 @@ impl HistoryPaginator {
             run_id: req.original_wft.work.execution.run_id.clone(),
             previous_wft_started_id: req.original_wft.work.update.previous_wft_started_id,
             wft_started_event_id: req.original_wft.work.update.wft_started_id,
+            id_of_last_event_in_last_extracted_update: None,
             client,
             event_queue: Default::default(),
             next_page_token: NextPageToken::FetchFromStart,
@@ -211,6 +213,7 @@ impl HistoryPaginator {
             final_events,
             previous_wft_started_id,
             wft_started_event_id,
+            id_of_last_event_in_last_extracted_update: None,
         }
     }
 
@@ -226,6 +229,7 @@ impl HistoryPaginator {
             final_events: vec![],
             previous_wft_started_id: -2,
             wft_started_event_id: -2,
+            id_of_last_event_in_last_extracted_update: None,
         }
     }
 
@@ -240,14 +244,45 @@ impl HistoryPaginator {
     /// we have two, or until we are at the end of history.
     pub(crate) async fn extract_next_update(&mut self) -> Result<HistoryUpdate, tonic::Status> {
         loop {
-            let no_next_page = !self.get_next_page().await?;
+            let fetch_happened = !self.get_next_page().await?;
             let current_events = mem::take(&mut self.event_queue);
             let seen_enough_events = current_events
                 .back()
                 .map(|e| e.event_id)
                 .unwrap_or_default()
                 >= self.wft_started_event_id;
-            if current_events.is_empty() || (no_next_page && !seen_enough_events) {
+
+            // This handles a special case where the server might send us a page token along with
+            // a real page which ends at the current end of history. The page token then points to
+            // en empty page. We need to detect this, and consider it the end of history.
+            //
+            // This case unfortunately cannot be handled earlier, because we might fetch a page
+            // from the server which contains two complete WFTs, and thus we are happy to return
+            // an update at that time. But, if the page has a next page token, we *cannot* conclude
+            // we are done with replay until we fetch that page. So, we have to wait until the next
+            // extraction to determine (after fetching the next page and finding it to be empty)
+            // that we are done. Fetching the page eagerly is another option, but would be wasteful
+            // the overwhelming majority of the time.
+            let already_sent_update_with_enough_events = self
+                .id_of_last_event_in_last_extracted_update
+                .unwrap_or_default()
+                >= self.wft_started_event_id;
+            if current_events.is_empty()
+                && !fetch_happened
+                && already_sent_update_with_enough_events
+            {
+                // We must return an empty update which also says is contains the final WFT so we
+                // know we're done with replay.
+                return Ok(HistoryUpdate::from_events(
+                    [],
+                    self.previous_wft_started_id,
+                    self.wft_started_event_id,
+                    true,
+                )
+                .0);
+            }
+
+            if current_events.is_empty() || (fetch_happened && !seen_enough_events) {
                 // If next page fetching happened, and we still ended up with no or insufficient
                 // events, something is wrong. We're expecting there to be more events to be able to
                 // extract this update, but server isn't giving us any. We have no choice except to
@@ -278,6 +313,8 @@ impl HistoryPaginator {
                 // There was not a meaningful WFT in the whole page. We must fetch more.
                 continue;
             }
+            self.id_of_last_event_in_last_extracted_update =
+                update.events.last().map(|e| e.event_id);
             return Ok(update);
         }
     }
@@ -1168,4 +1205,51 @@ pub mod tests {
 
     // TODO: Test we dont re-feed pointless updates if fetching returns <= events we already
     //   processed
+
+    #[tokio::test]
+    async fn handles_fetching_page_with_complete_wft_and_page_token_to_empty_page() {
+        let timer_hist = canned_histories::single_timer("t");
+        let workflow_task = timer_hist.get_full_history_info().unwrap();
+        let prev_started_wft_id = workflow_task.previous_started_event_id();
+        let wft_started_id = workflow_task.workflow_task_started_event_id();
+
+        let mut full_resp_with_npt: GetWorkflowExecutionHistoryResponse =
+            timer_hist.get_full_history_info().unwrap().into();
+        full_resp_with_npt.next_page_token = vec![1];
+
+        let mut mock_client = mock_workflow_client();
+        mock_client
+            .expect_get_workflow_execution_history()
+            .returning(move |_, _, _| Ok(full_resp_with_npt.clone()))
+            .times(1);
+        mock_client
+            .expect_get_workflow_execution_history()
+            .returning(move |_, _, _| {
+                Ok(GetWorkflowExecutionHistoryResponse {
+                    history: Some(History { events: vec![] }),
+                    raw_history: vec![],
+                    next_page_token: vec![],
+                    archived: false,
+                })
+            })
+            .times(1);
+
+        let mut paginator = HistoryPaginator::new(
+            workflow_task.into(),
+            prev_started_wft_id,
+            wft_started_id,
+            "wfid".to_string(),
+            "runid".to_string(),
+            NextPageToken::FetchFromStart,
+            Arc::new(mock_client),
+        );
+        let mut update = paginator.extract_next_update().await.unwrap();
+        let seq = update.take_next_wft_sequence(0).unwrap_events();
+        assert_eq!(seq.last().unwrap().event_id, 3);
+        let seq = update.take_next_wft_sequence(3).unwrap_events();
+        assert_eq!(seq.last().unwrap().event_id, 8);
+        assert_matches!(update.take_next_wft_sequence(8), NextWFT::NeedFetch);
+        let mut update = paginator.extract_next_update().await.unwrap();
+        assert_matches!(update.take_next_wft_sequence(8), NextWFT::ReplayOver);
+    }
 }

package/sdk-core/core/src/worker/workflow/machines/signal_external_state_machine.rs

@@ -234,7 +234,10 @@ impl WFMachinesAdapter for SignalExternalMachine {
                     SignalExternalWorkflowExecutionFailedCause::Unspecified => "unknown",
                     SignalExternalWorkflowExecutionFailedCause::ExternalWorkflowExecutionNotFound
                     | SignalExternalWorkflowExecutionFailedCause::NamespaceNotFound =>
-                        "it was not found"
+                        "it was not found",
+                    SignalExternalWorkflowExecutionFailedCause::SignalCountLimitExceeded => {
+                        "The per-workflow signal limit was exceeded"
+                    }
                 };
                 vec![ResolveSignalExternalWorkflow {
                     seq: self.shared_state.seq,