@temporalio/core-bridge 1.7.4 → 1.8.1

package/sdk-core/core/src/worker/mod.rs

@@ -10,33 +10,31 @@ pub(crate) use activities::{
     ExecutingLAId, LocalActRequest, LocalActivityExecutionResult, LocalActivityResolution,
     NewLocalAct,
 };
+pub(crate) use workflow::{wft_poller::new_wft_poller, LEGACY_QUERY_ID};
+
 #[cfg(test)]
 pub(crate) use workflow::ManagedWFFunc;
-pub(crate) use workflow::{wft_poller::new_wft_poller, LEGACY_QUERY_ID};
 
 use crate::{
+    abstractions::MeteredSemaphore,
     errors::CompleteWfError,
-    pollers::{
-        new_activity_task_buffer, new_workflow_task_buffer, BoxedActPoller, Poller,
-        WorkflowTaskPoller,
-    },
-    protosext::{validate_activity_completion, ValidPollWFTQResponse},
+    pollers::{new_activity_task_buffer, new_workflow_task_buffer, WorkflowTaskPoller},
+    protosext::validate_activity_completion,
     telemetry::{
         metrics::{
-            activity_poller, local_activity_worker_type, workflow_poller, workflow_sticky_poller,
-            MetricsContext,
+            activity_poller, activity_worker_type, local_activity_worker_type, workflow_poller,
+            workflow_sticky_poller, workflow_worker_type, MetricsContext,
         },
         TelemetryInstance,
     },
     worker::{
-        activities::{DispatchOrTimeoutLA, LACompleteAction, LocalActivityManager},
+        activities::{LACompleteAction, LocalActivityManager, NextPendingLAAction},
         client::WorkerClient,
         workflow::{LAReqSink, LocalResolution, WorkflowBasics, Workflows},
     },
     ActivityHeartbeat, CompleteActivityError, PollActivityError, PollWfError, WorkerTrait,
 };
-use activities::{LocalInFlightActInfo, WorkerActivityTasks};
-use futures::Stream;
+use activities::WorkerActivityTasks;
 use std::{
     convert::TryInto,
     future,
@@ -56,13 +54,25 @@ use temporal_sdk_core_protos::{
     temporal::api::{
         enums::v1::TaskQueueKind,
         taskqueue::v1::{StickyExecutionAttributes, TaskQueue},
-        workflowservice::v1::{get_system_info_response, PollActivityTaskQueueResponse},
+        workflowservice::v1::get_system_info_response,
     },
     TaskToken,
 };
 use tokio::sync::mpsc::unbounded_channel;
 use tokio_util::sync::CancellationToken;
 
+use crate::{abstractions::dbg_panic, pollers::BoxedActPoller};
+#[cfg(test)]
+use {
+    crate::{
+        pollers::{BoxedPoller, MockPermittedPollBuffer},
+        protosext::ValidPollWFTQResponse,
+    },
+    futures::stream::BoxStream,
+    futures_util::StreamExt,
+    temporal_sdk_core_protos::temporal::api::workflowservice::v1::PollActivityTaskQueueResponse,
+};
+
 /// A worker polls on a certain task queue
 pub struct Worker {
     config: WorkerConfig,
@@ -183,9 +193,7 @@ impl Worker {
         client: Arc<dyn WorkerClient>,
         telem_instance: Option<&TelemetryInstance>,
     ) -> Self {
-        info!(task_queue=%config.task_queue,
-              namespace=%config.namespace,
-              "Initializing worker");
+        info!(task_queue=%config.task_queue, namespace=%config.namespace, "Initializing worker");
         let metrics = if let Some(ti) = telem_instance {
             MetricsContext::top_level(config.namespace.clone(), ti)
                 .with_task_q(config.task_queue.clone())
@@ -194,68 +202,13 @@ impl Worker {
         };
         metrics.worker_registered();
 
-        let shutdown_token = CancellationToken::new();
-        let max_nonsticky_polls = if sticky_queue_name.is_some() {
-            config.max_nonsticky_polls()
-        } else {
-            config.max_concurrent_wft_polls
-        };
-        let max_sticky_polls = config.max_sticky_polls();
-        let wft_metrics = metrics.with_new_attrs([workflow_poller()]);
-        let mut wf_task_poll_buffer = new_workflow_task_buffer(
-            client.clone(),
-            config.task_queue.clone(),
-            false,
-            max_nonsticky_polls,
-            max_nonsticky_polls * 2,
-            shutdown_token.child_token(),
-        );
-        wf_task_poll_buffer.set_num_pollers_handler(move |np| wft_metrics.record_num_pollers(np));
-        let sticky_queue_poller = sticky_queue_name.as_ref().map(|sqn| {
-            let sticky_metrics = metrics.with_new_attrs([workflow_sticky_poller()]);
-            let mut sp = new_workflow_task_buffer(
-                client.clone(),
-                sqn.clone(),
-                true,
-                max_sticky_polls,
-                max_sticky_polls * 2,
-                shutdown_token.child_token(),
-            );
-            sp.set_num_pollers_handler(move |np| sticky_metrics.record_num_pollers(np));
-            sp
-        });
-        let act_poll_buffer = if config.no_remote_activities {
-            None
-        } else {
-            let mut ap = new_activity_task_buffer(
-                client.clone(),
-                config.task_queue.clone(),
-                config.max_concurrent_at_polls,
-                config.max_concurrent_at_polls * 2,
-                config.max_task_queue_activities_per_second,
-                shutdown_token.child_token(),
-            );
-            let act_metrics = metrics.with_new_attrs([activity_poller()]);
-            ap.set_num_pollers_handler(move |np| act_metrics.record_num_pollers(np));
-            Some(Box::from(ap)
-                as Box<
-                    dyn Poller<PollActivityTaskQueueResponse> + Send + Sync,
-                >)
-        };
-        let wf_task_poll_buffer = Box::new(WorkflowTaskPoller::new(
-            wf_task_poll_buffer,
-            sticky_queue_poller,
-        ));
-        let wft_stream = new_wft_poller(wf_task_poll_buffer, metrics.clone());
         Self::new_with_pollers(
             config,
             sticky_queue_name,
             client,
-            wft_stream,
-            act_poll_buffer,
+            TaskPollers::Real,
             metrics,
             telem_instance,
-            shutdown_token,
         )
     }
 
@@ -269,12 +222,110 @@ impl Worker {
         mut config: WorkerConfig,
         sticky_queue_name: Option<String>,
         client: Arc<dyn WorkerClient>,
-        wft_stream: impl Stream<Item = Result<ValidPollWFTQResponse, tonic::Status>> + Send + 'static,
-        act_poller: Option<BoxedActPoller>,
+        task_pollers: TaskPollers,
         metrics: MetricsContext,
         telem_instance: Option<&TelemetryInstance>,
-        shutdown_token: CancellationToken,
     ) -> Self {
+        let shutdown_token = CancellationToken::new();
+        let wft_semaphore = Arc::new(MeteredSemaphore::new(
+            config.max_outstanding_workflow_tasks,
+            metrics.with_new_attrs([workflow_worker_type()]),
+            MetricsContext::available_task_slots,
+        ));
+        let act_semaphore = Arc::new(MeteredSemaphore::new(
+            config.max_outstanding_activities,
+            metrics.with_new_attrs([activity_worker_type()]),
+            MetricsContext::available_task_slots,
+        ));
+
+        let (wft_stream, act_poller) = match task_pollers {
+            TaskPollers::Real => {
+                let max_nonsticky_polls = if sticky_queue_name.is_some() {
+                    config.max_nonsticky_polls()
+                } else {
+                    config.max_concurrent_wft_polls
+                };
+                let max_sticky_polls = config.max_sticky_polls();
+                let wft_metrics = metrics.with_new_attrs([workflow_poller()]);
+                let wf_task_poll_buffer = new_workflow_task_buffer(
+                    client.clone(),
+                    TaskQueue {
+                        name: config.task_queue.clone(),
+                        kind: TaskQueueKind::Normal as i32,
+                        normal_name: "".to_string(),
+                    },
+                    max_nonsticky_polls,
+                    wft_semaphore.clone(),
+                    shutdown_token.child_token(),
+                    Some(move |np| {
+                        wft_metrics.record_num_pollers(np);
+                    }),
+                );
+                let sticky_queue_poller = sticky_queue_name.as_ref().map(|sqn| {
+                    let sticky_metrics = metrics.with_new_attrs([workflow_sticky_poller()]);
+                    new_workflow_task_buffer(
+                        client.clone(),
+                        TaskQueue {
+                            name: sqn.clone(),
+                            kind: TaskQueueKind::Sticky as i32,
+                            normal_name: config.task_queue.clone(),
+                        },
+                        max_sticky_polls,
+                        wft_semaphore.clone(),
+                        shutdown_token.child_token(),
+                        Some(move |np| {
+                            sticky_metrics.record_num_pollers(np);
+                        }),
+                    )
+                });
+                let act_poll_buffer = if config.no_remote_activities {
+                    None
+                } else {
+                    let act_metrics = metrics.with_new_attrs([activity_poller()]);
+                    let ap = new_activity_task_buffer(
+                        client.clone(),
+                        config.task_queue.clone(),
+                        config.max_concurrent_at_polls,
+                        act_semaphore.clone(),
+                        config.max_task_queue_activities_per_second,
+                        shutdown_token.child_token(),
+                        Some(move |np| act_metrics.record_num_pollers(np)),
+                        config.max_worker_activities_per_second,
+                    );
+                    Some(Box::from(ap) as BoxedActPoller)
+                };
+                let wf_task_poll_buffer = Box::new(WorkflowTaskPoller::new(
+                    wf_task_poll_buffer,
+                    sticky_queue_poller,
+                ));
+                let wft_stream = new_wft_poller(wf_task_poll_buffer, metrics.clone());
+                #[cfg(test)]
+                let wft_stream = wft_stream.left_stream();
+                (wft_stream, act_poll_buffer)
+            }
+            #[cfg(test)]
+            TaskPollers::Mocked {
+                wft_stream,
+                act_poller,
+            } => {
+                let ap =
+                    act_poller.map(|ap| MockPermittedPollBuffer::new(act_semaphore.clone(), ap));
+                let wft_semaphore = wft_semaphore.clone();
+                let wfs = wft_stream.then(move |s| {
+                    let wft_semaphore = wft_semaphore.clone();
+                    async move {
+                        let permit = wft_semaphore
+                            .acquire_owned()
+                            .await
+                            .expect("Mock WFT stream should not see closed semaphore");
+                        s.map(|s| (s, permit))
+                    }
+                });
+                let wfs = wfs.right_stream();
+                (wfs, ap.map(|ap| Box::new(ap) as BoxedActPoller))
+            }
+        };
+
         let (hb_tx, hb_rx) = unbounded_channel();
         let local_act_mgr = Arc::new(LocalActivityManager::new(
             config.max_outstanding_local_activities,
@@ -284,8 +335,7 @@ impl Worker {
         ));
         let at_task_mgr = act_poller.map(|ap| {
             WorkerActivityTasks::new(
-                config.max_outstanding_activities,
-                config.max_worker_activities_per_second,
+                act_semaphore,
                 ap,
                 client.clone(),
                 metrics.clone(),
@@ -312,6 +362,7 @@ impl Worker {
                     worker_task_queue: Some(TaskQueue {
                         name: sq,
                         kind: TaskQueueKind::Sticky as i32,
+                        normal_name: config.task_queue.clone(),
                     }),
                     schedule_to_start_timeout: Some(
                         config
@@ -321,6 +372,7 @@ impl Worker {
                     ),
                 }),
                 client,
+                wft_semaphore,
                 wft_stream,
                 la_sink,
                 local_act_mgr.clone(),
@@ -335,7 +387,7 @@ impl Worker {
             config,
             shutdown_token,
             post_activate_hook: None,
-            // Complete if there configured not to poll on non-local activities.
+            // Non-local activities are already complete if configured not to poll for them.
             non_local_activities_complete: Arc::new(AtomicBool::new(!poll_on_non_local_activities)),
             local_activities_complete: Default::default(),
         }
@@ -393,9 +445,13 @@ impl Worker {
     }
 
     #[allow(unused)]
-    pub(crate) async fn available_wft_permits(&self) -> usize {
+    pub(crate) fn available_wft_permits(&self) -> usize {
         self.workflows.available_wft_permits()
     }
+    #[cfg(test)]
+    pub(crate) fn unused_wft_permits(&self) -> usize {
+        self.workflows.unused_wft_permits()
+    }
 
     /// Get new activity tasks (may be local or nonlocal). Local activities are returned first
     /// before polling the server if there are any.
@@ -437,14 +493,9 @@ impl Worker {
                 unreachable!()
             }
             match self.local_act_mgr.next_pending().await {
-                Some(DispatchOrTimeoutLA::Dispatch(r)) => Ok(Some(r)),
-                Some(DispatchOrTimeoutLA::Timeout {
-                    run_id,
-                    resolution,
-                    task,
-                }) => {
-                    self.notify_local_result(&run_id, LocalResolution::LocalActivity(resolution));
-                    Ok(task)
+                Some(NextPendingLAAction::Dispatch(r)) => Ok(Some(r)),
+                Some(NextPendingLAAction::Autocomplete(action)) => {
+                    Ok(self.handle_la_complete_action(action))
                 }
                 None => {
                     if self.shutdown_token.is_cancelled() {
@@ -485,23 +536,7 @@ impl Worker {
         validate_activity_completion(&status)?;
         if task_token.is_local_activity_task() {
             let as_la_res: LocalActivityExecutionResult = status.try_into()?;
-            match self.local_act_mgr.complete(&task_token, &as_la_res) {
-                LACompleteAction::Report(info) => self.complete_local_act(as_la_res, info, None),
-                LACompleteAction::LangDoesTimerBackoff(backoff, info) => {
-                    // This la needs to write a failure marker, and then we will tell lang how
-                    // long of a timer to schedule to back off for. We do this because there are
-                    // no other situations where core generates "internal" commands so it is much
-                    // simpler for lang to reply with the timer / next LA command than to do it
-                    // internally. Plus, this backoff hack we'd like to eliminate eventually.
-                    self.complete_local_act(as_la_res, info, Some(backoff));
-                }
-                LACompleteAction::WillBeRetried => {
-                    // Nothing to do here
-                }
-                LACompleteAction::Untracked => {
-                    warn!("Tried to complete untracked local activity {}", task_token);
-                }
-            }
+            self.complete_local_act(task_token, as_la_res);
             return Ok(());
         }
 
@@ -567,23 +602,28 @@ impl Worker {
         self.post_activate_hook = Some(Box::new(callback))
     }
 
-    fn complete_local_act(
-        &self,
-        la_res: LocalActivityExecutionResult,
-        info: LocalInFlightActInfo,
-        backoff: Option<prost_types::Duration>,
-    ) {
-        self.notify_local_result(
-            &info.la_info.workflow_exec_info.run_id,
-            LocalResolution::LocalActivity(LocalActivityResolution {
-                seq: info.la_info.schedule_cmd.seq,
-                result: la_res,
-                runtime: info.dispatch_time.elapsed(),
-                attempt: info.attempt,
-                backoff,
-                original_schedule_time: info.la_info.schedule_cmd.original_schedule_time,
-            }),
-        )
+    fn complete_local_act(&self, task_token: TaskToken, la_res: LocalActivityExecutionResult) {
+        if self
+            .handle_la_complete_action(self.local_act_mgr.complete(&task_token, la_res))
+            .is_some()
+        {
+            dbg_panic!("Should never be a task from direct completion");
+        }
+    }
+
+    fn handle_la_complete_action(&self, action: LACompleteAction) -> Option<ActivityTask> {
+        match action {
+            LACompleteAction::Report {
+                run_id,
+                resolution,
+                task,
+            } => {
+                self.notify_local_result(&run_id, LocalResolution::LocalActivity(resolution));
+                task
+            }
+            LACompleteAction::WillBeRetried(task) => task,
+            LACompleteAction::Untracked => None,
+        }
     }
 
     fn notify_local_result(&self, run_id: &str, res: LocalResolution) {
@@ -605,7 +645,6 @@ fn build_wf_basics(
 ) -> WorkflowBasics {
     WorkflowBasics {
         max_cached_workflows: config.max_cached_workflows,
-        max_outstanding_wfts: config.max_outstanding_workflow_tasks,
         shutdown_token,
         metrics,
         namespace: config.namespace.clone(),
@@ -618,6 +657,15 @@ fn build_wf_basics(
     }
 }
 
+pub(crate) enum TaskPollers {
+    Real,
+    #[cfg(test)]
+    Mocked {
+        wft_stream: BoxStream<'static, Result<ValidPollWFTQResponse, tonic::Status>>,
+        act_poller: Option<BoxedPoller<PollActivityTaskQueueResponse>>,
+    },
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -648,7 +696,7 @@ mod tests {
                 .as_ref()
                 .unwrap()
                 .remaining_activity_capacity(),
-            4
+            5
         );
     }
 

package/sdk-core/core/src/worker/workflow/driven_workflow.rs

@@ -40,10 +40,6 @@ impl DrivenWorkflow {
         debug!(run_id = %attribs.original_execution_run_id, "Driven WF start");
         let started_info = WorkflowStartedInfo {
             workflow_task_timeout: attribs.workflow_task_timeout.clone().try_into_or_none(),
-            workflow_execution_timeout: attribs
-                .workflow_execution_timeout
-                .clone()
-                .try_into_or_none(),
             memo: attribs.memo.clone(),
             search_attrs: attribs.search_attributes.clone(),
             retry_policy: attribs.retry_policy.clone(),

package/sdk-core/core/src/worker/workflow/machines/activity_state_machine.rs

@@ -17,7 +17,10 @@ use temporal_sdk_core_protos::{
         workflow_commands::{ActivityCancellationType, ScheduleActivity},
     },
     temporal::api::{
-        command::v1::{command, Command, RequestCancelActivityTaskCommandAttributes},
+        command::v1::{
+            command, schedule_activity_cmd_to_api, Command,
+            RequestCancelActivityTaskCommandAttributes,
+        },
         common::v1::{ActivityType, Payload, Payloads},
         enums::v1::{CommandType, EventType, RetryState},
         failure::v1::{failure::FailureInfo, ActivityFailureInfo, CanceledFailureInfo, Failure},
@@ -107,6 +110,7 @@ impl ActivityMachine {
     pub(super) fn new_scheduled(
         attrs: ScheduleActivity,
         internal_flags: InternalFlagsRef,
+        use_compatible_version: bool,
     ) -> NewMachineWithCommand {
         let mut s = Self::from_parts(
             Created {}.into(),
@@ -124,7 +128,10 @@ impl ActivityMachine {
             .expect("Scheduling activities doesn't fail");
         let command = Command {
             command_type: CommandType::ScheduleActivityTask as i32,
-            attributes: Some(s.shared_state().attrs.clone().into()),
+            attributes: Some(schedule_activity_cmd_to_api(
+                s.shared_state().attrs.clone(),
+                use_compatible_version,
+            )),
         };
         NewMachineWithCommand {
             command,

package/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs

@@ -22,7 +22,10 @@ use temporal_sdk_core_protos::{
         workflow_commands::StartChildWorkflowExecution,
     },
     temporal::api::{
-        command::v1::{Command, RequestCancelExternalWorkflowExecutionCommandAttributes},
+        command::v1::{
+            start_child_workflow_cmd_to_api, Command,
+            RequestCancelExternalWorkflowExecutionCommandAttributes,
+        },
         common::v1::{Payload, Payloads, WorkflowExecution, WorkflowType},
         enums::v1::{
             CommandType, EventType, RetryState, StartChildWorkflowExecutionFailedCause, TimeoutType,
@@ -359,6 +362,7 @@ impl ChildWorkflowMachine {
     pub(super) fn new_scheduled(
         attribs: StartChildWorkflowExecution,
         internal_flags: InternalFlagsRef,
+        use_compatible_version: bool,
     ) -> NewMachineWithCommand {
         let mut s = Self::from_parts(
             Created {}.into(),
@@ -379,7 +383,10 @@ impl ChildWorkflowMachine {
             .expect("Scheduling child workflows doesn't fail");
         let cmd = Command {
             command_type: CommandType::StartChildWorkflowExecution as i32,
-            attributes: Some(attribs.into()),
+            attributes: Some(start_child_workflow_cmd_to_api(
+                attribs,
+                use_compatible_version,
+            )),
         };
         NewMachineWithCommand {
             command: cmd,

package/sdk-core/core/src/worker/workflow/machines/continue_as_new_workflow_state_machine.rs

@@ -8,7 +8,7 @@ use std::convert::TryFrom;
 use temporal_sdk_core_protos::{
     coresdk::workflow_commands::ContinueAsNewWorkflowExecution,
     temporal::api::{
-        command::v1::Command,
+        command::v1::{continue_as_new_cmd_to_api, Command},
         enums::v1::{CommandType, EventType},
     },
 };
@@ -30,13 +30,16 @@ fsm! {
 #[derive(Debug, derive_more::Display)]
 pub(super) enum ContinueAsNewWorkflowCommand {}
 
-pub(super) fn continue_as_new(attribs: ContinueAsNewWorkflowExecution) -> NewMachineWithCommand {
+pub(super) fn continue_as_new(
+    attribs: ContinueAsNewWorkflowExecution,
+    use_compatible_version: bool,
+) -> NewMachineWithCommand {
     let mut machine = ContinueAsNewWorkflowMachine::from_parts(Created {}.into(), ());
     OnEventWrapper::on_event_mut(&mut machine, ContinueAsNewWorkflowMachineEvents::Schedule)
         .expect("Scheduling continue as new machine doesn't fail");
     let command = Command {
         command_type: CommandType::ContinueAsNewWorkflowExecution as i32,
-        attributes: Some(attribs.into()),
+        attributes: Some(continue_as_new_cmd_to_api(attribs, use_compatible_version)),
     };
     NewMachineWithCommand {
         command,