@temporalio/core-bridge 1.4.4 → 1.5.1

package/sdk-core/core/src/telemetry/prometheus_server.rs

@@ -1,4 +1,4 @@
-use crate::telemetry::{default_resource, metrics::SDKAggSelector};
+use crate::telemetry::default_resource;
 use hyper::{
     header::CONTENT_TYPE,
     service::{make_service_fn, service_fn},
@@ -7,7 +7,7 @@ use hyper::{
 use opentelemetry::{
     metrics::MetricsError,
     sdk::{
-        export::metrics::aggregation::TemporalitySelector,
+        export::metrics::{aggregation::TemporalitySelector, AggregatorSelector},
         metrics::{controllers, processors},
     },
 };
@@ -24,10 +24,11 @@ pub(super) struct PromServer {
 impl PromServer {
     pub fn new(
         addr: SocketAddr,
+        aggregation: impl AggregatorSelector + Send + Sync + 'static,
         temporality: impl TemporalitySelector + Send + Sync + 'static,
     ) -> Result<Self, MetricsError> {
         let controller =
-            controllers::basic(processors::factory(SDKAggSelector, temporality).with_memory(true))
+            controllers::basic(processors::factory(aggregation, temporality).with_memory(true))
                 .with_resource(default_resource())
                 .build();
         let exporter = ExporterBuilder::new(controller).try_init()?;

package/sdk-core/core/src/test_help/mod.rs

@@ -5,8 +5,11 @@ use crate::{
     protosext::ValidPollWFTQResponse,
     replay::TestHistoryBuilder,
     sticky_q_name_for_worker,
+    telemetry::metrics::MetricsContext,
     worker::{
-        client::{mocks::mock_workflow_client, MockWorkerClient, WorkerClient},
+        client::{
+            mocks::mock_workflow_client, MockWorkerClient, WorkerClient, WorkflowTaskCompletion,
+        },
         new_wft_poller,
     },
     TaskToken, Worker, WorkerConfig, WorkerConfigBuilder,
@@ -26,7 +29,6 @@ use std::{
     task::{Context, Poll},
     time::Duration,
 };
-use temporal_client::WorkflowTaskCompletion;
 use temporal_sdk_core_api::Worker as WorkerTrait;
 use temporal_sdk_core_protos::{
     coresdk::{
@@ -57,6 +59,7 @@ pub fn test_worker_cfg() -> WorkerConfigBuilder {
     wcb.namespace("default")
         .task_queue(TEST_Q)
         .worker_build_id("test_bin_id")
+        .ignore_evicts_on_shutdown(true)
         // Serial polling since it makes mocking much easier.
         .max_concurrent_wft_polls(1_usize);
     wcb
@@ -144,7 +147,7 @@ pub(crate) fn mock_worker(mocks: MocksHolder) -> Worker {
         mocks.client,
         mocks.inputs.wft_stream,
         act_poller,
-        Default::default(),
+        MetricsContext::no_op(),
         CancellationToken::new(),
     )
 }
@@ -214,7 +217,7 @@ impl MockWorkerInputs {
     }
     pub fn new_from_poller(wf_poller: BoxedWFPoller) -> Self {
         Self {
-            wft_stream: new_wft_poller(wf_poller, Default::default()).boxed(),
+            wft_stream: new_wft_poller(wf_poller, MetricsContext::no_op()).boxed(),
             act_poller: None,
             config: test_worker_cfg().build().unwrap(),
         }
@@ -503,7 +506,7 @@ pub(crate) fn build_mock_pollers(mut cfg: MockPollCfg) -> MocksHolder {
             .into_iter()
             .map(|response| {
                 let cur_attempt = attempts_at_task_num.entry(response.hashable()).or_insert(1);
-                let mut r = hist_to_poll_resp(&hist.hist, hist.wf_id.clone(), response, TEST_Q);
+                let mut r = hist_to_poll_resp(&hist.hist, hist.wf_id.clone(), response);
                 r.attempt = *cur_attempt;
                 *cur_attempt += 1;
                 r
@@ -654,7 +657,6 @@ pub fn hist_to_poll_resp(
     t: &TestHistoryBuilder,
     wf_id: impl Into<String>,
     response_type: ResponseType,
-    task_queue: impl Into<String>,
 ) -> QueueResponse<PollWorkflowTaskQueueResponse> {
     let run_id = t.get_orig_run_id();
     let wf = WorkflowExecution {
@@ -683,7 +685,7 @@ pub fn hist_to_poll_resp(
             }
         }
     };
-    let mut resp = hist_info.as_poll_wft_response(task_queue);
+    let mut resp = hist_info.as_poll_wft_response();
     resp.workflow_execution = Some(wf);
     QueueResponse { resp, delay_until }
 }

package/sdk-core/core/src/worker/activities/local_activities.rs

@@ -179,7 +179,7 @@ impl LocalActivityManager {
         Self::new(
             max_concurrent,
             "fake_ns".to_string(),
-            MetricsContext::default(),
+            MetricsContext::no_op(),
         )
     }
 
@@ -515,6 +515,7 @@ enum CancelOrTimeout {
     },
 }
 
+#[allow(clippy::large_enum_variant)]
 enum NewOrCancel {
     New(NewOrRetry, OwnedMeteredSemPermit),
     Cancel(CancelOrTimeout),

package/sdk-core/core/src/worker/activities.rs

@@ -7,6 +7,7 @@ pub(crate) use local_activities::{
     LocalInFlightActInfo, NewLocalAct,
 };
 
+use crate::telemetry::metrics::eager;
 use crate::{
     abstractions::{MeteredSemaphore, OwnedMeteredSemPermit},
     pollers::BoxedActPoller,
@@ -41,6 +42,7 @@ use temporal_sdk_core_protos::{
     },
 };
 use tokio::sync::Notify;
+use tracing::Span;
 
 #[derive(Debug, derive_more::Constructor)]
 struct PendingActivityCancel {
@@ -48,11 +50,15 @@ struct PendingActivityCancel {
     reason: ActivityCancelReason,
 }
 
-/// Contains minimal set of details that core needs to store while an activity is running.
+/// Contains details that core wants to store while an activity is running.
 #[derive(Debug)]
 struct InFlightActInfo {
     pub activity_type: String,
     pub workflow_type: String,
+    /// Only kept for logging reasons
+    pub workflow_id: String,
+    /// Only kept for logging reasons
+    pub workflow_run_id: String,
     start_time: Instant,
 }
 
@@ -71,19 +77,17 @@ struct RemoteInFlightActInfo {
     _permit: OwnedMeteredSemPermit,
 }
 impl RemoteInFlightActInfo {
-    fn new(
-        activity_type: String,
-        workflow_type: String,
-        heartbeat_timeout: Option<prost_types::Duration>,
-        permit: OwnedMeteredSemPermit,
-    ) -> Self {
+    fn new(poll_resp: &PollActivityTaskQueueResponse, permit: OwnedMeteredSemPermit) -> Self {
+        let wec = poll_resp.workflow_execution.clone().unwrap_or_default();
         Self {
             base: InFlightActInfo {
-                activity_type,
-                workflow_type,
+                activity_type: poll_resp.activity_type.clone().unwrap_or_default().name,
+                workflow_type: poll_resp.workflow_type.clone().unwrap_or_default().name,
+                workflow_id: wec.workflow_id,
+                workflow_run_id: wec.run_id,
                 start_time: Instant::now(),
             },
-            heartbeat_timeout,
+            heartbeat_timeout: poll_resp.heartbeat_timeout.clone(),
             issued_cancel_to_lang: false,
             known_not_found: false,
             _permit: permit,
@@ -202,7 +206,7 @@ impl WorkerActivityTasks {
                 cancel_task
             }
             task = self.non_poll_tasks.next() => {
-                Ok(Some(self.about_to_issue_task(task)))
+                Ok(Some(self.about_to_issue_task(task, true)))
             }
             (work, permit) = poll_with_semaphore => {
                 match work {
@@ -214,7 +218,7 @@ impl WorkerActivityTasks {
                         }
                         let work = self.about_to_issue_task(PermittedTqResp {
                            resp: work, permit
-                        });
+                        }, false);
                         Ok(Some(work))
                     }
                     None => {
@@ -234,12 +238,14 @@ impl WorkerActivityTasks {
     ) {
         if let Some((_, act_info)) = self.outstanding_activity_tasks.remove(&task_token) {
             let act_metrics = self.metrics.with_new_attrs([
-                activity_type(act_info.base.activity_type.clone()),
-                workflow_type(act_info.base.workflow_type.clone()),
+                activity_type(act_info.base.activity_type),
+                workflow_type(act_info.base.workflow_type),
             ]);
+            Span::current().record("workflow_id", act_info.base.workflow_id);
+            Span::current().record("run_id", act_info.base.workflow_run_id);
             act_metrics.act_execution_latency(act_info.base.start_time.elapsed());
             let known_not_found = act_info.known_not_found;
-            drop(act_info); // TODO: Get rid of dashmap. If we hold ref across await, bad stuff.
+
             self.heartbeat_manager.evict(task_token.clone()).await;
             self.complete_notify.notify_waiters();
 
@@ -369,19 +375,28 @@ impl WorkerActivityTasks {
     }
 
     /// Called when there is a new act task about to be bubbled up out of the manager
-    fn about_to_issue_task(&self, task: PermittedTqResp) -> ActivityTask {
+    fn about_to_issue_task(&self, task: PermittedTqResp, is_eager: bool) -> ActivityTask {
+        if let Some(ref act_type) = task.resp.activity_type {
+            if let Some(ref wf_type) = task.resp.workflow_type {
+                self.metrics
+                    .with_new_attrs([
+                        activity_type(act_type.name.clone()),
+                        workflow_type(wf_type.name.clone()),
+                        eager(is_eager),
+                    ])
+                    .act_task_received();
+            }
+        }
+        // There could be an else statement here but since the response should always contain both
+        // activity_type and workflow_type, we won't bother.
+
         if let Some(dur) = task.resp.sched_to_start() {
             self.metrics.act_sched_to_start_latency(dur);
         };
 
         self.outstanding_activity_tasks.insert(
             task.resp.task_token.clone().into(),
-            RemoteInFlightActInfo::new(
-                task.resp.activity_type.clone().unwrap_or_default().name,
-                task.resp.workflow_type.clone().unwrap_or_default().name,
-                task.resp.heartbeat_timeout.clone(),
-                task.permit,
-            ),
+            RemoteInFlightActInfo::new(&task.resp, task.permit),
         );
 
         ActivityTask::start_from_poll_resp(task.resp)
@@ -411,6 +426,8 @@ impl ActivitiesFromWFTsHandle {
     /// from WFT completion)
     pub(crate) fn add_tasks(&self, tasks: impl IntoIterator<Item = PermittedTqResp>) {
         for t in tasks.into_iter() {
+            // Technically we should be reporting `activity_task_received` here, but for simplicity
+            // and time insensitivity, that metric is tracked in `about_to_issue_task`.
             self.tx.try_send(t).expect("Receive half cannot be dropped");
         }
     }
@@ -449,7 +466,7 @@ mod tests {
             Some(2.0),
             poller,
             Arc::new(mock_manual_workflow_client()),
-            MetricsContext::default(),
+            MetricsContext::no_op(),
             Duration::from_secs(1),
             Duration::from_secs(1),
         );

package/sdk-core/core/src/worker/client/mocks.rs

@@ -16,7 +16,7 @@ pub(crate) fn mock_manual_workflow_client() -> MockManualWorkerClient {
 // results. This is really annoying b/c of the async trait stuff. Need
 // https://github.com/asomers/mockall/issues/189 to be fixed for it to go away.
 mockall::mock! {
-    pub ManualWorkerClient {}
+    pub(crate) ManualWorkerClient {}
     #[allow(unused)]
     impl WorkerClient for ManualWorkerClient {
         fn poll_workflow_task<'a, 'b>(&'a self, task_queue: String, is_sticky: bool)

package/sdk-core/core/src/worker/client.rs

@@ -2,15 +2,16 @@
 
 pub(crate) mod mocks;
 
-use temporal_client::{Client, RetryClient, WorkflowService, WorkflowTaskCompletion};
+use temporal_client::{Client, RetryClient, WorkflowService};
 use temporal_sdk_core_protos::{
     coresdk::workflow_commands::QueryResult,
     temporal::api::{
+        command::v1::Command,
         common::v1::{Payloads, WorkflowExecution},
         enums::v1::{TaskQueueKind, WorkflowTaskFailedCause},
         failure::v1::Failure,
         query::v1::WorkflowQueryResult,
-        taskqueue::v1::{TaskQueue, TaskQueueMetadata},
+        taskqueue::v1::{StickyExecutionAttributes, TaskQueue, TaskQueueMetadata, VersionId},
         workflowservice::v1::*,
     },
     TaskToken,
@@ -133,7 +134,9 @@ impl WorkerClient for WorkerClientBag {
             } else {
                 self.worker_build_id.clone()
             },
-            worker_versioning_build_id: self.versioning_build_id(),
+            worker_versioning_id: Some(VersionId {
+                worker_build_id: self.versioning_build_id(),
+            }),
         };
 
         Ok(self
@@ -159,7 +162,9 @@ impl WorkerClient for WorkerClientBag {
             task_queue_metadata: max_tasks_per_sec.map(|tps| TaskQueueMetadata {
                 max_tasks_per_second: Some(tps),
             }),
-            worker_versioning_build_id: self.versioning_build_id(),
+            worker_versioning_id: Some(VersionId {
+                worker_build_id: self.versioning_build_id(),
+            }),
         };
 
         Ok(self
@@ -181,6 +186,9 @@ impl WorkerClient for WorkerClientBag {
             sticky_attributes: request.sticky_attributes,
             return_new_workflow_task: request.return_new_workflow_task,
             force_create_new_workflow_task: request.force_create_new_workflow_task,
+            worker_versioning_id: Some(VersionId {
+                worker_build_id: self.versioning_build_id(),
+            }),
             binary_checksum: self.worker_build_id.clone(),
             query_results: request
                 .query_responses
@@ -345,3 +353,21 @@ impl WorkerClient for WorkerClientBag {
             .into_inner())
     }
 }
+
+/// A version of [RespondWorkflowTaskCompletedRequest] that will finish being filled out by the
+/// server client
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct WorkflowTaskCompletion {
+    /// The task token that would've been received from polling for a workflow activation
+    pub task_token: TaskToken,
+    /// A list of new commands to send to the server, such as starting a timer.
+    pub commands: Vec<Command>,
+    /// If set, indicate that next task should be queued on sticky queue with given attributes.
+    pub sticky_attributes: Option<StickyExecutionAttributes>,
+    /// Responses to queries in the `queries` field of the workflow task.
+    pub query_responses: Vec<QueryResult>,
+    /// Indicate that the task completion should return a new WFT if one is available
+    pub return_new_workflow_task: bool,
+    /// Force a new WFT to be created after this completion
+    pub force_create_new_workflow_task: bool,
+}

package/sdk-core/core/src/worker/mod.rs

@@ -74,7 +74,7 @@ impl WorkerTrait for Worker {
         self.next_workflow_activation().await
     }
 
-    #[instrument(level = "debug", skip(self))]
+    #[instrument(skip(self))]
     async fn poll_activity_task(&self) -> Result<ActivityTask, PollActivityError> {
         loop {
             match self.activity_poll().await.transpose() {
@@ -94,7 +94,6 @@ impl WorkerTrait for Worker {
         self.complete_workflow_activation(completion).await
     }
 
-    #[instrument(level = "debug", skip(self, completion), fields(completion=%&completion))]
     async fn complete_activity_task(
         &self,
         completion: ActivityTaskCompletion,
@@ -135,7 +134,11 @@ impl WorkerTrait for Worker {
         if let Some(atm) = self.at_task_mgr.as_ref() {
             atm.notify_shutdown();
         }
-        info!("Initiated shutdown");
+        info!(
+            task_queue=%self.config.task_queue,
+            namespace=%self.config.namespace,
+            "Initiated shutdown",
+        );
     }
 
     async fn shutdown(&self) {
@@ -155,7 +158,9 @@ impl Worker {
         client: Arc<dyn WorkerClient>,
         metrics: MetricsContext,
     ) -> Self {
-        info!(task_queue = %config.task_queue, "Initializing worker");
+        info!(task_queue=%config.task_queue,
+              namespace=%config.namespace,
+              "Initializing worker");
         metrics.worker_registered();
 
         let shutdown_token = CancellationToken::new();
@@ -224,7 +229,7 @@ impl Worker {
 
     #[cfg(test)]
     pub(crate) fn new_test(config: WorkerConfig, client: impl WorkerClient + 'static) -> Self {
-        Self::new(config, None, Arc::new(client), Default::default())
+        Self::new(config, None, Arc::new(client), MetricsContext::no_op())
     }
 
     pub(crate) fn new_with_pollers(
@@ -264,6 +269,7 @@ impl Worker {
                     metrics,
                     namespace: config.namespace.clone(),
                     task_queue: config.task_queue.clone(),
+                    ignore_evicts_on_shutdown: config.ignore_evicts_on_shutdown,
                 },
                 sticky_queue_name.map(|sq| StickyExecutionAttributes {
                     worker_task_queue: Some(TaskQueue {
@@ -317,6 +323,10 @@ impl Worker {
         }
     }
 
+    pub(crate) fn shutdown_token(&self) -> CancellationToken {
+        self.shutdown_token.clone()
+    }
+
     /// Returns number of currently cached workflows
     pub async fn cached_workflows(&self) -> usize {
         self.workflows
@@ -394,6 +404,9 @@ impl Worker {
         }
     }
 
+    #[instrument(skip(self, task_token, status),
+                 fields(task_token=%&task_token, status=%&status,
+                        task_queue=%self.config.task_queue, workflow_id, run_id))]
     pub(crate) async fn complete_activity(
         &self,
         task_token: TaskToken,
@@ -433,13 +446,14 @@ impl Worker {
         Ok(())
     }
 
-    #[instrument(level = "debug", skip(self), fields(run_id))]
+    #[instrument(skip(self), fields(run_id, workflow_id, task_queue=%self.config.task_queue))]
     pub(crate) async fn next_workflow_activation(&self) -> Result<WorkflowActivation, PollWfError> {
         self.workflows.next_workflow_activation().await
     }
 
-    #[instrument(level = "debug", skip(self, completion),
-      fields(completion=%&completion, run_id=%completion.run_id))]
+    #[instrument(skip(self, completion),
+                 fields(completion=%&completion, run_id=%completion.run_id, workflow_id,
+                        task_queue=%self.config.task_queue))]
     pub(crate) async fn complete_workflow_activation(
         &self,
         completion: WorkflowActivationCompletion,
@@ -470,16 +484,6 @@ impl Worker {
         self.post_activate_hook = Some(Box::new(callback))
     }
 
-    /// Used for replay workers - causes the worker to shutdown when the given run reaches the
-    /// given event number
-    pub(crate) fn set_shutdown_on_run_reaches_event(&mut self, run_id: String, last_event: i64) {
-        self.set_post_activate_hook(move |worker, activated_run_id, last_processed_event| {
-            if activated_run_id == run_id && last_processed_event >= last_event as usize {
-                worker.initiate_shutdown();
-            }
-        });
-    }
-
     fn complete_local_act(
         &self,
         la_res: LocalActivityExecutionResult,

package/sdk-core/core/src/worker/workflow/driven_workflow.rs

@@ -1,9 +1,7 @@
-use crate::worker::workflow::{WFCommand, WorkflowStartedInfo};
+use crate::worker::workflow::{OutgoingJob, WFCommand, WorkflowStartedInfo};
+use prost_types::Timestamp;
 use temporal_sdk_core_protos::{
-    coresdk::workflow_activation::{
-        start_workflow_from_attribs, workflow_activation_job, CancelWorkflow, SignalWorkflow,
-        WorkflowActivationJob,
-    },
+    coresdk::workflow_activation::{start_workflow_from_attribs, WorkflowActivationJob},
     temporal::api::history::v1::WorkflowExecutionStartedEventAttributes,
     utilities::TryIntoOrNone,
 };
@@ -14,7 +12,7 @@ pub struct DrivenWorkflow {
     started_attrs: Option<WorkflowStartedInfo>,
     fetcher: Box<dyn WorkflowFetcher>,
     /// Outgoing activation jobs that need to be sent to the lang sdk
-    outgoing_wf_activation_jobs: Vec<workflow_activation_job::Variant>,
+    outgoing_wf_activation_jobs: Vec<OutgoingJob>,
 }
 
 impl<WF> From<Box<WF>> for DrivenWorkflow
@@ -36,6 +34,7 @@ impl DrivenWorkflow {
         &mut self,
         workflow_id: String,
         randomness_seed: u64,
+        start_time: Timestamp,
         attribs: WorkflowExecutionStartedEventAttributes,
     ) {
         debug!(run_id = %attribs.original_execution_run_id, "Driven WF start");
@@ -49,7 +48,9 @@ impl DrivenWorkflow {
             search_attrs: attribs.search_attributes.clone(),
             retry_policy: attribs.retry_policy.clone(),
         };
-        self.send_job(start_workflow_from_attribs(attribs, workflow_id, randomness_seed).into());
+        self.send_job(
+            start_workflow_from_attribs(attribs, workflow_id, randomness_seed, start_time).into(),
+        );
         self.started_attrs = Some(started_info);
     }
 
@@ -59,12 +60,12 @@ impl DrivenWorkflow {
     }
 
     /// Enqueue a new job to be sent to the driven workflow
-    pub fn send_job(&mut self, job: workflow_activation_job::Variant) {
+    pub(super) fn send_job(&mut self, job: OutgoingJob) {
         self.outgoing_wf_activation_jobs.push(job);
     }
 
     /// Observe pending jobs
-    pub fn peek_pending_jobs(&self) -> &[workflow_activation_job::Variant] {
+    pub(super) fn peek_pending_jobs(&self) -> &[OutgoingJob] {
         self.outgoing_wf_activation_jobs.as_slice()
     }
 
@@ -75,16 +76,6 @@ impl DrivenWorkflow {
             .map(Into::into)
             .collect()
     }
-
-    /// Signal the workflow
-    pub fn signal(&mut self, signal: SignalWorkflow) {
-        self.send_job(workflow_activation_job::Variant::SignalWorkflow(signal));
-    }
-
-    /// Cancel the workflow
-    pub fn cancel(&mut self, attribs: CancelWorkflow) {
-        self.send_job(workflow_activation_job::Variant::CancelWorkflow(attribs));
-    }
 }
 
 #[async_trait::async_trait]