@temporalio/core-bridge 0.19.2 → 0.20.2

package/sdk-core/core/src/workflow/mod.rs

@@ -15,7 +15,7 @@ use crate::{
     worker::{LocalActRequest, LocalActivityResolution},
 };
 use machines::WorkflowMachines;
-use std::{result, sync::mpsc::Sender};
+use std::{result, sync::mpsc::Sender, time::Duration};
 use temporal_sdk_core_protos::{
     coresdk::{workflow_activation::WorkflowActivation, workflow_commands::*},
     temporal::api::command::v1::Command as ProtoCommand,
@@ -213,6 +213,7 @@ pub enum WFCommand {
     RequestCancelExternalWorkflow(RequestCancelExternalWorkflowExecution),
     SignalExternalWorkflow(SignalExternalWorkflowExecution),
     CancelSignalWorkflow(CancelSignalWorkflow),
+    UpsertSearchAttributes(UpsertWorkflowSearchAttributes),
 }
 
 impl TryFrom<WorkflowCommand> for WFCommand {
@@ -253,6 +254,9 @@ impl TryFrom<WorkflowCommand> for WFCommand {
             workflow_command::Variant::RequestCancelLocalActivity(s) => {
                 Ok(Self::RequestCancelLocalActivity(s))
             }
+            workflow_command::Variant::UpsertWorkflowSearchAttributesCommandAttributes(s) => {
+                Ok(Self::UpsertSearchAttributes(s))
+            }
         }
     }
 }
@@ -267,6 +271,14 @@ enum CommandID {
     CancelExternal(u32),
 }
 
+/// Details remembered from the workflow execution started event that we may need to recall later.
+/// Is a subset of `WorkflowExecutionStartedEventAttributes`, but avoids holding on to huge fields.
+#[derive(Debug, Clone)]
+pub struct WorkflowStartedInfo {
+    workflow_task_timeout: Option<Duration>,
+    workflow_execution_timeout: Option<Duration>,
+}
+
 #[cfg(test)]
 pub mod managed_wf {
     use super::*;

package/sdk-core/core/src/workflow/workflow_tasks/concurrency_manager.rs

@@ -12,6 +12,7 @@ use std::{
     collections::HashMap,
     fmt::Debug,
     ops::{Deref, DerefMut},
+    sync::Arc,
 };
 use temporal_sdk_core_protos::coresdk::workflow_activation::WorkflowActivation;
 
@@ -22,7 +23,7 @@ pub(crate) struct WorkflowConcurrencyManager {
 }
 
 struct ManagedRun {
-    wfm: Mutex<WorkflowManager>,
+    wfm: Arc<Mutex<WorkflowManager>>,
     wft: Option<OutstandingTask>,
     activation: Option<OutstandingActivation>,
     metrics: MetricsContext,
@@ -36,7 +37,7 @@ struct ManagedRun {
 impl ManagedRun {
     fn new(wfm: WorkflowManager, metrics: MetricsContext) -> Self {
         Self {
-            wfm: Mutex::new(wfm),
+            wfm: Arc::new(Mutex::new(wfm)),
             wft: None,
             activation: None,
             metrics,
@@ -266,16 +267,19 @@ impl WorkflowConcurrencyManager {
         F: for<'a> FnOnce(&'a mut WorkflowManager) -> BoxFuture<Result<Fout>>,
         Fout: Send + Debug,
     {
-        let readlock = self.runs.read();
-        let m = readlock
-            .get(run_id)
-            .ok_or_else(|| WFMachinesError::Fatal("Missing workflow machines".to_string()))?;
-        // This holds a non-async mutex across an await point which is technically a no-no, but
-        // we never access the machines for the same run simultaneously anyway. This should all
-        // get fixed with a generally different approach which moves the runs inside workers.
-        let mut wfm_mutex = m.wfm.lock();
-        let res = mutator(&mut wfm_mutex).await;
+        // TODO: Slightly less than ideal. We must avoid holding the read lock on the overall
+        //  machine map while async-ly mutating the inner machine. So, we clone the inner ArcMutex.
+        //  We should restructure things to avoid the top-level lock on the map.
+
+        let wfm = {
+            let readlock = self.runs.read();
+            let m = readlock
+                .get(run_id)
+                .ok_or_else(|| WFMachinesError::Fatal("Missing workflow machines".to_string()))?;
+            m.wfm.clone()
+        };
 
+        let res = mutator(&mut wfm.lock()).await;
         res
     }
 
@@ -321,6 +325,8 @@ impl WorkflowConcurrencyManager {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::test_help::canned_histories;
+    use tokio::sync::Barrier;
 
     // We test mostly error paths here since the happy paths are well covered by the tests of the
     // core sdk itself, and setting up the fake data is onerous here. If we make the concurrency
@@ -342,4 +348,57 @@ mod tests {
         // Should whine that the machines have nothing to do (history empty)
         assert_matches!(res.unwrap_err(), WFMachinesError::Fatal { .. });
     }
+
+    /// This test makes sure that if we're stuck on an await within the machine mutator we don't
+    /// cause a deadlock if a write happens during that. This test will hang without proper
+    /// implementation.
+    #[tokio::test]
+    async fn aba_deadlock_prevented() {
+        let run_id = "some_run_id";
+        let timer_hist = canned_histories::single_timer("t");
+        let access_barr: &'static Barrier = Box::leak(Box::new(Barrier::new(2)));
+        let wft = timer_hist.get_history_info(1).unwrap();
+
+        let mgr = WorkflowConcurrencyManager::new();
+        mgr.create_or_update(
+            run_id,
+            wft.clone().into(),
+            "fake_wf_id",
+            "fake_namespace",
+            "fake_wf_type",
+            &Default::default(),
+        )
+        .await
+        .unwrap();
+        // Perform access which blocks
+        let access_fut = mgr.access(run_id, |_wfm| {
+            async {
+                // Wait to make sure access has started
+                access_barr.wait().await;
+                // Wait to make sure write has finished
+                access_barr.wait().await;
+                Ok(())
+            }
+            .boxed()
+        });
+        let write_fut = async {
+            // Wait to make sure access has started
+            access_barr.wait().await;
+            // Now try writing
+            mgr.create_or_update(
+                "different_run_id",
+                wft.clone().into(),
+                "fake_wf_id",
+                "fake_namespace",
+                "fake_wf_type",
+                &Default::default(),
+            )
+            .await
+            .unwrap();
+            // Indicate write has finished
+            access_barr.wait().await;
+        };
+        let (r1, _) = tokio::join!(access_fut, write_fut);
+        r1.unwrap();
+    }
 }

package/sdk-core/core/src/workflow/workflow_tasks/mod.rs

@@ -7,8 +7,9 @@ use crate::{
     pending_activations::PendingActivations,
     protosext::{ValidPollWFTQResponse, WorkflowActivationExt},
     telemetry::metrics::MetricsContext,
-    worker::{LocalActRequest, LocalActivityResolution},
+    worker::{client::WorkerClientBag, LocalActRequest, LocalActivityResolution},
     workflow::{
+        history_update::NextPageToken,
         machines::WFMachinesError,
         workflow_tasks::{
             cache_manager::WorkflowCacheManager, concurrency_manager::WorkflowConcurrencyManager,
@@ -26,15 +27,13 @@ use std::{
     sync::Arc,
     time::{Duration, Instant},
 };
-use temporal_client::ServerGatewayApis;
 use temporal_sdk_core_protos::{
     coresdk::{
         workflow_activation::{
-            create_query_activation, remove_from_cache::EvictionReason, workflow_activation_job,
-            QueryWorkflow, WorkflowActivation,
+            create_query_activation, query_to_job, remove_from_cache::EvictionReason,
+            workflow_activation_job, QueryWorkflow, WorkflowActivation,
         },
         workflow_commands::QueryResult,
-        FromPayloadsExt,
     },
     temporal::api::command::v1::Command as ProtoCommand,
     TaskToken,
@@ -120,8 +119,6 @@ pub(crate) enum NewWfTaskOutcome {
     /// The workflow task should be auto-completed with an empty command list, as it must be replied
     /// to but there is no meaningful work for lang to do.
     Autocomplete,
-    /// Workflow task had partial history and workflow was not present in the cache.
-    CacheMiss,
     /// The workflow task ran into problems while being applied and we must now evict the workflow
     Evict(WorkflowUpdateError),
     /// No action should be taken. Possibly we are waiting for local activities to complete
@@ -153,6 +150,13 @@ pub(crate) enum ActivationAction {
     RespondLegacyQuery { result: QueryResult },
 }
 
+#[derive(Debug, Eq, PartialEq, Hash)]
+pub(crate) enum EvictionRequestResult {
+    EvictionIssued(Option<u32>),
+    NotFound,
+    EvictionAlreadyOutstanding,
+}
+
 macro_rules! machine_mut {
     ($myself:ident, $run_id:ident, $clos:expr) => {{
         $myself
@@ -250,7 +254,7 @@ impl WorkflowTaskManager {
         run_id: &str,
         message: impl Into<String>,
         reason: EvictionReason,
-    ) -> Option<u32> {
+    ) -> EvictionRequestResult {
         if self.workflow_machines.exists(run_id) {
             if !self.activation_has_eviction(run_id) {
                 let message = message.into();
@@ -259,13 +263,17 @@ impl WorkflowTaskManager {
                 self.pending_activations
                     .notify_needs_eviction(run_id, message, reason);
                 self.pending_activations_notifier.notify_waiters();
+                EvictionRequestResult::EvictionIssued(
+                    self.workflow_machines
+                        .get_task(run_id)
+                        .map(|wt| wt.info.attempt),
+                )
+            } else {
+                EvictionRequestResult::EvictionAlreadyOutstanding
             }
-            self.workflow_machines
-                .get_task(run_id)
-                .map(|wt| wt.info.attempt)
         } else {
             warn!(%run_id, "Eviction requested for unknown run");
-            None
+            EvictionRequestResult::NotFound
         }
     }
 
@@ -298,7 +306,7 @@ impl WorkflowTaskManager {
     pub(crate) async fn apply_new_poll_resp(
         &self,
         work: ValidPollWFTQResponse,
-        gateway: Arc<dyn ServerGatewayApis + Send + Sync>,
+        client: Arc<WorkerClientBag>,
     ) -> NewWfTaskOutcome {
         let mut work = if let Some(w) = self.workflow_machines.buffer_resp_if_outstanding_work(work)
         {
@@ -307,29 +315,28 @@ impl WorkflowTaskManager {
             return NewWfTaskOutcome::TaskBuffered;
         };
 
+        let start_event_id = work.history.events.first().map(|e| e.event_id);
         debug!(
             task_token = %&work.task_token,
             history_length = %work.history.events.len(),
+            start_event_id = ?start_event_id,
             attempt = %work.attempt,
+            run_id = %work.workflow_execution.run_id,
             "Applying new workflow task from server"
         );
         let task_start_time = Instant::now();
 
         // Check if there is a legacy query we either need to immediately issue an activation for
         // (if there is no more replay work to do) or we need to store for later answering.
-        let legacy_query = work.legacy_query.take().map(|q| QueryWorkflow {
-            query_id: LEGACY_QUERY_ID.to_string(),
-            query_type: q.query_type,
-            arguments: Vec::from_payloads(q.query_args),
-        });
+        let legacy_query = work
+            .legacy_query
+            .take()
+            .map(|q| query_to_job(LEGACY_QUERY_ID.to_string(), q));
 
         let (info, mut next_activation) =
-            match self.instantiate_or_update_workflow(work, gateway).await {
+            match self.instantiate_or_update_workflow(work, client).await {
                 Ok((info, next_activation)) => (info, next_activation),
                 Err(e) => {
-                    if let WFMachinesError::CacheMiss = e.source {
-                        return NewWfTaskOutcome::CacheMiss;
-                    }
                     return NewWfTaskOutcome::Evict(e);
                 }
             };
@@ -337,7 +344,7 @@ impl WorkflowTaskManager {
         // Immediately dispatch query activation if no other jobs
         let legacy_query = if next_activation.jobs.is_empty() {
             if let Some(lq) = legacy_query {
-                debug!("Dispatching legacy query {:?}", &lq);
+                debug!("Dispatching legacy query {}", &lq);
                 next_activation
                     .jobs
                     .push(workflow_activation_job::Variant::QueryWorkflow(lq).into());
@@ -467,9 +474,8 @@ impl WorkflowTaskManager {
 
                         let wft_timeout: Duration = wfm
                             .machines
-                            .started_attrs()
-                            .and_then(|attrs| attrs.workflow_task_timeout.clone())
-                            .and_then(|tt| tt.try_into().ok())
+                            .get_started_info()
+                            .and_then(|attrs| attrs.workflow_task_timeout)
                             .ok_or_else(|| {
                                 WFMachinesError::Fatal(
                                     "Workflow's start attribs were missing a well formed task timeout"
@@ -566,9 +572,10 @@ impl WorkflowTaskManager {
             FailedActivationOutcome::ReportLegacyQueryFailure(tt)
         } else {
             // Blow up any cached data associated with the workflow
-            let should_report = self
-                .request_eviction(run_id, failstr, reason)
-                .map_or(true, |attempt| attempt <= 1);
+            let should_report = match self.request_eviction(run_id, failstr, reason) {
+                EvictionRequestResult::EvictionIssued(Some(attempt)) => attempt <= 1,
+                _ => false,
+            };
             if should_report {
                 FailedActivationOutcome::Report(tt)
             } else {
@@ -584,7 +591,7 @@ impl WorkflowTaskManager {
     async fn instantiate_or_update_workflow(
         &self,
         poll_wf_resp: ValidPollWFTQResponse,
-        gateway: Arc<dyn ServerGatewayApis + Send + Sync>,
+        client: Arc<WorkerClientBag>,
     ) -> Result<(WorkflowTaskInfo, WorkflowActivation), WorkflowUpdateError> {
         let run_id = poll_wf_resp.workflow_execution.run_id.clone();
 
@@ -593,22 +600,39 @@ impl WorkflowTaskManager {
             task_token: poll_wf_resp.task_token,
         };
 
+        let poll_resp_is_incremental = poll_wf_resp
+            .history
+            .events
+            .get(0)
+            .map(|ev| ev.event_id > 1)
+            .unwrap_or_default();
+
+        let page_token = if !self.workflow_machines.exists(&run_id) && poll_resp_is_incremental {
+            debug!(run_id=?run_id, "Workflow task has partial history, but workflow is not in \
+                   cache. Will fetch history");
+            self.metrics.sticky_cache_miss();
+            NextPageToken::FetchFromStart
+        } else {
+            poll_wf_resp.next_page_token.into()
+        };
+        let history_update = HistoryUpdate::new(
+            HistoryPaginator::new(
+                poll_wf_resp.history,
+                poll_wf_resp.workflow_execution.workflow_id.clone(),
+                poll_wf_resp.workflow_execution.run_id,
+                page_token,
+                client.clone(),
+            ),
+            poll_wf_resp.previous_started_event_id,
+        );
+
         match self
             .workflow_machines
             .create_or_update(
                 &run_id,
-                HistoryUpdate::new(
-                    HistoryPaginator::new(
-                        poll_wf_resp.history,
-                        poll_wf_resp.workflow_execution.workflow_id.clone(),
-                        poll_wf_resp.workflow_execution.run_id.clone(),
-                        poll_wf_resp.next_page_token,
-                        gateway.clone(),
-                    ),
-                    poll_wf_resp.previous_started_event_id,
-                ),
+                history_update,
                 &poll_wf_resp.workflow_execution.workflow_id,
-                &gateway.get_options().namespace,
+                client.namespace(),
                 &poll_wf_resp.workflow_type,
                 &self.metrics,
             )

package/sdk-core/core-api/Cargo.toml

@@ -2,14 +2,22 @@
 name = "temporal-sdk-core-api"
 version = "0.1.0"
 edition = "2021"
+authors = ["Spencer Judge <spencer@temporal.io>"]
+license-file = "LICENSE.txt"
+description = "Interface definitions for the Temporal Core SDK"
+homepage = "https://temporal.io/"
+repository = "https://github.com/temporalio/sdk-core"
+keywords = ["temporal", "workflow"]
+categories = ["development-tools"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
 anyhow = "1.0"
 async-trait = "0.1"
-derive_builder = "0.10"
+derive_builder = "0.11"
 log = "0.4"
+opentelemetry = "0.17"
 prost-types = "0.9"
 thiserror = "1.0"
 tonic = "0.6"

package/sdk-core/core-api/LICENSE.txt

@@ -0,0 +1,23 @@
+Temporal Core SDK
+
+The MIT License
+
+Copyright (c) 2021 Temporal Technologies, Inc. All Rights Reserved
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/sdk-core/core-api/src/errors.rs

@@ -1,29 +1,17 @@
 //! Error types exposed by public APIs
 
 use prost_types::TimestampOutOfSystemRangeError;
-use temporal_client::GatewayInitError;
 use temporal_sdk_core_protos::coresdk::{
     activity_result::ActivityExecutionResult,
     workflow_activation::remove_from_cache::EvictionReason,
     workflow_completion::WorkflowActivationCompletion,
 };
 
-/// Errors thrown during initialization of [crate::Core]
-#[derive(thiserror::Error, Debug)]
-pub enum CoreInitError {
-    /// Server connection error. Crashing and restarting the worker is likely best.
-    #[error("Server connection error: {0:?}")]
-    GatewayInitError(#[from] GatewayInitError),
-    /// There was a problem initializing telemetry
-    #[error("Telemetry initialization error: {0:?}")]
-    TelemetryInitError(anyhow::Error),
-}
-
-/// Errors thrown by [crate::Core::poll_workflow_activation]
+/// Errors thrown by [crate::Worker::poll_workflow_activation]
 #[derive(thiserror::Error, Debug)]
 pub enum PollWfError {
-    /// [crate::Core::shutdown] was called, and there are no more replay tasks to be handled. Lang
-    /// must call [crate::Core::complete_workflow_activation] for any remaining tasks, and then may
+    /// [crate::Worker::shutdown] was called, and there are no more replay tasks to be handled. Lang
+    /// must call [crate::Worker::complete_workflow_activation] for any remaining tasks, and then may
     /// exit.
     #[error("Core is shut down and there are no more workflow replay tasks")]
     ShutDown,
@@ -36,15 +24,12 @@ pub enum PollWfError {
     /// even though we already cancelled it)
     #[error("Unhandled error when auto-completing workflow task: {0:?}")]
     AutocompleteError(#[from] CompleteWfError),
-    /// There is no worker registered for the queue being polled
-    #[error("No worker registered for queue: {0}")]
-    NoWorkerForQueue(String),
 }
 
-/// Errors thrown by [crate::Core::poll_activity_task]
+/// Errors thrown by [crate::Worker::poll_activity_task]
 #[derive(thiserror::Error, Debug)]
 pub enum PollActivityError {
-    /// [crate::Core::shutdown] was called, we will no longer fetch new activity tasks. Lang must
+    /// [crate::Worker::shutdown] was called, we will no longer fetch new activity tasks. Lang must
     /// ensure it is finished with any workflow replay, see [PollWfError::ShutDown]
     #[error("Core is shut down")]
     ShutDown,
@@ -52,12 +37,9 @@ pub enum PollActivityError {
     /// errors, so lang should consider this fatal.
     #[error("Unhandled grpc error when activity polling: {0:?}")]
     TonicError(#[from] tonic::Status),
-    /// There is no worker registered for the queue being polled
-    #[error("No worker registered for queue: {0}")]
-    NoWorkerForQueue(String),
 }
 
-/// Errors thrown by [crate::Core::complete_workflow_activation]
+/// Errors thrown by [crate::Worker::complete_workflow_activation]
 #[derive(thiserror::Error, Debug)]
 #[allow(clippy::large_enum_variant)]
 pub enum CompleteWfError {
@@ -78,7 +60,7 @@ pub enum CompleteWfError {
     TonicError(#[from] tonic::Status),
 }
 
-/// Errors thrown by [crate::Core::complete_activity_task]
+/// Errors thrown by [crate::Worker::complete_activity_task]
 #[derive(thiserror::Error, Debug)]
 pub enum CompleteActivityError {
     /// Lang SDK sent us a malformed activity completion. This likely means a bug in the lang sdk.
@@ -98,14 +80,6 @@ pub enum CompleteActivityError {
     NoWorkerForQueue(String),
 }
 
-/// Errors thrown by [crate::Core::register_worker]
-#[derive(thiserror::Error, Debug)]
-pub enum WorkerRegistrationError {
-    /// A worker has already been registered on this queue
-    #[error("Worker already registered for queue: {0}")]
-    WorkerAlreadyRegisteredForQueue(String),
-}
-
 /// Errors thrown inside of workflow machines
 #[derive(thiserror::Error, Debug)]
 pub enum WFMachinesError {
@@ -116,10 +90,6 @@ pub enum WFMachinesError {
 
     #[error("Unrecoverable network error while fetching history: {0}")]
     HistoryFetchingError(tonic::Status),
-
-    /// Should always be caught internally and turned into a workflow task failure
-    #[error("Unable to process partial event history because workflow is no longer cached.")]
-    CacheMiss,
 }
 
 impl WFMachinesError {
@@ -129,7 +99,6 @@ impl WFMachinesError {
             WFMachinesError::Fatal(_) | WFMachinesError::HistoryFetchingError(_) => {
                 EvictionReason::Fatal
             }
-            WFMachinesError::CacheMiss => EvictionReason::CacheMiss,
         }
     }
 }