@temporalio/core-bridge 1.9.0 → 1.9.2

package/sdk-core/core/src/telemetry/metrics.rs

@@ -632,14 +632,14 @@ pub struct MetricsCallBuffer<I>
 where
     I: BufferInstrumentRef,
 {
-    calls_rx: crossbeam::channel::Receiver<MetricEvent<I>>,
+    calls_rx: crossbeam_channel::Receiver<MetricEvent<I>>,
     calls_tx: LogErrOnFullSender<MetricEvent<I>>,
 }
 #[derive(Clone, Debug)]
-struct LogErrOnFullSender<I>(crossbeam::channel::Sender<I>);
+struct LogErrOnFullSender<I>(crossbeam_channel::Sender<I>);
 impl<I> LogErrOnFullSender<I> {
     fn send(&self, v: I) {
-        if let Err(crossbeam::channel::TrySendError::Full(_)) = self.0.try_send(v) {
+        if let Err(crossbeam_channel::TrySendError::Full(_)) = self.0.try_send(v) {
             error!(
                 "Core's metrics buffer is full! Dropping call to record metrics. \
                  Make sure you drain the metric buffer often!"
@@ -654,7 +654,7 @@ where
 {
     /// Create a new buffer with the given capacity
     pub fn new(buffer_size: usize) -> Self {
-        let (calls_tx, calls_rx) = crossbeam::channel::bounded(buffer_size);
+        let (calls_tx, calls_rx) = crossbeam_channel::bounded(buffer_size);
         MetricsCallBuffer {
             calls_rx,
             calls_tx: LogErrOnFullSender(calls_tx),

package/sdk-core/core/src/telemetry/mod.rs

@@ -271,9 +271,7 @@ impl TemporalitySelector for ConstantTemporality {
         self.0
     }
 }
-fn metric_temporality_to_selector(
-    t: MetricTemporality,
-) -> impl TemporalitySelector + Send + Sync + Clone {
+fn metric_temporality_to_selector(t: MetricTemporality) -> impl TemporalitySelector + Clone {
     match t {
         MetricTemporality::Cumulative => ConstantTemporality(Temporality::Cumulative),
         MetricTemporality::Delta => ConstantTemporality(Temporality::Delta),

package/sdk-core/core/src/test_help/mod.rs

@@ -20,6 +20,7 @@ use mockall::TimesRange;
 use parking_lot::RwLock;
 use std::{
     collections::{BTreeMap, HashMap, HashSet, VecDeque},
+    fmt::Debug,
     ops::{Deref, DerefMut},
     pin::Pin,
     sync::{
@@ -722,6 +723,14 @@ impl<T> DerefMut for QueueResponse<T> {
         &mut self.resp
     }
 }
+impl<T> Debug for QueueResponse<T>
+where
+    T: Debug,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.resp.fmt(f)
+    }
+}
 
 pub fn hist_to_poll_resp(
     t: &TestHistoryBuilder,

package/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs

@@ -450,8 +450,7 @@ mod test {
         hm.shutdown().await;
     }
 
-    /// Ensure that heartbeat can be called from a tight loop without any throttle_interval, resulting in two
-    /// interactions with the server - one immediately and one after 500ms after the throttle_interval.
+    /// Ensure that heartbeat can be called from a tight loop and correctly throttle
     #[tokio::test]
     async fn process_tight_loop_and_shutdown() {
         let mut mock_client = mock_workflow_client();

package/sdk-core/core/src/worker/activities/local_activities.rs

@@ -1246,7 +1246,7 @@ mod tests {
         // Wait more than the timeout before grabbing the task
         sleep(timeout + Duration::from_millis(10)).await;
 
-        assert!(dbg!(lam.next_pending().await.unwrap()).is_timeout(false));
+        assert!(lam.next_pending().await.unwrap().is_timeout(false));
         assert_eq!(lam.num_in_backoff(), 0);
         assert_eq!(lam.num_outstanding(), 0);
     }

package/sdk-core/core/src/worker/activities.rs

@@ -152,6 +152,9 @@ pub(crate) struct WorkerActivityTasks {
     /// eager activities). Tasks received in this stream hold a "tracked" permit that is issued by
     /// the `eager_activities_semaphore`.
     eager_activities_tx: UnboundedSender<TrackedPermittedTqResp>,
+    /// Ensures that no activities are in the middle of flushing their results to server while we
+    /// try to shut down.
+    completers_lock: tokio::sync::RwLock<()>,
 
     metrics: MetricsContext,
 
@@ -230,6 +233,7 @@ impl WorkerActivityTasks {
             default_heartbeat_throttle_interval,
             poll_returned_shutdown_token: CancellationToken::new(),
             outstanding_activity_tasks,
+            completers_lock: Default::default(),
         }
     }
 
@@ -283,6 +287,7 @@ impl WorkerActivityTasks {
 
     pub(crate) async fn shutdown(&self) {
         self.initiate_shutdown();
+        let _ = self.completers_lock.write().await;
         self.poll_returned_shutdown_token.cancelled().await;
         self.heartbeat_manager.shutdown().await;
     }
@@ -321,10 +326,10 @@ impl WorkerActivityTasks {
                 jh.abort()
             };
             self.heartbeat_manager.evict(task_token.clone()).await;
-            self.complete_notify.notify_waiters();
 
             // No need to report activities which we already know the server doesn't care about
             if !known_not_found {
+                let _flushing_guard = self.completers_lock.read().await;
                 let maybe_net_err = match status {
                     aer::Status::WillCompleteAsync(_) => None,
                     aer::Status::Completed(ar::Success { result }) => client
@@ -364,8 +369,8 @@ impl WorkerActivityTasks {
                             {
                                 details
                             } else {
-                                warn!(task_token = ? task_token,
-                                "Expected activity cancelled status with CanceledFailureInfo");
+                                warn!(task_token=?task_token,
+                                    "Expected activity cancelled status with CanceledFailureInfo");
                                 None
                             };
                             client
@@ -376,9 +381,11 @@ impl WorkerActivityTasks {
                     }
                 };
 
+                self.complete_notify.notify_waiters();
+
                 if let Some(e) = maybe_net_err {
                     if e.code() == tonic::Code::NotFound {
-                        warn!(task_token = ?task_token, details = ?e, "Activity not found on \
+                        warn!(task_token=?task_token, details=?e, "Activity not found on \
                         completion. This may happen if the activity has already been cancelled but \
                         completed anyway.");
                     } else {

package/sdk-core/core/src/worker/mod.rs

@@ -179,8 +179,14 @@ impl WorkerTrait for Worker {
         // Let the manager know that shutdown has been initiated to try to unblock the local
         // activity poll in case this worker is an activity-only worker.
         self.local_act_mgr.shutdown_initiated();
+
         if !self.workflows.ever_polled() {
             self.local_act_mgr.workflows_have_shutdown();
+        } else {
+            // Bump the workflow stream with a pointless input, since if a client initiates shutdown
+            // and then immediately blocks waiting on a workflow activation poll, it's possible that
+            // there may not be any more inputs ever, and that poll will never resolve.
+            self.workflows.send_get_state_info_msg();
         }
     }
 
@@ -189,7 +195,6 @@ impl WorkerTrait for Worker {
     }
 
     async fn finalize_shutdown(self) {
-        self.shutdown().await;
         self.finalize_shutdown().await
     }
 }

package/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs

@@ -1250,7 +1250,6 @@ mod tests {
         });
 
         let mut worker = build_fake_sdk(mock_cfg);
-        dbg!("Past thing");
         worker.register_wf(DEFAULT_WORKFLOW_TYPE, la_wf);
         worker.register_activity(
             DEFAULT_ACTIVITY_TYPE,

package/sdk-core/core/src/worker/workflow/machines/upsert_search_attributes_state_machine.rs

@@ -208,7 +208,9 @@ mod tests {
     use temporal_sdk_core_api::Worker;
     use temporal_sdk_core_protos::{
         coresdk::{
-            workflow_commands::SetPatchMarker, workflow_completion::WorkflowActivationCompletion,
+            workflow_activation::{workflow_activation_job, WorkflowActivationJob},
+            workflow_commands::SetPatchMarker,
+            workflow_completion::WorkflowActivationCompletion,
             AsJsonPayloadExt,
         },
         temporal::api::{
@@ -332,24 +334,38 @@ mod tests {
             t.add_has_change_marker(&patch_id, false);
         }
         t.add_upsert_search_attrs_for_patch(&[patch_id.clone()]);
+        t.add_we_signaled("hi", vec![]);
         t.add_full_wf_task();
         t.add_workflow_execution_completed();
 
         let mut mp = MockPollCfg::from_resp_batches(
             "fakeid",
             t,
-            [ResponseType::ToTaskNum(1), ResponseType::AllHistory],
+            [ResponseType::ToTaskNum(1), ResponseType::ToTaskNum(2)],
             mock_workflow_client(),
         );
         // Ensure the upsert command has an empty map when not using the patched command
         if !with_patched_cmd {
             mp.completion_asserts = Some(Box::new(|wftc| {
-                assert_matches!(wftc.commands.first().and_then(|c| c.attributes.as_ref()).unwrap(),
-            Attributes::UpsertWorkflowSearchAttributesCommandAttributes(attrs)
-            if attrs.search_attributes.as_ref().unwrap().indexed_fields.is_empty())
+                let cmd_attrs = wftc
+                    .commands
+                    .first()
+                    .and_then(|c| c.attributes.as_ref())
+                    .unwrap();
+                if matches!(
+                    cmd_attrs,
+                    Attributes::CompleteWorkflowExecutionCommandAttributes(_)
+                ) {
+                    return;
+                }
+                assert_matches!(cmd_attrs,
+                    Attributes::UpsertWorkflowSearchAttributesCommandAttributes(attrs)
+                    if attrs.search_attributes.clone().unwrap_or_default().indexed_fields.is_empty());
             }));
         }
-        let core = mock_worker(build_mock_pollers(mp));
+        let mut mock = build_mock_pollers(mp);
+        mock.worker_cfg(|w| w.max_cached_workflows = 1);
+        let core = mock_worker(mock);
 
         let mut ver_upsert = HashMap::new();
         ver_upsert.insert(
@@ -379,6 +395,12 @@ mod tests {
         .unwrap();
         // Now ensure that encountering the upsert in history works fine
         let act = core.poll_workflow_activation().await.unwrap();
+        assert_matches!(
+            act.jobs.as_slice(),
+            [WorkflowActivationJob {
+                variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
+            }]
+        );
         core.complete_execution(&act.run_id).await;
     }
 }

package/sdk-core/core/src/worker/workflow/machines/workflow_machines/local_acts.rs

@@ -20,6 +20,8 @@ pub(super) struct LocalActivityData {
     /// Maps local activity sequence numbers to their resolutions as found when looking ahead at
     /// next WFT
     preresolutions: HashMap<u32, ResolveDat>,
+    /// Set true if the workflow is terminating
+    am_terminating: bool,
 }
 
 impl LocalActivityData {
@@ -45,6 +47,10 @@ impl LocalActivityData {
         wf_id: &str,
         run_id: &str,
     ) -> Vec<LocalActRequest> {
+        if self.am_terminating {
+            return vec![LocalActRequest::CancelAllInRun(run_id.to_string())];
+        }
+
         self.cancel_requests
             .drain(..)
             .map(LocalActRequest::Cancel)
@@ -65,6 +71,9 @@ impl LocalActivityData {
 
     /// Returns all outstanding local activities, whether executing or requested and in the queue
     pub(super) fn outstanding_la_count(&self) -> usize {
+        if self.am_terminating {
+            return 0;
+        }
         self.executing.len() + self.new_requests.len()
     }
 
@@ -82,4 +91,10 @@ impl LocalActivityData {
             .position(|req| req.seq == seq)
             .map(|i| self.new_requests.remove(i))
     }
+
+    /// Store that this workflow is terminating, and thus no new LA requests need be processed,
+    /// and any executing LAs should not prevent us from shutting down.
+    pub(super) fn indicate_terminating(&mut self) {
+        self.am_terminating = true;
+    }
 }

package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs

@@ -781,7 +781,10 @@ impl WorkflowMachines {
                 Ok(EventHandlingOutcome::Normal)
             };
         }
-        if event.event_type() == EventType::Unspecified || event.attributes.is_none() {
+        if event.event_type() == EventType::Unspecified
+            || event.event_type() == EventType::WorkflowExecutionUpdateRequested
+            || event.attributes.is_none()
+        {
             return if !event.worker_may_ignore {
                 Err(WFMachinesError::Fatal(format!(
                     "Event type is unspecified! This history is invalid. Event detail: {event:?}"
@@ -861,16 +864,14 @@ impl WorkflowMachines {
         let event_id = event.event_id;
 
         let consumed_cmd = loop {
-            if let Some(peek_machine) = self.commands.front() {
-                let mach = self.machine(peek_machine.machine);
-                match patch_marker_handling(event, mach, next_event)? {
-                    EventHandlingOutcome::SkipCommand => {
-                        self.commands.pop_front();
-                        continue;
-                    }
-                    eho @ EventHandlingOutcome::SkipEvent { .. } => return Ok(eho),
-                    EventHandlingOutcome::Normal => {}
+            let maybe_machine = self.commands.front().map(|mk| self.machine(mk.machine));
+            match patch_marker_handling(event, maybe_machine, next_event)? {
+                EventHandlingOutcome::SkipCommand => {
+                    self.commands.pop_front();
+                    continue;
                 }
+                eho @ EventHandlingOutcome::SkipEvent { .. } => return Ok(eho),
+                EventHandlingOutcome::Normal => {}
             }
 
             let maybe_command = self.commands.pop_front();
@@ -1469,6 +1470,9 @@ impl WorkflowMachines {
         let cwfm = self.add_new_command_machine(machine);
         self.workflow_end_time = Some(SystemTime::now());
         self.current_wf_task_commands.push_back(cwfm);
+        // Wipe out any pending / executing local activity data since we're about to terminate
+        // and there's nothing to be done with them.
+        self.local_activity_data.indicate_terminating();
     }
 
     /// Add a new command/machines for that command to the current workflow task
@@ -1598,11 +1602,11 @@ enum EventHandlingOutcome {
 /// [WorkflowMachines::handle_command_event]
 fn patch_marker_handling(
     event: &HistoryEvent,
-    mach: &Machines,
+    mach: Option<&Machines>,
     next_event: Option<&HistoryEvent>,
 ) -> Result<EventHandlingOutcome> {
     let patch_machine = match mach {
-        Machines::PatchMachine(pm) => Some(pm),
+        Some(Machines::PatchMachine(pm)) => Some(pm),
         _ => None,
     };
     let patch_details = event.get_patch_marker_details();
@@ -1633,9 +1637,9 @@ fn patch_marker_handling(
                 Ok(EventHandlingOutcome::Normal)
             }
         } else {
-            // Version markers can be skipped in the event they are deprecated
-            // Is deprecated. We can simply ignore this event, as deprecated change
-            // markers are allowed without matching changed calls.
+            // Version markers can be skipped in the event they are deprecated. We can simply
+            // ignore this event, as deprecated change markers are allowed without matching changed
+            // calls.
             if deprecated {
                 debug!("Deprecated patch marker tried against non-patch machine, skipping.");
                 skip_one_or_two_events(next_event)

package/sdk-core/core/src/worker/workflow/mod.rs

@@ -50,6 +50,7 @@ use std::{
     collections::VecDeque,
     fmt::Debug,
     future::Future,
+    mem,
     mem::discriminant,
     ops::DerefMut,
     rc::Rc,
@@ -179,58 +180,60 @@ impl Workflows {
         // We must spawn a task to constantly poll the activation stream, because otherwise
         // activation completions would not cause anything to happen until the next poll.
         let tracing_sub = telem_instance.and_then(|ti| ti.trace_subscriber());
-        let processing_task = thread::spawn(move || {
-            if let Some(ts) = tracing_sub {
-                set_trace_subscriber_for_current_thread(ts);
-            }
-            let rt = tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .thread_name("workflow-processing")
-                .build()
-                .unwrap();
-            let local = LocalSet::new();
-            local.block_on(&rt, async move {
-                let mut stream = WFStream::build(
-                    basics,
-                    extracted_wft_stream,
-                    locals_stream,
-                    local_activity_request_sink,
-                );
+        let processing_task = thread::Builder::new()
+            .name("workflow-processing".to_string())
+            .spawn(move || {
+                if let Some(ts) = tracing_sub {
+                    set_trace_subscriber_for_current_thread(ts);
+                }
+                let rt = tokio::runtime::Builder::new_current_thread()
+                    .enable_all()
+                    .build()
+                    .unwrap();
+                let local = LocalSet::new();
+                local.block_on(&rt, async move {
+                    let mut stream = WFStream::build(
+                        basics,
+                        extracted_wft_stream,
+                        locals_stream,
+                        local_activity_request_sink,
+                    );
 
-                // However, we want to avoid plowing ahead until we've been asked to poll at least
-                // once. This supports activity-only workers.
-                let do_poll = tokio::select! {
-                    sp = start_polling_rx => {
-                        sp.is_ok()
-                    }
-                    _ = shutdown_tok.cancelled() => {
-                        false
+                    // However, we want to avoid plowing ahead until we've been asked to poll at least
+                    // once. This supports activity-only workers.
+                    let do_poll = tokio::select! {
+                        sp = start_polling_rx => {
+                            sp.is_ok()
+                        }
+                        _ = shutdown_tok.cancelled() => {
+                            false
+                        }
+                    };
+                    if !do_poll {
+                        return;
                     }
-                };
-                if !do_poll {
-                    return;
-                }
-                while let Some(output) = stream.next().await {
-                    match output {
-                        Ok(o) => {
-                            for fetchreq in o.fetch_histories {
-                                fetch_tx
-                                    .send(fetchreq)
-                                    .expect("Fetch channel must not be dropped");
-                            }
-                            for act in o.activations {
-                                activation_tx
-                                    .send(Ok(act))
-                                    .expect("Activation processor channel not dropped");
+                    while let Some(output) = stream.next().await {
+                        match output {
+                            Ok(o) => {
+                                for fetchreq in o.fetch_histories {
+                                    fetch_tx
+                                        .send(fetchreq)
+                                        .expect("Fetch channel must not be dropped");
+                                }
+                                for act in o.activations {
+                                    activation_tx
+                                        .send(Ok(act))
+                                        .expect("Activation processor channel not dropped");
+                                }
                             }
+                            Err(e) => activation_tx
+                                .send(Err(e))
+                                .expect("Activation processor channel not dropped"),
                         }
-                        Err(e) => activation_tx
-                            .send(Err(e))
-                            .expect("Activation processor channel not dropped"),
                     }
-                }
-            });
-        });
+                });
+            })
+            .expect("Must be able to spawn workflow processing thread");
         Self {
             task_queue,
             local_tx,
@@ -497,10 +500,17 @@ impl Workflows {
         });
     }
 
-    /// Query the state of workflow management. Can return `None` if workflow state is shut down.
-    pub(super) fn get_state_info(&self) -> impl Future<Output = Option<WorkflowStateInfo>> {
+    /// Send a `GetStateInfoMsg` to the workflow stream. Can be used to bump the stream if there
+    /// would otherwise be no new inputs.
+    pub(super) fn send_get_state_info_msg(&self) -> oneshot::Receiver<WorkflowStateInfo> {
         let (tx, rx) = oneshot::channel();
         self.send_local(GetStateInfoMsg { response_tx: tx });
+        rx
+    }
+
+    /// Query the state of workflow management. Can return `None` if workflow state is shut down.
+    pub(super) fn get_state_info(&self) -> impl Future<Output = Option<WorkflowStateInfo>> {
+        let rx = self.send_get_state_info_msg();
         async move { rx.await.ok() }
     }
 
@@ -1023,17 +1033,31 @@ struct BufferedTasks {
     /// supersede any old one.
     wft: Option<PermittedWFT>,
     /// For query only tasks, multiple may be received concurrently and it's OK to buffer more
-    /// than one - however they should be dropped if, by the time we try to process them, we
-    /// have already processed a newer real WFT than the one the query was targeting (otherwise
-    /// we'd return data from the "future").
+    /// than one - however they must all be handled before applying the next "real" wft (after the
+    /// current one has been processed).
     query_only_tasks: VecDeque<PermittedWFT>,
+    /// These are query-only tasks for the *buffered* wft, if any. They will all be discarded if
+    /// a buffered wft is replaced before being handled. They move to `query_only_tasks` once the
+    /// buffered task is taken.
+    query_only_tasks_for_buffered: VecDeque<PermittedWFT>,
 }
 
 impl BufferedTasks {
+    /// Buffers a new task. If it is a query-only task, multiple such tasks may be buffered which
+    /// all will be handled at the end of the current WFT. If a new WFT which would advance history
+    /// is provided, it will be buffered - but if another such task comes in while there is already
+    /// one buffered, the old one will be overriden, and all queries will be invalidated.
     fn buffer(&mut self, task: PermittedWFT) {
         if task.work.is_query_only() {
-            self.query_only_tasks.push_back(task);
+            if self.wft.is_none() {
+                self.query_only_tasks.push_back(task);
+            } else {
+                self.query_only_tasks_for_buffered.push_back(task);
+            }
         } else {
+            if self.wft.is_some() {
+                self.query_only_tasks_for_buffered.clear();
+            }
             let _ = self.wft.insert(task);
         }
     }
@@ -1042,12 +1066,18 @@ impl BufferedTasks {
         self.wft.is_some() || !self.query_only_tasks.is_empty()
     }
 
-    /// Remove and return the next WFT from the buffer that should be applied. WFTs which would
-    /// advance workflow state are returned before query-only tasks.
+    /// Remove and return the next WFT from the buffer that should be applied. Queries are returned
+    /// first for the current workflow task, if there are any. If not, the next WFT that would
+    /// advance history is returned.
     fn get_next_wft(&mut self) -> Option<PermittedWFT> {
-        self.wft
-            .take()
-            .or_else(|| self.query_only_tasks.pop_front())
+        if let Some(q) = self.query_only_tasks.pop_front() {
+            return Some(q);
+        }
+        if let Some(t) = self.wft.take() {
+            self.query_only_tasks = mem::take(&mut self.query_only_tasks_for_buffered);
+            return Some(t);
+        }
+        None
     }
 }
 
@@ -1310,7 +1340,7 @@ impl From<OutgoingJob> for WorkflowActivationJob {
 /// Errors thrown inside of workflow machines
 #[derive(thiserror::Error, Debug)]
 pub(crate) enum WFMachinesError {
-    #[error("Nondeterminism error: {0}")]
+    #[error("[TMPRL1100] Nondeterminism error: {0}")]
     Nondeterminism(String),
     #[error("Fatal error in workflow machines: {0}")]
     Fatal(String),

package/sdk-core/core/src/worker/workflow/workflow_stream/saved_wf_inputs.rs

@@ -9,7 +9,7 @@ use crate::{
         LocalActRequest, LocalActivityResolution,
     },
 };
-use crossbeam::queue::SegQueue;
+use crossbeam_queue::SegQueue;
 use futures::Stream;
 use futures_util::StreamExt;
 use serde::{Deserialize, Serialize};

package/sdk-core/core-api/Cargo.toml

@@ -18,7 +18,7 @@ otel_impls = ["opentelemetry"]
 [dependencies]
 async-trait = "0.1"
 derive_builder = "0.12"
-derive_more = "0.99"
+derive_more = { workspace = true }
 opentelemetry = { workspace = true, optional = true }
 prost-types = "0.11"
 serde = { version = "1.0", default_features = false, features = ["derive"] }
@@ -31,4 +31,4 @@ url = "2.3"
 
 [dependencies.temporal-sdk-core-protos]
 path = "../sdk-core-protos"
-version = "0.1"
+version = "0.1"

package/sdk-core/fsm/rustfsm_procmacro/Cargo.toml

@@ -14,7 +14,7 @@ name = "tests"
 path = "tests/progress.rs"
 
 [dependencies]
-derive_more = "0.99"
+derive_more = { workspace = true }
 proc-macro2 = "1.0"
 syn = { version = "2.0", features = ["default", "extra-traits"] }
 quote = "1.0"

package/sdk-core/sdk/Cargo.toml

@@ -17,8 +17,8 @@ async-trait = "0.1"
 thiserror = "1.0"
 anyhow = "1.0"
 base64 = "0.21"
-crossbeam = "0.8"
-derive_more = "0.99"
+crossbeam-channel = "0.5"
+derive_more = { workspace = true }
 futures = "0.3"
 once_cell = "1.10"
 parking_lot = { version = "0.12", features = ["send_guard"] }