@temporalio/core-bridge 1.7.4 → 1.8.1

package/sdk-core/core/src/pollers/poll_buffer.rs

@@ -1,51 +1,65 @@
 use crate::{
+    abstractions::{dbg_panic, MeteredSemaphore, OwnedMeteredSemPermit},
     pollers::{self, Poller},
     worker::client::WorkerClient,
 };
 use futures::{prelude::stream::FuturesUnordered, StreamExt};
+use futures_util::{future::BoxFuture, FutureExt};
+use governor::{Quota, RateLimiter};
 use std::{
     fmt::Debug,
     future::Future,
     sync::{
-        atomic::{AtomicUsize, Ordering},
+        atomic::{AtomicBool, AtomicUsize, Ordering},
         Arc,
     },
+    time::Duration,
 };
-use temporal_sdk_core_protos::temporal::api::workflowservice::v1::{
-    PollActivityTaskQueueResponse, PollWorkflowTaskQueueResponse,
+use temporal_sdk_core_protos::temporal::api::{
+    taskqueue::v1::TaskQueue,
+    workflowservice::v1::{PollActivityTaskQueueResponse, PollWorkflowTaskQueueResponse},
 };
 use tokio::{
     sync::{
-        mpsc::{channel, Receiver},
-        Mutex, Semaphore,
+        broadcast,
+        mpsc::{unbounded_channel, UnboundedReceiver},
+        Mutex,
     },
     task::JoinHandle,
 };
 use tokio_util::sync::CancellationToken;
 
 pub struct LongPollBuffer<T> {
-    buffered_polls: Mutex<Receiver<pollers::Result<T>>>,
+    buffered_polls: Mutex<UnboundedReceiver<pollers::Result<(T, OwnedMeteredSemPermit)>>>,
     shutdown: CancellationToken,
-    /// This semaphore exists to ensure that we only poll server as many times as core actually
-    /// *asked* it to be polled - otherwise we might spin and buffer polls constantly. This also
-    /// means unit tests can continue to function in a predictable manner when calling mocks.
-    polls_requested: Arc<Semaphore>,
     join_handles: FuturesUnordered<JoinHandle<()>>,
-    /// Called every time the number of pollers is changed
-    num_pollers_changed: Option<Box<dyn Fn(usize) + Send + Sync>>,
-    active_pollers: Arc<AtomicUsize>,
+    /// Pollers won't actually start polling until initialized & value is sent
+    starter: broadcast::Sender<()>,
+    did_start: AtomicBool,
 }
 
-struct ActiveCounter<'a>(&'a AtomicUsize);
-impl<'a> ActiveCounter<'a> {
-    fn new(a: &'a AtomicUsize) -> Self {
-        a.fetch_add(1, Ordering::Relaxed);
-        Self(a)
+struct ActiveCounter<'a, F: Fn(usize)>(&'a AtomicUsize, Option<F>);
+impl<'a, F> ActiveCounter<'a, F>
+where
+    F: Fn(usize),
+{
+    fn new(a: &'a AtomicUsize, change_fn: Option<F>) -> Self {
+        let v = a.fetch_add(1, Ordering::Relaxed) + 1;
+        if let Some(cfn) = change_fn.as_ref() {
+            cfn(v);
+        }
+        Self(a, change_fn)
     }
 }
-impl Drop for ActiveCounter<'_> {
+impl<F> Drop for ActiveCounter<'_, F>
+where
+    F: Fn(usize),
+{
     fn drop(&mut self) {
-        self.0.fetch_sub(1, Ordering::Relaxed);
+        let v = self.0.fetch_sub(1, Ordering::Relaxed) - 1;
+        if let Some(cfn) = self.1.as_ref() {
+            cfn(v)
+        }
     }
 }
 
@@ -53,42 +67,67 @@ impl<T> LongPollBuffer<T>
 where
     T: Send + Debug + 'static,
 {
-    pub fn new<FT>(
+    pub(crate) fn new<FT, DelayFut>(
         poll_fn: impl Fn() -> FT + Send + Sync + 'static,
+        poll_semaphore: Arc<MeteredSemaphore>,
         max_pollers: usize,
-        buffer_size: usize,
         shutdown: CancellationToken,
+        num_pollers_handler: Option<impl Fn(usize) + Send + Sync + 'static>,
+        pre_permit_delay: Option<impl Fn() -> DelayFut + Send + Sync + 'static>,
     ) -> Self
     where
         FT: Future<Output = pollers::Result<T>> + Send,
+        DelayFut: Future<Output = ()> + Send,
     {
-        let (tx, rx) = channel(buffer_size);
-        let polls_requested = Arc::new(Semaphore::new(0));
+        let (tx, rx) = unbounded_channel();
+        let (starter, wait_for_start) = broadcast::channel(1);
         let active_pollers = Arc::new(AtomicUsize::new(0));
         let join_handles = FuturesUnordered::new();
         let pf = Arc::new(poll_fn);
+        let nph = num_pollers_handler.map(Arc::new);
+        let pre_permit_delay = pre_permit_delay.map(Arc::new);
         for _ in 0..max_pollers {
             let tx = tx.clone();
             let pf = pf.clone();
             let shutdown = shutdown.clone();
-            let polls_requested = polls_requested.clone();
             let ap = active_pollers.clone();
+            let poll_semaphore = poll_semaphore.clone();
+            let nph = nph.clone();
+            let pre_permit_delay = pre_permit_delay.clone();
+            let mut wait_for_start = wait_for_start.resubscribe();
             let jh = tokio::spawn(async move {
+                tokio::select! {
+                    _ = wait_for_start.recv() => (),
+                    _ = shutdown.cancelled() => return,
+                }
+                drop(wait_for_start);
+
+                let nph = nph.as_ref().map(|a| a.as_ref());
                 loop {
                     if shutdown.is_cancelled() {
                         break;
                     }
-                    let sp = tokio::select! {
-                        sp = polls_requested.acquire() => sp.expect("Polls semaphore not dropped"),
-                        _ = shutdown.cancelled() => continue,
+                    if let Some(ref ppd) = pre_permit_delay {
+                        tokio::select! {
+                            _ = ppd() => (),
+                            _ = shutdown.cancelled() => break,
+                        }
+                    }
+                    let permit = tokio::select! {
+                        p = poll_semaphore.acquire_owned() => p,
+                        _ = shutdown.cancelled() => break,
+                    };
+                    let permit = if let Ok(p) = permit {
+                        p
+                    } else {
+                        break;
                     };
-                    let _active_guard = ActiveCounter::new(ap.as_ref());
+                    let _active_guard = ActiveCounter::new(ap.as_ref(), nph);
                     let r = tokio::select! {
                         r = pf() => r,
-                        _ = shutdown.cancelled() => continue,
+                        _ = shutdown.cancelled() => break,
                     };
-                    sp.forget();
-                    let _ = tx.send(r).await;
+                    let _ = tx.send(r.map(|r| (r, permit)));
                 }
             });
             join_handles.push(jh);
@@ -96,50 +135,29 @@ where
         Self {
             buffered_polls: Mutex::new(rx),
             shutdown,
-            polls_requested,
             join_handles,
-            num_pollers_changed: None,
-            active_pollers,
+            starter,
+            did_start: AtomicBool::new(false),
         }
     }
-
-    /// Set a function that will be called every time the number of pollers changes.
-    /// TODO: Currently a bit weird, will make more sense once we implement dynamic poller scaling.
-    pub fn set_num_pollers_handler(&mut self, handler: impl Fn(usize) + Send + Sync + 'static) {
-        self.num_pollers_changed = Some(Box::new(handler));
-    }
 }
 
 #[async_trait::async_trait]
-impl<T> Poller<T> for LongPollBuffer<T>
+impl<T> Poller<(T, OwnedMeteredSemPermit)> for LongPollBuffer<T>
 where
     T: Send + Sync + Debug + 'static,
 {
-    /// Poll the buffer. Adds one permit to the polling pool - the point of this being that the
-    /// buffer may support many concurrent pollers, but there is no reason to have them poll unless
-    /// enough polls have actually been requested. Calling this function adds a permit that any
-    /// concurrent poller may fulfill.
-    ///
-    /// EX: If this function is only ever called serially and always `await`ed, there will be no
-    /// concurrent polling. If it is called many times and the futures are awaited concurrently,
-    /// then polling will happen concurrently.
+    /// Poll for the next item from this poller
     ///
-    /// Returns `None` if the poll buffer has been shut down
+    /// Returns `None` if the poller has been shut down
     #[instrument(name = "long_poll", level = "trace", skip(self))]
-    async fn poll(&self) -> Option<pollers::Result<T>> {
-        self.polls_requested.add_permits(1);
-        if let Some(fun) = self.num_pollers_changed.as_ref() {
-            fun(self.active_pollers.load(Ordering::Relaxed));
+    async fn poll(&self) -> Option<pollers::Result<(T, OwnedMeteredSemPermit)>> {
+        if !self.did_start.fetch_or(true, Ordering::Relaxed) {
+            let _ = self.starter.send(());
         }
 
         let mut locked = self.buffered_polls.lock().await;
-        let res = (*locked).recv().await;
-
-        if let Some(fun) = self.num_pollers_changed.as_ref() {
-            fun(self.active_pollers.load(Ordering::Relaxed));
-        }
-
-        res
+        (*locked).recv().await
     }
 
     fn notify_shutdown(&self) {
@@ -148,7 +166,13 @@ where
 
     async fn shutdown(mut self) {
         self.notify_shutdown();
-        while self.join_handles.next().await.is_some() {}
+        while let Some(jh) = self.join_handles.next().await {
+            if let Err(e) = jh {
+                if !e.is_cancelled() {
+                    dbg_panic!("Poller task did not terminate cleanly: {:?}", e);
+                }
+            }
+        }
     }
 
     async fn shutdown_box(self: Box<Self>) {
@@ -165,8 +189,10 @@ pub struct WorkflowTaskPoller {
 }
 
 #[async_trait::async_trait]
-impl Poller<PollWorkflowTaskQueueResponse> for WorkflowTaskPoller {
-    async fn poll(&self) -> Option<pollers::Result<PollWorkflowTaskQueueResponse>> {
+impl Poller<(PollWorkflowTaskQueueResponse, OwnedMeteredSemPermit)> for WorkflowTaskPoller {
+    async fn poll(
+        &self,
+    ) -> Option<pollers::Result<(PollWorkflowTaskQueueResponse, OwnedMeteredSemPermit)>> {
         if let Some(sq) = self.sticky_poller.as_ref() {
             tokio::select! {
                 r = self.normal_poller.poll() => r,
@@ -200,51 +226,106 @@ impl Poller<PollWorkflowTaskQueueResponse> for WorkflowTaskPoller {
 pub type PollWorkflowTaskBuffer = LongPollBuffer<PollWorkflowTaskQueueResponse>;
 pub(crate) fn new_workflow_task_buffer(
     client: Arc<dyn WorkerClient>,
-    task_queue: String,
-    is_sticky: bool,
+    task_queue: TaskQueue,
     concurrent_pollers: usize,
-    buffer_size: usize,
+    semaphore: Arc<MeteredSemaphore>,
     shutdown: CancellationToken,
+    num_pollers_handler: Option<impl Fn(usize) + Send + Sync + 'static>,
 ) -> PollWorkflowTaskBuffer {
     LongPollBuffer::new(
         move || {
             let client = client.clone();
             let task_queue = task_queue.clone();
-            async move { client.poll_workflow_task(task_queue, is_sticky).await }
+            async move { client.poll_workflow_task(task_queue).await }
         },
+        semaphore,
         concurrent_pollers,
-        buffer_size,
         shutdown,
+        num_pollers_handler,
+        None::<fn() -> BoxFuture<'static, ()>>,
     )
 }
 
 pub type PollActivityTaskBuffer = LongPollBuffer<PollActivityTaskQueueResponse>;
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn new_activity_task_buffer(
     client: Arc<dyn WorkerClient>,
     task_queue: String,
     concurrent_pollers: usize,
-    buffer_size: usize,
+    semaphore: Arc<MeteredSemaphore>,
     max_tps: Option<f64>,
     shutdown: CancellationToken,
+    num_pollers_handler: Option<impl Fn(usize) + Send + Sync + 'static>,
+    max_worker_acts_per_sec: Option<f64>,
 ) -> PollActivityTaskBuffer {
+    let rate_limiter = max_worker_acts_per_sec.and_then(|ps| {
+        Quota::with_period(Duration::from_secs_f64(ps.recip()))
+            .map(|q| Arc::new(RateLimiter::direct(q)))
+    });
     LongPollBuffer::new(
         move || {
             let client = client.clone();
             let task_queue = task_queue.clone();
             async move { client.poll_activity_task(task_queue, max_tps).await }
         },
+        semaphore,
         concurrent_pollers,
-        buffer_size,
         shutdown,
+        num_pollers_handler,
+        rate_limiter.map(|rl| {
+            move || {
+                let rl = rl.clone();
+                async move { rl.until_ready().await }.boxed()
+            }
+        }),
     )
 }
 
+#[cfg(test)]
+#[derive(derive_more::Constructor)]
+pub(crate) struct MockPermittedPollBuffer<PT> {
+    sem: Arc<MeteredSemaphore>,
+    inner: PT,
+}
+
+#[cfg(test)]
+#[async_trait::async_trait]
+impl<T, PT> Poller<(T, OwnedMeteredSemPermit)> for MockPermittedPollBuffer<PT>
+where
+    T: Send + Sync + 'static,
+    PT: Poller<T> + Send + Sync + 'static,
+{
+    async fn poll(&self) -> Option<pollers::Result<(T, OwnedMeteredSemPermit)>> {
+        let p = self
+            .sem
+            .acquire_owned()
+            .await
+            .expect("Semaphore in poller not closed!");
+        self.inner.poll().await.map(|r| r.map(|r| (r, p)))
+    }
+
+    fn notify_shutdown(&self) {
+        self.inner.notify_shutdown();
+    }
+
+    async fn shutdown(self) {
+        self.inner.shutdown().await;
+    }
+
+    async fn shutdown_box(self: Box<Self>) {
+        self.inner.shutdown().await;
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::worker::client::mocks::mock_manual_workflow_client;
+    use crate::{
+        telemetry::metrics::MetricsContext, worker::client::mocks::mock_manual_workflow_client,
+    };
     use futures::FutureExt;
     use std::time::Duration;
+    use temporal_sdk_core_protos::temporal::api::enums::v1::TaskQueueKind;
     use tokio::{select, sync::mpsc::channel};
 
     #[tokio::test]
@@ -253,7 +334,7 @@ mod tests {
         mock_client
             .expect_poll_workflow_task()
             .times(2)
-            .returning(move |_, _| {
+            .returning(move |_| {
                 async {
                     tokio::time::sleep(Duration::from_millis(100)).await;
                     Ok(Default::default())
@@ -263,11 +344,19 @@ mod tests {
 
         let pb = new_workflow_task_buffer(
             Arc::new(mock_client),
-            "someq".to_string(),
-            false,
-            1,
+            TaskQueue {
+                name: "sometq".to_string(),
+                kind: TaskQueueKind::Normal as i32,
+                normal_name: "".to_string(),
+            },
             1,
+            Arc::new(MeteredSemaphore::new(
+                10,
+                MetricsContext::no_op(),
+                |_, _| {},
+            )),
             CancellationToken::new(),
+            None::<fn(usize)>,
         );
 
         // Poll a bunch of times, "interrupting" it each time, we should only actually have polled

package/sdk-core/core/src/protosext/mod.rs

@@ -337,10 +337,7 @@ impl Default for LACloseTimeouts {
 }
 
 impl ValidScheduleLA {
-    pub fn from_schedule_la(
-        v: ScheduleLocalActivity,
-        wf_exe_timeout: Option<Duration>,
-    ) -> Result<Self, anyhow::Error> {
+    pub fn from_schedule_la(v: ScheduleLocalActivity) -> Result<Self, anyhow::Error> {
         let original_schedule_time = v
             .original_schedule_time
             .map(|x| {
@@ -354,9 +351,7 @@ impl ValidScheduleLA {
                 x.try_into()
                     .map_err(|_| anyhow!("Could not convert schedule_to_close_timeout"))
             })
-            .transpose()?
-            // Default to execution timeout if unset
-            .or(wf_exe_timeout);
+            .transpose()?;
         let mut schedule_to_start_timeout = v
             .schedule_to_start_timeout
             .map(|x| {
@@ -392,7 +387,8 @@ impl ValidScheduleLA {
             }
             (None, None) => {
                 return Err(anyhow!(
-                    "One of schedule_to_close or start_to_close timeouts must be set"
+                    "One or both of schedule_to_close or start_to_close timeouts must be set for \
+                     local activities"
                 ))
             }
         };

package/sdk-core/core/src/replay/mod.rs

@@ -89,7 +89,7 @@ pub(crate) fn mock_client_from_histories(historator: Historator) -> impl WorkerC
     let hist_allow_tx = historator.replay_done_tx.clone();
     let historator = Arc::new(TokioMutex::new(historator));
 
-    mg.expect_poll_workflow_task().returning(move |_, _| {
+    mg.expect_poll_workflow_task().returning(move |_| {
         let historator = historator.clone();
         async move {
             let mut hlock = historator.lock().await;

package/sdk-core/core/src/telemetry/metrics.rs

@@ -87,6 +87,7 @@ struct Instruments {
     sticky_cache_hit: Counter<u64>,
     sticky_cache_miss: Counter<u64>,
     sticky_cache_size: Histogram<u64>,
+    sticky_cache_evictions: Counter<u64>,
 }
 
 impl MetricsContext {
@@ -288,6 +289,13 @@ impl MetricsContext {
             .sticky_cache_size
             .record(&self.ctx, size, &self.kvs);
     }
+
+    /// Count a workflow being evicted from the cache
+    pub(crate) fn cache_eviction(&self) {
+        self.instruments
+            .sticky_cache_evictions
+            .add(&self.ctx, 1, &self.kvs);
+    }
 }
 
 impl Instruments {
@@ -327,6 +335,7 @@ impl Instruments {
             sticky_cache_hit: meter.counter("sticky_cache_hit"),
             sticky_cache_miss: meter.counter("sticky_cache_miss"),
             sticky_cache_size: meter.histogram(STICKY_CACHE_SIZE_NAME),
+            sticky_cache_evictions: meter.counter("sticky_cache_total_forced_eviction"),
         }
     }
 }

package/sdk-core/core/src/telemetry/mod.rs

@@ -297,6 +297,9 @@ pub fn telemetry_init(opts: TelemetryOptions) -> Result<TelemetryInstance, anyho
             .with(forward_layer)
             .with(export_layer);
 
+        #[cfg(feature = "tokio-console")]
+        let reg = reg.with(console_subscriber::spawn());
+
         tx.send(TelemetryInstance::new(
             Arc::new(reg),
             logs_out,

package/sdk-core/core/src/test_help/mod.rs

@@ -1,7 +1,7 @@
 pub(crate) use temporal_sdk_core_test_utils::canned_histories;
 
 use crate::{
-    pollers::{BoxedActPoller, BoxedPoller, BoxedWFPoller, MockManualPoller, MockPoller},
+    pollers::{BoxedPoller, MockManualPoller, MockPoller},
     protosext::ValidPollWFTQResponse,
     replay::TestHistoryBuilder,
     sticky_q_name_for_worker,
@@ -10,7 +10,7 @@ use crate::{
         client::{
             mocks::mock_workflow_client, MockWorkerClient, WorkerClient, WorkflowTaskCompletion,
         },
-        new_wft_poller,
+        TaskPollers,
     },
     TaskToken, Worker, WorkerConfig, WorkerConfigBuilder,
 };
@@ -53,7 +53,6 @@ use temporal_sdk_core_protos::{
 use temporal_sdk_core_test_utils::TestWorker;
 use tokio::sync::{mpsc::unbounded_channel, Notify};
 use tokio_stream::wrappers::UnboundedReceiverStream;
-use tokio_util::sync::CancellationToken;
 
 pub const TEST_Q: &str = "q";
 
@@ -148,11 +147,12 @@ pub(crate) fn mock_worker(mocks: MocksHolder) -> Worker {
         mocks.inputs.config,
         sticky_q,
         mocks.client,
-        mocks.inputs.wft_stream,
-        act_poller,
+        TaskPollers::Mocked {
+            wft_stream: mocks.inputs.wft_stream,
+            act_poller,
+        },
         MetricsContext::no_op(),
         None,
-        CancellationToken::new(),
     )
 }
 
@@ -187,7 +187,7 @@ impl MocksHolder {
     pub fn worker_cfg(&mut self, mutator: impl FnOnce(&mut WorkerConfig)) {
         mutator(&mut self.inputs.config);
     }
-    pub fn set_act_poller(&mut self, poller: BoxedActPoller) {
+    pub fn set_act_poller(&mut self, poller: BoxedPoller<PollActivityTaskQueueResponse>) {
         self.inputs.act_poller = Some(poller);
     }
     /// Can be used for tests that need to avoid auto-shutdown due to running out of mock responses
@@ -199,13 +199,13 @@ impl MocksHolder {
 
 pub struct MockWorkerInputs {
     pub wft_stream: BoxStream<'static, Result<ValidPollWFTQResponse, tonic::Status>>,
-    pub act_poller: Option<BoxedActPoller>,
+    pub act_poller: Option<BoxedPoller<PollActivityTaskQueueResponse>>,
     pub config: WorkerConfig,
 }
 
 impl Default for MockWorkerInputs {
     fn default() -> Self {
-        Self::new_from_poller(Box::from(mock_poller()))
+        Self::new(stream::empty().boxed())
     }
 }
 
@@ -219,13 +219,6 @@ impl MockWorkerInputs {
             config: test_worker_cfg().build().unwrap(),
         }
     }
-    pub fn new_from_poller(wf_poller: BoxedWFPoller) -> Self {
-        Self {
-            wft_stream: new_wft_poller(wf_poller, MetricsContext::no_op()).boxed(),
-            act_poller: None,
-            config: test_worker_cfg().build().unwrap(),
-        }
-    }
 }
 
 impl MocksHolder {

package/sdk-core/core/src/worker/activities/activity_task_poller_stream.rs

@@ -1,23 +1,11 @@
-use crate::{
-    abstractions::MeteredSemaphore, pollers::BoxedActPoller, worker::activities::PermittedTqResp,
-    MetricsContext,
-};
+use crate::{pollers::BoxedActPoller, worker::activities::PermittedTqResp, MetricsContext};
 use futures::{stream, Stream};
-use governor::{
-    clock::DefaultClock,
-    middleware::NoOpMiddleware,
-    state::{InMemoryState, NotKeyed},
-    RateLimiter,
-};
-use std::sync::Arc;
 use temporal_sdk_core_protos::temporal::api::workflowservice::v1::PollActivityTaskQueueResponse;
 use tokio::select;
 use tokio_util::sync::CancellationToken;
 
 struct StreamState {
     poller: BoxedActPoller,
-    semaphore: Arc<MeteredSemaphore>,
-    rate_limiter: Option<RateLimiter<NotKeyed, InMemoryState, DefaultClock, NoOpMiddleware>>,
     metrics: MetricsContext,
     shutdown_token: CancellationToken,
     poller_was_shutdown: bool,
@@ -25,15 +13,11 @@ struct StreamState {
 
 pub(crate) fn new_activity_task_poller(
     poller: BoxedActPoller,
-    semaphore: Arc<MeteredSemaphore>,
-    rate_limiter: Option<RateLimiter<NotKeyed, InMemoryState, DefaultClock, NoOpMiddleware>>,
     metrics: MetricsContext,
     shutdown_token: CancellationToken,
 ) -> impl Stream<Item = Result<PermittedTqResp, tonic::Status>> {
     let state = StreamState {
         poller,
-        semaphore,
-        rate_limiter,
         metrics,
         shutdown_token,
         poller_was_shutdown: false,
@@ -41,21 +25,12 @@ pub(crate) fn new_activity_task_poller(
     stream::unfold(state, |mut state| async move {
         loop {
             let poll = async {
-                let permit = state
-                    .semaphore
-                    .acquire_owned()
-                    .await
-                    .expect("outstanding activity semaphore not closed");
-                if !state.poller_was_shutdown {
-                    if let Some(ref rl) = state.rate_limiter {
-                        rl.until_ready().await;
-                    }
-                }
                 loop {
                     return match state.poller.poll().await {
-                        Some(Ok(resp)) => {
+                        Some(Ok((resp, permit))) => {
                             if resp == PollActivityTaskQueueResponse::default() {
-                                // We get the default proto in the event that the long poll times out.
+                                // We get the default proto in the event that the long poll times
+                                // out.
                                 debug!("Poll activity task timeout");
                                 state.metrics.act_poll_timeout();
                                 continue;
@@ -66,8 +41,8 @@ pub(crate) fn new_activity_task_poller(
                             warn!(error=?e, "Error while polling for activity tasks");
                             Some(Err(e))
                         }
-                        // If poller returns None, it's dead, thus we also return None to terminate this
-                        // stream.
+                        // If poller returns None, it's dead, thus we also return None to terminate
+                        // this stream.
                         None => None,
                     };
                 }