@temporalio/core-bridge 1.4.3 → 1.5.0

package/sdk-core/core/src/worker/workflow/managed_run.rs

@@ -357,6 +357,7 @@ impl ManagedRun {
         let query_responses = data.query_responses;
         let has_query_responses = !query_responses.is_empty();
         let is_query_playback = data.has_pending_query && !has_query_responses;
+        let mut force_new_wft = due_to_heartbeat_timeout;
 
         // We only actually want to send commands back to the server if there are no more
         // pending activations and we are caught up on replay. We don't want to complete a wft
@@ -366,19 +367,26 @@ impl ManagedRun {
         // either.
         let no_commands_and_evicting =
             outgoing_cmds.commands.is_empty() && data.activation_was_only_eviction;
+        let should_respond = !(self.wfm.machines.has_pending_jobs()
+            || outgoing_cmds.replaying
+            || is_query_playback
+            || no_commands_and_evicting);
+        // If there are pending LA resolutions, and we're responding to a query here,
+        // we want to make sure to force a new task, as otherwise once we tell lang about
+        // the LA resolution there wouldn't be any task to reply to with the result of iterating
+        // the workflow.
+        if has_query_responses && self.wfm.machines.has_pending_la_resolutions() {
+            force_new_wft = true;
+        }
         let to_be_sent = ServerCommandsWithWorkflowInfo {
             task_token: data.task_token,
             action: ActivationAction::WftComplete {
-                force_new_wft: due_to_heartbeat_timeout,
+                force_new_wft,
                 commands: outgoing_cmds.commands,
                 query_responses,
             },
         };
 
-        let should_respond = !(self.wfm.machines.has_pending_jobs()
-            || outgoing_cmds.replaying
-            || is_query_playback
-            || no_commands_and_evicting);
         let outcome = if should_respond || has_query_responses {
             ActivationCompleteOutcome::ReportWFTSuccess(to_be_sent)
         } else {

package/sdk-core/core/src/worker/workflow/mod.rs

@@ -24,7 +24,7 @@ use crate::{
     telemetry::VecDisplayer,
     worker::{
         activities::{ActivitiesFromWFTsHandle, PermittedTqResp},
-        client::WorkerClient,
+        client::{WorkerClient, WorkflowTaskCompletion},
         workflow::{
             managed_run::{ManagedRun, WorkflowManager},
             wft_poller::validate_wft,
@@ -36,6 +36,7 @@ use crate::{
 };
 use futures::{stream::BoxStream, Stream, StreamExt};
 use std::{
+    collections::HashSet,
     fmt::{Debug, Display, Formatter},
     future::Future,
     ops::DerefMut,
@@ -43,12 +44,12 @@ use std::{
     sync::Arc,
     time::{Duration, Instant},
 };
-use temporal_client::WorkflowTaskCompletion;
 use temporal_sdk_core_api::errors::{CompleteWfError, PollWfError};
 use temporal_sdk_core_protos::{
     coresdk::{
         workflow_activation::{
-            remove_from_cache::EvictionReason, QueryWorkflow, WorkflowActivation,
+            remove_from_cache::EvictionReason, workflow_activation_job, QueryWorkflow,
+            WorkflowActivation, WorkflowActivationJob,
         },
         workflow_commands::*,
         workflow_completion,
@@ -108,6 +109,7 @@ pub(super) struct WorkflowBasics {
     pub metrics: MetricsContext,
     pub namespace: String,
     pub task_queue: String,
+    pub ignore_evicts_on_shutdown: bool,
 }
 
 impl Workflows {
@@ -180,7 +182,7 @@ impl Workflows {
                 }
                 stream.next().await.unwrap_or(Err(PollWfError::ShutDown))?
             };
-            Span::current().record("run_id", &r.run_id());
+            Span::current().record("run_id", r.run_id());
             match r {
                 ActivationOrAuto::LangActivation(act) | ActivationOrAuto::ReadyForQueries(act) => {
                     debug!(activation=%act, "Sending activation to lang");
@@ -458,32 +460,34 @@ impl Workflows {
         commands: &mut [Command],
     ) -> Vec<OwnedMeteredSemPermit> {
         let mut reserved = vec![];
-        if let Some(at_handle) = self.activity_tasks_handle.as_ref() {
-            for cmd in commands {
-                if let Some(Attributes::ScheduleActivityTaskCommandAttributes(attrs)) =
-                    cmd.attributes.as_mut()
-                {
-                    // If request_eager_execution was already false, that means lang explicitly
-                    // told us it didn't want to eagerly execute for some reason. So, we only
-                    // ever turn *off* eager execution if a slot is not available or the activity
-                    // is scheduled on a different task queue.
-                    if attrs.request_eager_execution {
-                        let same_task_queue = attrs
-                            .task_queue
+        for cmd in commands {
+            if let Some(Attributes::ScheduleActivityTaskCommandAttributes(attrs)) =
+                cmd.attributes.as_mut()
+            {
+                // If request_eager_execution was already false, that means lang explicitly
+                // told us it didn't want to eagerly execute for some reason. So, we only
+                // ever turn *off* eager execution if a slot is not available or the activity
+                // is scheduled on a different task queue.
+                if attrs.request_eager_execution {
+                    let same_task_queue = attrs
+                        .task_queue
+                        .as_ref()
+                        .map(|q| q.name == self.task_queue)
+                        .unwrap_or_default();
+                    if same_task_queue
+                        && reserved.len() < MAX_EAGER_ACTIVITY_RESERVATIONS_PER_WORKFLOW_TASK
+                    {
+                        if let Some(p) = self
+                            .activity_tasks_handle
                             .as_ref()
-                            .map(|q| q.name == self.task_queue)
-                            .unwrap_or_default();
-                        if same_task_queue
-                            && reserved.len() < MAX_EAGER_ACTIVITY_RESERVATIONS_PER_WORKFLOW_TASK
+                            .and_then(|h| h.reserve_slot())
                         {
-                            if let Some(p) = at_handle.reserve_slot() {
-                                reserved.push(p);
-                            } else {
-                                attrs.request_eager_execution = false;
-                            }
+                            reserved.push(p);
                         } else {
                             attrs.request_eager_execution = false;
                         }
+                    } else {
+                        attrs.request_eager_execution = false;
                     }
                 }
             }
@@ -546,6 +550,11 @@ struct ManagedRunHandle {
     run_actions_tx: UnboundedSender<RunAction>,
     /// Handle to the task where the actual machines live
     handle: JoinHandle<()>,
+
+    /// We track if we have recorded useful debugging values onto a certain span yet, to overcome
+    /// duplicating field values. Remove this once https://github.com/tokio-rs/tracing/issues/2334
+    /// is fixed.
+    recorded_span_ids: HashSet<tracing::Id>,
     metrics: MetricsContext,
 }
 impl ManagedRunHandle {
@@ -567,9 +576,10 @@ impl ManagedRunHandle {
             more_pending_work: false,
             trying_to_evict: None,
             last_action_acked: true,
+            run_actions_tx,
             handle,
+            recorded_span_ids: Default::default(),
             metrics,
-            run_actions_tx,
         }
     }
 
@@ -770,6 +780,11 @@ impl OutstandingActivation {
 pub struct WorkflowTaskInfo {
     pub task_token: TaskToken,
     pub attempt: u32,
+    /// Exists to allow easy tagging of spans with workflow ids. Is duplicative of info inside the
+    /// run machines themselves, but that can't be accessed easily. Would be nice to somehow have a
+    /// shared repository, or refcounts, or whatever, for strings like these that get duped all
+    /// sorts of places.
+    pub wf_id: String,
 }
 
 #[derive(Debug)]
@@ -890,10 +905,12 @@ fn validate_completion(
                 )
             {
                 return Err(CompleteWfError::MalformedWorkflowCompletion {
-                    reason: "Workflow completion had a legacy query response along with other \
-                                commands. This is not allowed and constitutes an error in the \
-                                lang SDK"
-                        .to_owned(),
+                    reason: format!(
+                        "Workflow completion had a legacy query response along with other \
+                         commands. This is not allowed and constitutes an error in the \
+                         lang SDK. Commands: {:?}",
+                        commands
+                    ),
                     run_id: completion.run_id,
                 });
             }
@@ -991,6 +1008,14 @@ enum RunUpdateResponseKind {
     Good(GoodRunUpdate),
     Fail(FailRunUpdate),
 }
+impl RunUpdateResponseKind {
+    pub(crate) fn run_id(&self) -> &str {
+        match self {
+            RunUpdateResponseKind::Good(g) => &g.run_id,
+            RunUpdateResponseKind::Fail(f) => &f.run_id,
+        }
+    }
+}
 
 #[derive(Debug)]
 struct GoodRunUpdate {
@@ -1076,6 +1101,7 @@ pub enum WFCommand {
     SignalExternalWorkflow(SignalExternalWorkflowExecution),
     CancelSignalWorkflow(CancelSignalWorkflow),
     UpsertSearchAttributes(UpsertWorkflowSearchAttributes),
+    ModifyWorkflowProperties(ModifyWorkflowProperties),
 }
 
 impl TryFrom<WorkflowCommand> for WFCommand {
@@ -1117,6 +1143,9 @@ impl TryFrom<WorkflowCommand> for WFCommand {
             workflow_command::Variant::UpsertWorkflowSearchAttributes(s) => {
                 Ok(Self::UpsertSearchAttributes(s))
             }
+            workflow_command::Variant::ModifyWorkflowProperties(s) => {
+                Ok(Self::ModifyWorkflowProperties(s))
+            }
         }
     }
 }
@@ -1144,3 +1173,28 @@ pub struct WorkflowStartedInfo {
 
 type LocalActivityRequestSink =
     Arc<dyn Fn(Vec<LocalActRequest>) -> Vec<LocalActivityResolution> + Send + Sync>;
+
+/// Wraps outgoing activation job protos with some internal details core might care about
+#[derive(Debug, derive_more::Display)]
+#[display(fmt = "{}", variant)]
+struct OutgoingJob {
+    variant: workflow_activation_job::Variant,
+    /// Since LA resolutions are not distinguished from non-LA resolutions as far as lang is
+    /// concerned, but core cares about that sometimes, attach that info here.
+    is_la_resolution: bool,
+}
+impl<WA: Into<workflow_activation_job::Variant>> From<WA> for OutgoingJob {
+    fn from(wa: WA) -> Self {
+        Self {
+            variant: wa.into(),
+            is_la_resolution: false,
+        }
+    }
+}
+impl From<OutgoingJob> for WorkflowActivationJob {
+    fn from(og: OutgoingJob) -> Self {
+        Self {
+            variant: Some(og.variant),
+        }
+    }
+}

package/sdk-core/core/src/worker/workflow/wft_poller.rs

@@ -69,7 +69,7 @@ mod tests {
             .times(1)
             .returning(|| Some(Ok(PollWorkflowTaskQueueResponse::default())));
         mock_poller.expect_poll().times(1).returning(|| None);
-        let stream = new_wft_poller(Box::new(mock_poller), Default::default());
+        let stream = new_wft_poller(Box::new(mock_poller), MetricsContext::no_op());
         pin_mut!(stream);
         assert_matches!(stream.next().await, None);
     }
@@ -81,7 +81,7 @@ mod tests {
             .expect_poll()
             .times(1)
             .returning(|| Some(Err(tonic::Status::internal("ahhh"))));
-        let stream = new_wft_poller(Box::new(mock_poller), Default::default());
+        let stream = new_wft_poller(Box::new(mock_poller), MetricsContext::no_op());
         pin_mut!(stream);
         assert_matches!(stream.next().await, Some(Err(_)));
     }

package/sdk-core/core/src/worker/workflow/workflow_stream.rs

@@ -41,9 +41,27 @@ pub(crate) struct WFStream {
     /// Ensures we stay at or below this worker's maximum concurrent workflow task limit
     wft_semaphore: MeteredSemaphore,
     shutdown_token: CancellationToken,
+    ignore_evicts_on_shutdown: bool,
 
     metrics: MetricsContext,
 }
+impl WFStream {
+    fn record_span_fields(&mut self, run_id: &str, span: &Span) {
+        if let Some(run_handle) = self.runs.get_mut(run_id) {
+            if let Some(spid) = span.id() {
+                if run_handle.recorded_span_ids.contains(&spid) {
+                    return;
+                }
+                run_handle.recorded_span_ids.insert(spid);
+
+                if let Some(wid) = run_handle.wft.as_ref().map(|wft| &wft.info.wf_id) {
+                    span.record("workflow_id", wid.as_str());
+                }
+            }
+        }
+    }
+}
+
 /// All possible inputs to the [WFStream]
 #[derive(derive_more::From, Debug)]
 enum WFStreamInput {
@@ -81,6 +99,18 @@ pub(super) enum LocalInputs {
     RequestEviction(RequestEvictMsg),
     GetStateInfo(GetStateInfoMsg),
 }
+impl LocalInputs {
+    fn run_id(&self) -> Option<&str> {
+        Some(match self {
+            LocalInputs::Completion(c) => c.completion.run_id(),
+            LocalInputs::LocalResolution(lr) => &lr.run_id,
+            LocalInputs::PostActivation(pa) => &pa.run_id,
+            LocalInputs::RunUpdateResponse(rur) => rur.run_id(),
+            LocalInputs::RequestEviction(re) => &re.run_id,
+            LocalInputs::GetStateInfo(_) => return None,
+        })
+    }
+}
 #[derive(Debug, derive_more::From)]
 #[allow(clippy::large_enum_variant)] // PollerDead only ever gets used once, so not important.
 enum ExternalPollerInputs {
@@ -130,10 +160,9 @@ impl WFStream {
             MetricsContext::available_task_slots,
         );
         let wft_sem_clone = wft_semaphore.clone();
-        let proceeder = move || {
-            let wft_sem_clone = wft_sem_clone.clone();
-            async move { wft_sem_clone.acquire_owned().await.unwrap() }
-        };
+        let proceeder = stream::unfold(wft_sem_clone, |sem| async move {
+            Some((sem.acquire_owned().await.unwrap(), sem))
+        });
         let poller_wfts = stream_when_allowed(external_wfts, proceeder);
         let (run_update_tx, run_update_rx) = unbounded_channel();
         let local_rx = stream::select(
@@ -165,6 +194,7 @@ impl WFStream {
             client,
             wft_semaphore,
             shutdown_token: basics.shutdown_token,
+            ignore_evicts_on_shutdown: basics.ignore_evicts_on_shutdown,
             metrics: basics.metrics,
         };
         all_inputs
@@ -180,6 +210,9 @@ impl WFStream {
                     }
                     WFStreamInput::Local(local_input) => {
                         let _span_g = local_input.span.enter();
+                        if let Some(rid) = local_input.input.run_id() {
+                            state.record_span_fields(rid, &local_input.span);
+                        }
                         match local_input.input {
                             LocalInputs::RunUpdateResponse(resp) => {
                                 state.process_run_update_response(resp)
@@ -281,7 +314,7 @@ impl WFStream {
                             // If there are in-poll queries, insert jobs for those queries into the
                             // activation, but only if we hit the cache. If we didn't, those queries
                             // will need to be dealt with once replay is over
-                            if !wft.pending_queries.is_empty() && wft.hit_cache {
+                            if wft.hit_cache {
                                 put_queries_in_act(&mut activation, wft);
                             }
                         }
@@ -376,8 +409,9 @@ impl WFStream {
         }
     }
 
-    #[instrument(level = "debug", skip(self, pwft), 
-                 fields(run_id=%pwft.wft.workflow_execution.run_id))]
+    #[instrument(skip(self, pwft),
+                 fields(run_id=%pwft.wft.workflow_execution.run_id,
+                        workflow_id=%pwft.wft.workflow_execution.workflow_id))]
     fn instantiate_or_update(&mut self, pwft: PermittedWFT) {
         let (mut work, permit) = if let Some(w) = self.buffer_resp_if_outstanding_work(pwft) {
             (w.wft, w.permit)
@@ -407,6 +441,7 @@ impl WFStream {
         let wft_info = WorkflowTaskInfo {
             attempt: work.attempt,
             task_token: work.task_token,
+            wf_id: work.workflow_execution.workflow_id.clone(),
         };
         let poll_resp_is_incremental = work
             .history
@@ -476,8 +511,6 @@ impl WFStream {
         })
     }
 
-    #[instrument(level = "debug", skip(self, complete),
-                 fields(run_id=%complete.completion.run_id()))]
     fn process_completion(&mut self, complete: WFActCompleteMsg) {
         match complete.completion {
             ValidatedCompletion::Success { run_id, commands } => {
@@ -691,7 +724,7 @@ impl WFStream {
                 EvictionRequestResult::EvictionAlreadyRequested(attempts)
             }
         } else {
-            warn!(run_id=%info.run_id, "Eviction requested for unknown run");
+            debug!(run_id=%info.run_id, "Eviction requested for unknown run");
             EvictionRequestResult::NotFound
         }
     }
@@ -875,7 +908,7 @@ impl WFStream {
         let all_runs_ready = self
             .runs
             .handles()
-            .all(|r| !r.has_any_pending_work(true, false));
+            .all(|r| !r.has_any_pending_work(self.ignore_evicts_on_shutdown, false));
         if self.shutdown_token.is_cancelled() && all_runs_ready {
             info!("Workflow shutdown is done");
             true
@@ -931,6 +964,18 @@ impl WFStream {
 
 /// Drains pending queries from the workflow task and appends them to the activation's jobs
 fn put_queries_in_act(act: &mut WorkflowActivation, wft: &mut OutstandingTask) {
+    // Nothing to do if there are no pending queries
+    if wft.pending_queries.is_empty() {
+        return;
+    }
+
+    let has_legacy = wft.has_pending_legacy_query();
+    // Cannot dispatch legacy query if there are any other jobs - which can happen if, ex, a local
+    // activity resolves while we've gotten a legacy query after heartbeating.
+    if has_legacy && !act.jobs.is_empty() {
+        return;
+    }
+
     debug!(queries=?wft.pending_queries, "Dispatching queries");
     let query_jobs = wft
         .pending_queries

package/sdk-core/core-api/Cargo.toml

@@ -13,14 +13,15 @@ categories = ["development-tools"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow = "1.0"
 async-trait = "0.1"
-derive_builder = "0.11"
-log = "0.4"
+derive_builder = "0.12"
 opentelemetry = "0.18"
 prost-types = "0.11"
+serde_json = "1.0"
 thiserror = "1.0"
 tonic = "0.8"
+tracing-core = "0.1"
+url = "2.3"
 
 [dependencies.temporal-sdk-core-protos]
 path = "../sdk-core-protos"

package/sdk-core/core-api/src/lib.rs

@@ -1,13 +1,11 @@
 pub mod errors;
+pub mod telemetry;
 pub mod worker;
 
 use crate::{
     errors::{CompleteActivityError, CompleteWfError, PollActivityError, PollWfError},
     worker::WorkerConfig,
 };
-use log::Level;
-use opentelemetry::metrics::Meter;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use temporal_sdk_core_protos::coresdk::{
     activity_task::ActivityTask, workflow_activation::WorkflowActivation,
     workflow_completion::WorkflowActivationCompletion, ActivityHeartbeat, ActivityTaskCompletion,
@@ -109,43 +107,3 @@ pub trait Worker: Send + Sync {
     /// This should be called only after [Worker::shutdown] has resolved.
     async fn finalize_shutdown(self);
 }
-
-/// Should be backed by a process-wide singleton who is responsible for telemetry and logging
-/// management.
-pub trait CoreTelemetry {
-    /// Core buffers logs that should be shuttled over to lang so that they may be rendered with
-    /// the user's desired logging library. Use this function to grab the most recent buffered logs
-    /// since the last time it was called. A fixed number of such logs are retained at maximum, with
-    /// the oldest being dropped when full.
-    ///
-    /// Returns the list of logs from oldest to newest. Returns an empty vec if the feature is not
-    /// configured.
-    fn fetch_buffered_logs(&self) -> Vec<CoreLog>;
-
-    /// If metrics gathering is enabled, returns the OTel meter for core telemetry, which can be
-    /// used to create metrics instruments, or passed to things that create/record metrics (ex:
-    /// clients).
-    fn get_metric_meter(&self) -> Option<&Meter>;
-}
-
-/// A log line (which ultimately came from a tracing event) exported from Core->Lang
-#[derive(Debug)]
-pub struct CoreLog {
-    /// Log message
-    pub message: String,
-    /// Time log was generated (not when it was exported to lang)
-    pub timestamp: SystemTime,
-    /// Message level
-    pub level: Level,
-    // KV pairs aren't meaningfully exposed yet to the log interface by tracing
-}
-
-impl CoreLog {
-    /// Return timestamp as ms since epoch
-    pub fn millis_since_epoch(&self) -> u128 {
-        self.timestamp
-            .duration_since(UNIX_EPOCH)
-            .unwrap_or(Duration::ZERO)
-            .as_millis()
-    }
-}

package/sdk-core/core-api/src/telemetry.rs

@@ -0,0 +1,147 @@
+use opentelemetry::metrics::Meter;
+use std::{
+    collections::HashMap,
+    net::SocketAddr,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+use tracing_core::Level;
+use url::Url;
+
+/// Each core runtime instance has a telemetry subsystem associated with it, this trait defines the
+/// operations that lang might want to perform on that telemetry after it's initialized.
+pub trait CoreTelemetry {
+    /// Each worker buffers logs that should be shuttled over to lang so that they may be rendered
+    /// with the user's desired logging library. Use this function to grab the most recent buffered
+    /// logs since the last time it was called. A fixed number of such logs are retained at maximum,
+    /// with the oldest being dropped when full.
+    ///
+    /// Returns the list of logs from oldest to newest. Returns an empty vec if the feature is not
+    /// configured.
+    fn fetch_buffered_logs(&self) -> Vec<CoreLog>;
+
+    /// If metrics gathering is enabled, returns the OTel meter for core telemetry, which can be
+    /// used to create metrics instruments, or passed to things that create/record metrics (ex:
+    /// clients).
+    fn get_metric_meter(&self) -> Option<&Meter>;
+}
+
+/// Telemetry configuration options. Construct with [TelemetryOptionsBuilder]
+#[derive(Debug, Clone, derive_builder::Builder)]
+#[non_exhaustive]
+pub struct TelemetryOptions {
+    /// Optional trace exporter - set as None to disable.
+    #[builder(setter(into, strip_option), default)]
+    pub tracing: Option<TraceExportConfig>,
+    /// Optional logger - set as None to disable.
+    #[builder(setter(into, strip_option), default)]
+    pub logging: Option<Logger>,
+    /// Optional metrics exporter - set as None to disable.
+    #[builder(setter(into, strip_option), default)]
+    pub metrics: Option<MetricsExporter>,
+
+    /// If set true, do not prefix metrics with `temporal_`. Will be removed eventually as
+    /// the prefix is consistent with other SDKs.
+    #[builder(default)]
+    pub no_temporal_prefix_for_metrics: bool,
+
+    /// Specifies the aggregation temporality for metric export. Defaults to cumulative.
+    #[builder(default = "MetricTemporality::Cumulative")]
+    pub metric_temporality: MetricTemporality,
+}
+
+/// Options for exporting to an OpenTelemetry Collector
+#[derive(Debug, Clone)]
+pub struct OtelCollectorOptions {
+    /// The url of the OTel collector to export telemetry and metrics to. Lang SDK should also
+    /// export to this same collector.
+    pub url: Url,
+    /// Optional set of HTTP headers to send to the Collector, e.g for authentication.
+    pub headers: HashMap<String, String>,
+    /// Optionally specify how frequently metrics should be exported. Defaults to 1 second.
+    pub metric_periodicity: Option<Duration>,
+}
+
+/// Configuration for the external export of traces
+#[derive(Debug, Clone)]
+pub struct TraceExportConfig {
+    /// An [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html) filter string.
+    pub filter: String,
+    /// Where they should go
+    pub exporter: TraceExporter,
+}
+
+/// Control where traces are exported.
+#[derive(Debug, Clone)]
+pub enum TraceExporter {
+    /// Export traces to an OpenTelemetry Collector <https://opentelemetry.io/docs/collector/>.
+    Otel(OtelCollectorOptions),
+}
+
+/// Control where metrics are exported
+#[derive(Debug, Clone)]
+pub enum MetricsExporter {
+    /// Export metrics to an OpenTelemetry Collector <https://opentelemetry.io/docs/collector/>.
+    Otel(OtelCollectorOptions),
+    /// Expose metrics directly via an embedded http server bound to the provided address.
+    Prometheus(SocketAddr),
+}
+
+/// Control where logs go
+#[derive(Debug, Clone)]
+pub enum Logger {
+    /// Log directly to console.
+    Console {
+        /// An [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html) filter string.
+        filter: String,
+    },
+    /// Forward logs to Lang - collectable with `fetch_global_buffered_logs`.
+    Forward {
+        /// An [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html) filter string.
+        filter: String,
+    },
+}
+
+/// Types of aggregation temporality for metric export.
+/// See: <https://github.com/open-telemetry/opentelemetry-specification/blob/ce50e4634efcba8da445cc23523243cb893905cb/specification/metrics/datamodel.md#temporality>
+#[derive(Debug, Clone, Copy)]
+pub enum MetricTemporality {
+    /// Successive data points repeat the starting timestamp
+    Cumulative,
+    /// Successive data points advance the starting timestamp
+    Delta,
+}
+
+impl Default for TelemetryOptions {
+    fn default() -> Self {
+        TelemetryOptionsBuilder::default().build().unwrap()
+    }
+}
+
+/// A log line (which ultimately came from a tracing event) exported from Core->Lang
+#[derive(Debug)]
+pub struct CoreLog {
+    /// The module within core this message originated from
+    pub target: String,
+    /// Log message
+    pub message: String,
+    /// Time log was generated (not when it was exported to lang)
+    pub timestamp: SystemTime,
+    /// Message level
+    pub level: Level,
+    /// Arbitrary k/v pairs (span k/vs are collapsed with event k/vs here). We could change this
+    /// to include them in `span_contexts` instead, but there's probably not much value for log
+    /// forwarding.
+    pub fields: HashMap<String, serde_json::Value>,
+    /// A list of the outermost to the innermost span names
+    pub span_contexts: Vec<String>,
+}
+
+impl CoreLog {
+    /// Return timestamp as ms since epoch
+    pub fn millis_since_epoch(&self) -> u128 {
+        self.timestamp
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or(Duration::ZERO)
+            .as_millis()
+    }
+}

package/sdk-core/core-api/src/worker.rs

@@ -95,6 +95,16 @@ pub struct WorkerConfig {
     /// todo: link to feature docs
     #[builder(default = "false")]
     pub use_worker_versioning: bool,
+
+    /// If set false (default), shutdown will not finish until all pending evictions have been
+    /// issued and replied to. If set true shutdown will be considered complete when the only
+    /// remaining work is pending evictions.
+    ///
+    /// This flag is useful during tests to avoid needing to deal with lots of uninteresting
+    /// evictions during shutdown. Alternatively, if a lang implementation finds it easy to clean
+    /// up during shutdown, setting this true saves some back-and-forth.
+    #[builder(default = "false")]
+    pub ignore_evicts_on_shutdown: bool,
 }
 
 impl WorkerConfig {
@@ -114,6 +124,9 @@ impl WorkerConfigBuilder {
         if self.max_concurrent_wft_polls == Some(0) {
             return Err("`max_concurrent_wft_polls` must be at least 1".to_owned());
         }
+        if self.max_concurrent_at_polls == Some(0) {
+            return Err("`max_concurrent_at_polls` must be at least 1".to_owned());
+        }
         if self.max_cached_workflows > Some(0)
             && self.max_outstanding_workflow_tasks > self.max_cached_workflows
         {