@temporalio/core-bridge 1.4.3 → 1.5.0

package/sdk-core/test-utils/Cargo.toml

@@ -14,6 +14,7 @@ async-trait = "0.1"
 base64 = "0.13"
 futures = "0.3"
 log = "0.4"
+once_cell = "1.16"
 parking_lot = "0.12"
 prost = "0.11"
 prost-types = "0.11"

package/sdk-core/test-utils/src/lib.rs

@@ -6,7 +6,7 @@ extern crate tracing;
 
 pub mod canned_histories;
 
-use crate::stream::TryStreamExt;
+use crate::stream::{Stream, TryStreamExt};
 use futures::{future, stream, stream::FuturesUnordered, StreamExt};
 use parking_lot::Mutex;
 use prost::Message;
@@ -18,14 +18,23 @@ use std::{
 use temporal_client::{
     Client, RetryClient, WorkflowClientTrait, WorkflowExecutionInfo, WorkflowOptions,
 };
-use temporal_sdk::{interceptors::WorkerInterceptor, IntoActivityFunc, Worker, WorkflowFunction};
+use temporal_sdk::{
+    interceptors::{FailOnNondeterminismInterceptor, WorkerInterceptor},
+    IntoActivityFunc, Worker, WorkflowFunction,
+};
 use temporal_sdk_core::{
     ephemeral_server::{EphemeralExe, EphemeralExeVersion},
-    init_replay_worker, init_worker, telemetry_init, ClientOptions, ClientOptionsBuilder, Logger,
-    MetricsExporter, OtelCollectorOptions, TelemetryOptions, TelemetryOptionsBuilder,
-    TraceExporter, WorkerConfig, WorkerConfigBuilder,
+    init_replay_worker, init_worker,
+    replay::HistoryForReplay,
+    ClientOptions, ClientOptionsBuilder, CoreRuntime, WorkerConfig, WorkerConfigBuilder,
+};
+use temporal_sdk_core_api::{
+    telemetry::{
+        Logger, MetricsExporter, OtelCollectorOptions, TelemetryOptions, TelemetryOptionsBuilder,
+        TraceExportConfig, TraceExporter,
+    },
+    Worker as CoreWorker,
 };
-use temporal_sdk_core_api::Worker as CoreWorker;
 use temporal_sdk_core_protos::{
     coresdk::{
         workflow_commands::{
@@ -61,21 +70,43 @@ pub async fn init_core_and_create_wf(test_name: &str) -> CoreWfStarter {
     starter
 }
 
-/// Create a worker replay instance preloaded with a provided history. Returns the worker impl
-/// and the task queue name as in [init_core_and_create_wf].
-pub fn init_core_replay_preloaded(
-    test_name: &str,
-    history: &History,
-) -> (Arc<dyn CoreWorker>, String) {
-    telemetry_init(&get_integ_telem_options()).expect("Telemetry inits cleanly");
+/// Create a worker replay instance preloaded with provided histories. Returns the worker impl.
+pub fn init_core_replay_preloaded<I>(test_name: &str, histories: I) -> Arc<dyn CoreWorker>
+where
+    I: IntoIterator<Item = HistoryForReplay> + 'static,
+    <I as IntoIterator>::IntoIter: Send,
+{
+    init_core_replay_stream(test_name, stream::iter(histories))
+}
+pub fn init_core_replay_stream<I>(test_name: &str, histories: I) -> Arc<dyn CoreWorker>
+where
+    I: Stream<Item = HistoryForReplay> + Send + 'static,
+{
     let worker_cfg = WorkerConfigBuilder::default()
         .namespace(NAMESPACE)
         .task_queue(test_name)
         .worker_build_id("test_bin_id")
         .build()
         .expect("Configuration options construct properly");
-    let worker = init_replay_worker(worker_cfg, history).expect("Replay worker must init properly");
-    (Arc::new(worker), test_name.to_string())
+    let worker =
+        init_replay_worker(worker_cfg, histories).expect("Replay worker must init properly");
+    Arc::new(worker)
+}
+pub fn replay_sdk_worker<I>(histories: I) -> Worker
+where
+    I: IntoIterator<Item = HistoryForReplay> + 'static,
+    <I as IntoIterator>::IntoIter: Send,
+{
+    replay_sdk_worker_stream(stream::iter(histories))
+}
+pub fn replay_sdk_worker_stream<I>(histories: I) -> Worker
+where
+    I: Stream<Item = HistoryForReplay> + Send + 'static,
+{
+    let core = init_core_replay_stream("replay_worker_test", histories);
+    let mut worker = Worker::new_from_core(core, "replay_q".to_string());
+    worker.set_worker_interceptor(Box::new(FailOnNondeterminismInterceptor {}));
+    worker
 }
 
 /// Load history from a file containing the protobuf serialization of it
@@ -87,11 +118,21 @@ pub async fn history_from_proto_binary(path_from_root: &str) -> Result<History,
     Ok(History::decode(&*bytes)?)
 }
 
+static INTEG_TESTS_RT: once_cell::sync::OnceCell<CoreRuntime> = once_cell::sync::OnceCell::new();
+pub fn init_integ_telem() {
+    INTEG_TESTS_RT.get_or_init(|| {
+        let telemetry_options = get_integ_telem_options();
+        let rt =
+            CoreRuntime::new_assume_tokio(telemetry_options).expect("Core runtime inits cleanly");
+        let _ = tracing::subscriber::set_global_default(rt.trace_subscriber());
+        rt
+    });
+}
+
 /// Implements a builder pattern to help integ tests initialize core and create workflows
 pub struct CoreWfStarter {
     /// Used for both the task queue and workflow id
     task_queue_name: String,
-    telemetry_options: TelemetryOptions,
     pub worker_config: WorkerConfig,
     wft_timeout: Option<Duration>,
     initted_worker: OnceCell<InitializedWorker>,
@@ -110,9 +151,9 @@ impl CoreWfStarter {
     }
 
     pub fn new_tq_name(task_queue: &str) -> Self {
+        init_integ_telem();
         Self {
             task_queue_name: task_queue.to_owned(),
-            telemetry_options: get_integ_telem_options(),
             worker_config: WorkerConfigBuilder::default()
                 .namespace(NAMESPACE)
                 .task_queue(task_queue)
@@ -181,19 +222,21 @@ impl CoreWfStarter {
         &mut self,
         wf_id: impl Into<String>,
         run_id: impl Into<String>,
-        // TODO: Need not be passed in
         worker: &mut Worker,
     ) -> Result<(), anyhow::Error> {
+        let wf_id = wf_id.into();
         // Fetch history and replay it
         let history = self
             .get_client()
             .await
-            .get_workflow_execution_history(wf_id.into(), Some(run_id.into()), vec![])
+            .get_workflow_execution_history(wf_id.clone(), Some(run_id.into()), vec![])
             .await?
             .history
             .expect("history field must be populated");
-        let (replay_worker, _) = init_core_replay_preloaded(worker.task_queue(), &history);
+        let with_id = HistoryForReplay::new(history, wf_id);
+        let replay_worker = init_core_replay_preloaded(worker.task_queue(), [with_id]);
         worker.with_new_core_worker(replay_worker);
+        worker.set_worker_interceptor(Box::new(FailOnNondeterminismInterceptor {}));
         worker.run().await.unwrap();
         Ok(())
     }
@@ -239,14 +282,18 @@ impl CoreWfStarter {
     async fn get_or_init(&mut self) -> &InitializedWorker {
         self.initted_worker
             .get_or_init(|| async {
-                telemetry_init(&self.telemetry_options).expect("Telemetry inits cleanly");
                 let client = Arc::new(
                     get_integ_server_options()
                         .connect(self.worker_config.namespace.clone(), None, None)
                         .await
                         .expect("Must connect"),
                 );
-                let worker = init_worker(self.worker_config.clone(), client.clone());
+                let worker = init_worker(
+                    INTEG_TESTS_RT.get().unwrap(),
+                    self.worker_config.clone(),
+                    client.clone(),
+                )
+                .expect("Worker inits cleanly");
                 InitializedWorker {
                     worker: Arc::new(worker),
                     client,
@@ -442,7 +489,6 @@ impl WorkerInterceptor for TestWorkerCompletionIceptor {
 
 /// Returns the client options used to connect to the server used for integration tests.
 pub fn get_integ_server_options() -> ClientOptions {
-    telemetry_init(&get_integ_telem_options()).expect("Telemetry inits cleanly");
     let temporal_server_address = match env::var(INTEG_SERVER_TARGET_ENV_VAR) {
         Ok(addr) => addr,
         Err(_) => "http://localhost:7233".to_owned(),
@@ -459,6 +505,8 @@ pub fn get_integ_server_options() -> ClientOptions {
 
 pub fn get_integ_telem_options() -> TelemetryOptions {
     let mut ob = TelemetryOptionsBuilder::default();
+    let filter_string =
+        env::var("RUST_LOG").unwrap_or_else(|_| "temporal_sdk_core=INFO".to_string());
     if let Some(url) = env::var(OTEL_URL_ENV_VAR)
         .ok()
         .map(|x| x.parse::<Url>().unwrap())
@@ -466,8 +514,12 @@ pub fn get_integ_telem_options() -> TelemetryOptions {
         let opts = OtelCollectorOptions {
             url,
             headers: Default::default(),
+            metric_periodicity: None,
         };
-        ob.tracing(TraceExporter::Otel(opts.clone()));
+        ob.tracing(TraceExportConfig {
+            filter: filter_string.clone(),
+            exporter: TraceExporter::Otel(opts.clone()),
+        });
         ob.metrics(MetricsExporter::Otel(opts));
     }
     if let Some(addr) = env::var(PROM_ENABLE_ENV_VAR)
@@ -476,10 +528,11 @@ pub fn get_integ_telem_options() -> TelemetryOptions {
     {
         ob.metrics(MetricsExporter::Prometheus(addr));
     }
-    ob.tracing_filter(env::var("RUST_LOG").unwrap_or_else(|_| "temporal_sdk_core=INFO".to_string()))
-        .logging(Logger::Console)
-        .build()
-        .unwrap()
+    ob.logging(Logger::Console {
+        filter: filter_string,
+    })
+    .build()
+    .unwrap()
 }
 
 pub fn default_cached_download() -> EphemeralExe {
@@ -504,7 +557,6 @@ pub fn schedule_activity_cmd(
         seq,
         activity_id: activity_id.to_string(),
         activity_type: "test_activity".to_string(),
-        namespace: NAMESPACE.to_owned(),
         task_queue: task_q.to_owned(),
         schedule_to_start_timeout: Some(activity_timeout.try_into().expect("duration fits")),
         start_to_close_timeout: Some(activity_timeout.try_into().expect("duration fits")),

package/sdk-core/tests/integ_tests/metrics_tests.rs

@@ -0,0 +1,37 @@
+use temporal_client::WorkflowService;
+use temporal_sdk_core::CoreRuntime;
+use temporal_sdk_core_api::telemetry::MetricsExporter;
+use temporal_sdk_core_protos::temporal::api::workflowservice::v1::ListNamespacesRequest;
+use temporal_sdk_core_test_utils::{get_integ_server_options, get_integ_telem_options};
+
+#[tokio::test]
+async fn prometheus_metrics_exported() {
+    let mut telemopts = get_integ_telem_options();
+    let addr = "127.0.0.1:10919";
+    telemopts.metrics = Some(MetricsExporter::Prometheus(addr.parse().unwrap()));
+    let rt = CoreRuntime::new_assume_tokio(telemopts).unwrap();
+    let opts = get_integ_server_options();
+    let mut raw_client = opts
+        .connect_no_namespace(rt.metric_meter(), None)
+        .await
+        .unwrap();
+    assert!(raw_client.get_client().capabilities().is_some());
+
+    let _ = raw_client
+        .list_namespaces(ListNamespacesRequest::default())
+        .await
+        .unwrap();
+
+    let body = reqwest::get(format!("http://{}/metrics", addr))
+        .await
+        .unwrap()
+        .text()
+        .await
+        .unwrap();
+    assert!(body.contains(
+        "request_latency_count{operation=\"ListNamespaces\",service_name=\"temporal-core-sdk\"} 1"
+    ));
+    assert!(body.contains(
+        "request_latency_count{operation=\"GetSystemInfo\",service_name=\"temporal-core-sdk\"} 1"
+    ));
+}

package/sdk-core/tests/integ_tests/polling_tests.rs

@@ -3,7 +3,6 @@ use futures::future::join_all;
 use std::time::Duration;
 use temporal_client::WorkflowOptions;
 use temporal_sdk::{WfContext, WorkflowResult};
-use temporal_sdk_core_api::errors::PollWfError;
 use temporal_sdk_core_protos::coresdk::{
     activity_task::activity_task as act_task,
     workflow_activation::{workflow_activation_job, FireTimer, WorkflowActivationJob},
@@ -132,15 +131,3 @@ async fn can_paginate_long_history() {
         .unwrap();
     worker.run_until_done().await.unwrap();
 }
-
-// TODO: Takes ages now, fix somehow
-#[tokio::test]
-async fn poll_of_nonexistent_namespace_is_fatal() {
-    let mut starter = CoreWfStarter::new("whatever_yo");
-    starter.worker_config.namespace = "I do not exist".to_string();
-    let worker = starter.get_worker().await;
-    assert_matches!(
-        worker.poll_workflow_activation().await,
-        Err(PollWfError::TonicError(_))
-    );
-}

package/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs

@@ -1,19 +1,24 @@
 use anyhow::anyhow;
 use futures::future::join_all;
+use futures_util::stream::{FuturesUnordered, StreamExt};
 use std::time::Duration;
-use temporal_client::WorkflowOptions;
+use temporal_client::{WorkflowClientTrait, WorkflowOptions};
 use temporal_sdk::{
-    interceptors::WorkerInterceptor, ActContext, ActivityCancelledError, CancellableFuture,
-    LocalActivityOptions, WfContext, WorkflowResult,
+    interceptors::WorkerInterceptor, ActContext, ActivityCancelledError, ActivityOptions,
+    CancellableFuture, LocalActivityOptions, WfContext, WorkflowResult,
 };
+use temporal_sdk_core::replay::HistoryForReplay;
 use temporal_sdk_core_protos::{
     coresdk::{
         workflow_commands::ActivityCancellationType,
         workflow_completion::WorkflowActivationCompletion, AsJsonPayloadExt,
     },
     temporal::api::common::v1::RetryPolicy,
+    TestHistoryBuilder,
+};
+use temporal_sdk_core_test_utils::{
+    history_from_proto_binary, init_integ_telem, replay_sdk_worker, CoreWfStarter,
 };
-use temporal_sdk_core_test_utils::CoreWfStarter;
 use tokio_util::sync::CancellationToken;
 
 pub async fn echo(_ctx: ActContext, e: String) -> anyhow::Result<String> {
@@ -632,3 +637,139 @@ async fn repro_nondeterminism_with_timer_bug() {
         .await
         .unwrap();
 }
+
+async fn la_problem_workflow(ctx: WfContext) -> WorkflowResult<()> {
+    ctx.local_activity(LocalActivityOptions {
+        activity_type: "delay".to_string(),
+        input: "hi".as_json_payload().expect("serializes fine"),
+        retry_policy: RetryPolicy {
+            initial_interval: Some(prost_dur!(from_micros(15))),
+            backoff_coefficient: 1_000.,
+            maximum_interval: Some(prost_dur!(from_millis(1500))),
+            maximum_attempts: 4,
+            non_retryable_error_types: vec![],
+        },
+        timer_backoff_threshold: Some(Duration::from_secs(1)),
+        ..Default::default()
+    })
+    .await;
+    ctx.activity(ActivityOptions {
+        activity_type: "delay".to_string(),
+        start_to_close_timeout: Some(Duration::from_secs(20)),
+        input: "hi!".as_json_payload().expect("serializes fine"),
+        ..Default::default()
+    })
+    .await;
+    Ok(().into())
+}
+
+// Expensive to run - worth enabling on a stress/regression pipeline.
+#[ignore]
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn evict_while_la_running_no_interference() {
+    let wf_name = "evict_while_la_running_no_interference";
+    let mut starter = CoreWfStarter::new(wf_name);
+    starter.max_local_at(20);
+    starter.max_cached_workflows(20);
+    let mut worker = starter.worker().await;
+
+    worker.register_wf(wf_name.to_owned(), la_problem_workflow);
+    worker.register_activity("delay", |_: ActContext, _: String| async {
+        tokio::time::sleep(Duration::from_secs(15)).await;
+        Ok(())
+    });
+
+    let client = starter.get_client().await;
+    let subfs = FuturesUnordered::new();
+    for i in 1..100 {
+        let wf_id = format!("{}-{}", wf_name, i);
+        let run_id = worker
+            .submit_wf(
+                &wf_id,
+                wf_name.to_owned(),
+                vec![],
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+        let cw = worker.core_worker.clone();
+        let client = client.clone();
+        subfs.push(async move {
+            // Evict the workflow
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            cw.request_workflow_eviction(&run_id);
+            // Wake up workflow by sending signal
+            client
+                .signal_workflow_execution(
+                    wf_id,
+                    run_id.clone(),
+                    "whaatever".to_string(),
+                    None,
+                    None,
+                )
+                .await
+                .unwrap();
+        });
+    }
+    let runf = async {
+        worker.run_until_done().await.unwrap();
+    };
+    tokio::join!(subfs.collect::<Vec<_>>(), runf);
+}
+
+#[rstest::rstest]
+#[tokio::test]
+async fn weird_la_nondeterminism_repro(#[values(true, false)] fix_hist: bool) {
+    init_integ_telem();
+    let mut hist = history_from_proto_binary(
+        "histories/evict_while_la_running_no_interference-85_history.bin",
+    )
+    .await
+    .unwrap();
+    if fix_hist {
+        // Replace broken ending with accurate ending
+        hist.events.truncate(20);
+        let mut thb = TestHistoryBuilder::from_history(hist.events);
+        thb.add_workflow_task_completed();
+        thb.add_workflow_execution_completed();
+        hist = thb.get_full_history_info().unwrap().into();
+    }
+
+    let mut worker = replay_sdk_worker([HistoryForReplay::new(hist, "fake".to_owned())]);
+    worker.register_wf(
+        "evict_while_la_running_no_interference",
+        la_problem_workflow,
+    );
+    worker.register_activity("delay", |_: ActContext, _: String| async {
+        tokio::time::sleep(Duration::from_secs(15)).await;
+        Ok(())
+    });
+    worker.run().await.unwrap();
+}
+
+#[tokio::test]
+async fn second_weird_la_nondeterminism_repro() {
+    init_integ_telem();
+    let mut hist = history_from_proto_binary(
+        "histories/evict_while_la_running_no_interference-23_history.bin",
+    )
+    .await
+    .unwrap();
+    // Chop off uninteresting ending
+    hist.events.truncate(24);
+    let mut thb = TestHistoryBuilder::from_history(hist.events);
+    // thb.add_workflow_task_completed();
+    thb.add_workflow_execution_completed();
+    hist = thb.get_full_history_info().unwrap().into();
+
+    let mut worker = replay_sdk_worker([HistoryForReplay::new(hist, "fake".to_owned())]);
+    worker.register_wf(
+        "evict_while_la_running_no_interference",
+        la_problem_workflow,
+    );
+    worker.register_activity("delay", |_: ActContext, _: String| async {
+        tokio::time::sleep(Duration::from_secs(15)).await;
+        Ok(())
+    });
+    worker.run().await.unwrap();
+}

package/sdk-core/tests/integ_tests/workflow_tests/modify_wf_properties.rs

@@ -0,0 +1,53 @@
+use temporal_client::WorkflowClientTrait;
+use temporal_sdk::{WfContext, WorkflowResult};
+use temporal_sdk_core_protos::coresdk::{AsJsonPayloadExt, FromJsonPayloadExt};
+use temporal_sdk_core_test_utils::CoreWfStarter;
+use uuid::Uuid;
+
+static FIELD_A: &str = "cat_name";
+static FIELD_B: &str = "cute_level";
+
+async fn memo_upserter(ctx: WfContext) -> WorkflowResult<()> {
+    ctx.upsert_memo([
+        (FIELD_A.to_string(), "enchi".as_json_payload().unwrap()),
+        (FIELD_B.to_string(), 9001.as_json_payload().unwrap()),
+    ]);
+    Ok(().into())
+}
+
+#[tokio::test]
+async fn sends_modify_wf_props() {
+    let wf_name = "can_upsert_memo";
+    let wf_id = Uuid::new_v4();
+    let mut starter = CoreWfStarter::new(wf_name);
+    let mut worker = starter.worker().await;
+
+    worker.register_wf(wf_name, memo_upserter);
+    let run_id = worker
+        .submit_wf(wf_id.to_string(), wf_name, vec![], Default::default())
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+
+    let memo = starter
+        .get_client()
+        .await
+        .describe_workflow_execution(wf_id.to_string(), Some(run_id))
+        .await
+        .unwrap()
+        .workflow_execution_info
+        .unwrap()
+        .memo
+        .unwrap()
+        .fields;
+    let catname = memo.get(FIELD_A).unwrap();
+    let cuteness = memo.get(FIELD_B).unwrap();
+    for payload in [catname, cuteness] {
+        assert_eq!(
+            &b"json/plain".to_vec(),
+            payload.metadata.get("encoding").unwrap()
+        );
+    }
+    assert_eq!("enchi", String::from_json_payload(catname).unwrap());
+    assert_eq!(9001, usize::from_json_payload(cuteness).unwrap());
+}