temporalio 0.0.1 → 0.0.2

data/bridge/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs

@@ -1,19 +1,27 @@
 use anyhow::anyhow;
 use futures::future::join_all;
-use std::time::Duration;
-use temporal_client::WorkflowOptions;
+use futures_util::stream::{FuturesUnordered, StreamExt};
+use std::{
+    sync::atomic::{AtomicU8, Ordering},
+    time::Duration,
+};
+use temporal_client::{WorkflowClientTrait, WorkflowOptions};
 use temporal_sdk::{
-    interceptors::WorkerInterceptor, ActContext, ActivityCancelledError, CancellableFuture,
-    LocalActivityOptions, WfContext, WorkflowResult,
+    interceptors::WorkerInterceptor, ActContext, ActivityCancelledError, ActivityOptions,
+    CancellableFuture, LocalActivityOptions, WfContext, WorkflowResult,
 };
+use temporal_sdk_core::replay::HistoryForReplay;
 use temporal_sdk_core_protos::{
     coresdk::{
         workflow_commands::ActivityCancellationType,
         workflow_completion::WorkflowActivationCompletion, AsJsonPayloadExt,
     },
-    temporal::api::common::v1::RetryPolicy,
+    temporal::api::{common::v1::RetryPolicy, enums::v1::TimeoutType},
+    TestHistoryBuilder,
+};
+use temporal_sdk_core_test_utils::{
+    history_from_proto_binary, init_integ_telem, replay_sdk_worker, CoreWfStarter,
 };
-use temporal_sdk_core_test_utils::CoreWfStarter;
 use tokio_util::sync::CancellationToken;
 
 pub async fn echo(_ctx: ActContext, e: String) -> anyhow::Result<String> {
@@ -403,6 +411,11 @@ async fn x_to_close_timeout(#[case] is_schedule: bool) {
     } else {
         (None, Some(Duration::from_secs(2)))
     };
+    let timeout_type = if is_schedule {
+        TimeoutType::ScheduleToClose
+    } else {
+        TimeoutType::StartToClose
+    };
 
     worker.register_wf(wf_name.to_owned(), move |ctx: WfContext| async move {
         let res = ctx
@@ -422,7 +435,7 @@ async fn x_to_close_timeout(#[case] is_schedule: bool) {
                 ..Default::default()
             })
             .await;
-        assert!(res.timed_out());
+        assert_eq!(res.timed_out(), Some(timeout_type));
         Ok(().into())
     });
     worker.register_activity("echo", |ctx: ActContext, _: String| async move {
@@ -467,21 +480,23 @@ async fn schedule_to_close_timeout_across_timer_backoff(#[case] cached: bool) {
                 activity_type: "echo".to_string(),
                 input: "hi".as_json_payload().expect("serializes fine"),
                 retry_policy: RetryPolicy {
-                    initial_interval: Some(prost_dur!(from_micros(15))),
+                    initial_interval: Some(prost_dur!(from_millis(15))),
                     backoff_coefficient: 1_000.,
-                    maximum_interval: Some(prost_dur!(from_millis(1500))),
+                    maximum_interval: Some(prost_dur!(from_millis(1000))),
                     maximum_attempts: 40,
                     non_retryable_error_types: vec![],
                 },
-                timer_backoff_threshold: Some(Duration::from_secs(1)),
-                schedule_to_close_timeout: Some(Duration::from_secs(3)),
+                timer_backoff_threshold: Some(Duration::from_millis(500)),
+                schedule_to_close_timeout: Some(Duration::from_secs(2)),
                 ..Default::default()
             })
             .await;
-        assert!(res.timed_out());
+        assert_eq!(res.timed_out(), Some(TimeoutType::ScheduleToClose));
         Ok(().into())
     });
-    worker.register_activity("echo", |_: ActContext, _: String| async {
+    let num_attempts: &'static _ = Box::leak(Box::new(AtomicU8::new(0)));
+    worker.register_activity("echo", move |_: ActContext, _: String| async {
+        num_attempts.fetch_add(1, Ordering::Relaxed);
         Result::<(), _>::Err(anyhow!("Oh no I failed!"))
     });
 
@@ -495,6 +510,9 @@ async fn schedule_to_close_timeout_across_timer_backoff(#[case] cached: bool) {
         .await
         .unwrap();
     worker.run_until_done().await.unwrap();
+    // 3 attempts b/c first backoff is very small, then the next 2 attempts take at least 2 seconds
+    // b/c of timer backoff.
+    assert_eq!(3, num_attempts.load(Ordering::Relaxed));
 }
 
 #[rstest::rstest]
@@ -632,3 +650,139 @@ async fn repro_nondeterminism_with_timer_bug() {
         .await
         .unwrap();
 }
+
+async fn la_problem_workflow(ctx: WfContext) -> WorkflowResult<()> {
+    ctx.local_activity(LocalActivityOptions {
+        activity_type: "delay".to_string(),
+        input: "hi".as_json_payload().expect("serializes fine"),
+        retry_policy: RetryPolicy {
+            initial_interval: Some(prost_dur!(from_micros(15))),
+            backoff_coefficient: 1_000.,
+            maximum_interval: Some(prost_dur!(from_millis(1500))),
+            maximum_attempts: 4,
+            non_retryable_error_types: vec![],
+        },
+        timer_backoff_threshold: Some(Duration::from_secs(1)),
+        ..Default::default()
+    })
+    .await;
+    ctx.activity(ActivityOptions {
+        activity_type: "delay".to_string(),
+        start_to_close_timeout: Some(Duration::from_secs(20)),
+        input: "hi!".as_json_payload().expect("serializes fine"),
+        ..Default::default()
+    })
+    .await;
+    Ok(().into())
+}
+
+// Expensive to run - worth enabling on a stress/regression pipeline.
+#[ignore]
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn evict_while_la_running_no_interference() {
+    let wf_name = "evict_while_la_running_no_interference";
+    let mut starter = CoreWfStarter::new(wf_name);
+    starter.max_local_at(20);
+    starter.max_cached_workflows(20);
+    let mut worker = starter.worker().await;
+
+    worker.register_wf(wf_name.to_owned(), la_problem_workflow);
+    worker.register_activity("delay", |_: ActContext, _: String| async {
+        tokio::time::sleep(Duration::from_secs(15)).await;
+        Ok(())
+    });
+
+    let client = starter.get_client().await;
+    let subfs = FuturesUnordered::new();
+    for i in 1..100 {
+        let wf_id = format!("{}-{}", wf_name, i);
+        let run_id = worker
+            .submit_wf(
+                &wf_id,
+                wf_name.to_owned(),
+                vec![],
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+        let cw = worker.core_worker.clone();
+        let client = client.clone();
+        subfs.push(async move {
+            // Evict the workflow
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            cw.request_workflow_eviction(&run_id);
+            // Wake up workflow by sending signal
+            client
+                .signal_workflow_execution(
+                    wf_id,
+                    run_id.clone(),
+                    "whaatever".to_string(),
+                    None,
+                    None,
+                )
+                .await
+                .unwrap();
+        });
+    }
+    let runf = async {
+        worker.run_until_done().await.unwrap();
+    };
+    tokio::join!(subfs.collect::<Vec<_>>(), runf);
+}
+
+#[rstest::rstest]
+#[tokio::test]
+async fn weird_la_nondeterminism_repro(#[values(true, false)] fix_hist: bool) {
+    init_integ_telem();
+    let mut hist = history_from_proto_binary(
+        "histories/evict_while_la_running_no_interference-85_history.bin",
+    )
+    .await
+    .unwrap();
+    if fix_hist {
+        // Replace broken ending with accurate ending
+        hist.events.truncate(20);
+        let mut thb = TestHistoryBuilder::from_history(hist.events);
+        thb.add_workflow_task_completed();
+        thb.add_workflow_execution_completed();
+        hist = thb.get_full_history_info().unwrap().into();
+    }
+
+    let mut worker = replay_sdk_worker([HistoryForReplay::new(hist, "fake".to_owned())]);
+    worker.register_wf(
+        "evict_while_la_running_no_interference",
+        la_problem_workflow,
+    );
+    worker.register_activity("delay", |_: ActContext, _: String| async {
+        tokio::time::sleep(Duration::from_secs(15)).await;
+        Ok(())
+    });
+    worker.run().await.unwrap();
+}
+
+#[tokio::test]
+async fn second_weird_la_nondeterminism_repro() {
+    init_integ_telem();
+    let mut hist = history_from_proto_binary(
+        "histories/evict_while_la_running_no_interference-23_history.bin",
+    )
+    .await
+    .unwrap();
+    // Chop off uninteresting ending
+    hist.events.truncate(24);
+    let mut thb = TestHistoryBuilder::from_history(hist.events);
+    // thb.add_workflow_task_completed();
+    thb.add_workflow_execution_completed();
+    hist = thb.get_full_history_info().unwrap().into();
+
+    let mut worker = replay_sdk_worker([HistoryForReplay::new(hist, "fake".to_owned())]);
+    worker.register_wf(
+        "evict_while_la_running_no_interference",
+        la_problem_workflow,
+    );
+    worker.register_activity("delay", |_: ActContext, _: String| async {
+        tokio::time::sleep(Duration::from_secs(15)).await;
+        Ok(())
+    });
+    worker.run().await.unwrap();
+}

data/bridge/sdk-core/tests/integ_tests/workflow_tests/modify_wf_properties.rs

@@ -0,0 +1,53 @@
+use temporal_client::WorkflowClientTrait;
+use temporal_sdk::{WfContext, WorkflowResult};
+use temporal_sdk_core_protos::coresdk::{AsJsonPayloadExt, FromJsonPayloadExt};
+use temporal_sdk_core_test_utils::CoreWfStarter;
+use uuid::Uuid;
+
+static FIELD_A: &str = "cat_name";
+static FIELD_B: &str = "cute_level";
+
+async fn memo_upserter(ctx: WfContext) -> WorkflowResult<()> {
+    ctx.upsert_memo([
+        (FIELD_A.to_string(), "enchi".as_json_payload().unwrap()),
+        (FIELD_B.to_string(), 9001.as_json_payload().unwrap()),
+    ]);
+    Ok(().into())
+}
+
+#[tokio::test]
+async fn sends_modify_wf_props() {
+    let wf_name = "can_upsert_memo";
+    let wf_id = Uuid::new_v4();
+    let mut starter = CoreWfStarter::new(wf_name);
+    let mut worker = starter.worker().await;
+
+    worker.register_wf(wf_name, memo_upserter);
+    let run_id = worker
+        .submit_wf(wf_id.to_string(), wf_name, vec![], Default::default())
+        .await
+        .unwrap();
+    worker.run_until_done().await.unwrap();
+
+    let memo = starter
+        .get_client()
+        .await
+        .describe_workflow_execution(wf_id.to_string(), Some(run_id))
+        .await
+        .unwrap()
+        .workflow_execution_info
+        .unwrap()
+        .memo
+        .unwrap()
+        .fields;
+    let catname = memo.get(FIELD_A).unwrap();
+    let cuteness = memo.get(FIELD_B).unwrap();
+    for payload in [catname, cuteness] {
+        assert_eq!(
+            &b"json/plain".to_vec(),
+            payload.metadata.get("encoding").unwrap()
+        );
+    }
+    assert_eq!("enchi", String::from_json_payload(catname).unwrap());
+    assert_eq!(9001, usize::from_json_payload(cuteness).unwrap());
+}

data/bridge/sdk-core/tests/integ_tests/workflow_tests/replay.rs

@@ -1,6 +1,8 @@
 use assert_matches::assert_matches;
-use std::time::Duration;
-use temporal_sdk::{WfContext, Worker, WorkflowFunction};
+use parking_lot::Mutex;
+use std::{collections::HashSet, sync::Arc, time::Duration};
+use temporal_sdk::{interceptors::WorkerInterceptor, WfContext, Worker, WorkflowFunction};
+use temporal_sdk_core::replay::{HistoryFeeder, HistoryForReplay};
 use temporal_sdk_core_api::errors::{PollActivityError, PollWfError};
 use temporal_sdk_core_protos::{
     coresdk::{
@@ -8,20 +10,29 @@ use temporal_sdk_core_protos::{
         workflow_commands::{ScheduleActivity, StartTimer},
         workflow_completion::WorkflowActivationCompletion,
     },
-    DEFAULT_WORKFLOW_TYPE,
+    TestHistoryBuilder, DEFAULT_WORKFLOW_TYPE,
 };
 use temporal_sdk_core_test_utils::{
-    canned_histories, history_from_proto_binary, init_core_replay_preloaded, WorkerTestHelpers,
+    canned_histories, history_from_proto_binary, init_core_replay_preloaded, replay_sdk_worker,
+    replay_sdk_worker_stream, WorkerTestHelpers,
 };
 use tokio::join;
 
+fn test_hist_to_replay(t: TestHistoryBuilder) -> HistoryForReplay {
+    let hi = t.get_full_history_info().unwrap().into();
+    HistoryForReplay::new(hi, "fake".to_string())
+}
+
 #[tokio::test]
 async fn timer_workflow_replay() {
-    let (core, _) = init_core_replay_preloaded(
+    let core = init_core_replay_preloaded(
         "timer_workflow_replay",
-        &history_from_proto_binary("histories/timer_workflow_history.bin")
-            .await
-            .unwrap(),
+        [HistoryForReplay::new(
+            history_from_proto_binary("histories/timer_workflow_history.bin")
+                .await
+                .unwrap(),
+            "fake".to_owned(),
+        )],
     );
     let task = core.poll_workflow_activation().await.unwrap();
     core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
@@ -43,6 +54,14 @@ async fn timer_workflow_replay() {
         );
     };
     let poll_fut = async {
+        let evict_task = core
+            .poll_workflow_activation()
+            .await
+            .expect("Should be an eviction activation");
+        assert!(evict_task.eviction_reason().is_some());
+        core.complete_workflow_activation(WorkflowActivationCompletion::empty(evict_task.run_id))
+            .await
+            .unwrap();
         assert_matches!(
             core.poll_workflow_activation().await,
             Err(PollWfError::ShutDown)
@@ -64,11 +83,14 @@ async fn timer_workflow_replay() {
 
 #[tokio::test]
 async fn workflow_nondeterministic_replay() {
-    let (core, _) = init_core_replay_preloaded(
+    let core = init_core_replay_preloaded(
         "timer_workflow_replay",
-        &history_from_proto_binary("histories/timer_workflow_history.bin")
-            .await
-            .unwrap(),
+        [HistoryForReplay::new(
+            history_from_proto_binary("histories/timer_workflow_history.bin")
+                .await
+                .unwrap(),
+            "fake".to_owned(),
+        )],
     );
     let task = core.poll_workflow_activation().await.unwrap();
     core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
@@ -102,9 +124,7 @@ async fn replay_using_wf_function() {
     let num_timers = 10;
     let t = canned_histories::long_sequential_timers(num_timers as usize);
     let func = timers_wf(num_timers);
-    let (worker, _) =
-        init_core_replay_preloaded("replay_bench", &t.get_full_history_info().unwrap().into());
-    let mut worker = Worker::new_from_core(worker, "replay_bench".to_string());
+    let mut worker = replay_sdk_worker([test_hist_to_replay(t)]);
     worker.register_wf(DEFAULT_WORKFLOW_TYPE, func);
     worker.run().await.unwrap();
 }
@@ -117,16 +137,69 @@ async fn replay_ok_ending_with_terminated_or_timed_out() {
     t2.add_workflow_execution_timed_out();
     for t in [t1, t2] {
         let func = timers_wf(1);
-        let (worker, _) = init_core_replay_preloaded(
-            "replay_ok_terminate",
-            &t.get_full_history_info().unwrap().into(),
-        );
-        let mut worker = Worker::new_from_core(worker, "replay_ok_terminate".to_string());
+        let mut worker = replay_sdk_worker([test_hist_to_replay(t)]);
         worker.register_wf(DEFAULT_WORKFLOW_TYPE, func);
         worker.run().await.unwrap();
     }
 }
 
+#[rstest::rstest]
+#[tokio::test]
+async fn multiple_histories_replay(#[values(false, true)] use_feeder: bool) {
+    let num_timers = 10;
+    let seq_timer_wf = timers_wf(num_timers);
+    let one_timer_wf = timers_wf(1);
+    let mut one_timer_hist = canned_histories::single_timer("1");
+    one_timer_hist.set_wf_type("onetimer");
+    let mut seq_timer_hist = canned_histories::long_sequential_timers(num_timers as usize);
+    seq_timer_hist.set_wf_type("seqtimer");
+    let (feeder, stream) = HistoryFeeder::new(1);
+    let mut worker = if use_feeder {
+        replay_sdk_worker_stream(stream)
+    } else {
+        replay_sdk_worker([
+            test_hist_to_replay(one_timer_hist.clone()),
+            test_hist_to_replay(seq_timer_hist.clone()),
+        ])
+    };
+    let runs_ctr_i = UniqueRunsCounter::default();
+    let runs_ctr = runs_ctr_i.runs.clone();
+    worker.set_worker_interceptor(Box::new(runs_ctr_i));
+    worker.register_wf("onetimer", one_timer_wf);
+    worker.register_wf("seqtimer", seq_timer_wf);
+
+    if use_feeder {
+        let feed_fut = async move {
+            feeder
+                .feed(test_hist_to_replay(one_timer_hist))
+                .await
+                .unwrap();
+            feeder
+                .feed(test_hist_to_replay(seq_timer_hist))
+                .await
+                .unwrap();
+        };
+        let (_, runr) = join!(feed_fut, worker.run());
+        runr.unwrap();
+    } else {
+        worker.run().await.unwrap();
+    }
+    assert_eq!(runs_ctr.lock().len(), 2);
+}
+
+#[tokio::test]
+async fn multiple_histories_can_handle_dupe_run_ids() {
+    let mut hist1 = canned_histories::single_timer("1");
+    hist1.set_wf_type("onetimer");
+    let mut worker = replay_sdk_worker([
+        test_hist_to_replay(hist1.clone()),
+        test_hist_to_replay(hist1.clone()),
+        test_hist_to_replay(hist1),
+    ]);
+    worker.register_wf("onetimer", timers_wf(1));
+    worker.run().await.unwrap();
+}
+
 fn timers_wf(num_timers: u32) -> WorkflowFunction {
     WorkflowFunction::new(move |ctx: WfContext| async move {
         for _ in 1..=num_timers {
@@ -135,3 +208,16 @@ fn timers_wf(num_timers: u32) -> WorkflowFunction {
         Ok(().into())
     })
 }
+
+#[derive(Default)]
+struct UniqueRunsCounter {
+    runs: Arc<Mutex<HashSet<String>>>,
+}
+#[async_trait::async_trait(?Send)]
+impl WorkerInterceptor for UniqueRunsCounter {
+    async fn on_workflow_activation_completion(&self, completion: &WorkflowActivationCompletion) {
+        self.runs.lock().insert(completion.run_id.clone());
+    }
+
+    fn on_shutdown(&self, _: &Worker) {}
+}

data/bridge/sdk-core/tests/integ_tests/workflow_tests.rs

@@ -6,6 +6,7 @@ mod child_workflows;
 mod continue_as_new;
 mod determinism;
 mod local_activities;
+mod modify_wf_properties;
 mod patches;
 mod replay;
 mod resets;
@@ -28,6 +29,7 @@ use temporal_client::{WorkflowClientTrait, WorkflowOptions};
 use temporal_sdk::{
     interceptors::WorkerInterceptor, ActContext, ActivityOptions, WfContext, WorkflowResult,
 };
+use temporal_sdk_core::replay::HistoryForReplay;
 use temporal_sdk_core_api::{errors::PollWfError, Worker};
 use temporal_sdk_core_protos::{
     coresdk::{
@@ -37,7 +39,7 @@ use temporal_sdk_core_protos::{
         workflow_completion::WorkflowActivationCompletion,
         ActivityTaskCompletion, AsJsonPayloadExt, IntoCompletion,
     },
-    temporal::api::failure::v1::Failure,
+    temporal::api::{failure::v1::Failure, history::v1::history_event},
 };
 use temporal_sdk_core_test_utils::{
     history_from_proto_binary, init_core_and_create_wf, init_core_replay_preloaded,
@@ -102,6 +104,7 @@ async fn parallel_workflows_same_queue() {
     for handle in handles {
         handle.await.unwrap()
     }
+    core.shutdown().await;
 }
 
 static RUN_CT: AtomicUsize = AtomicUsize::new(0);
@@ -179,13 +182,20 @@ async fn shutdown_aborts_actively_blocked_poll() {
 #[tokio::test]
 async fn fail_wf_task(#[values(true, false)] replay: bool) {
     let core = if replay {
-        let (core, _) = init_core_replay_preloaded(
-            "fail_wf_task",
-            &history_from_proto_binary("histories/fail_wf_task.bin")
-                .await
-                .unwrap(),
-        );
-        core
+        // We need to send the history twice, since we fail it the first time.
+        let mut hist_proto = history_from_proto_binary("histories/fail_wf_task.bin")
+            .await
+            .unwrap();
+        let hist = HistoryForReplay::new(hist_proto.clone(), "fake".to_string());
+        if let Some(history_event::Attributes::WorkflowExecutionStartedEventAttributes(
+            ref mut attrs,
+        )) = hist_proto.events[0].attributes
+        {
+            attrs.original_execution_run_id = "run2".to_string();
+            attrs.first_execution_run_id = "run2".to_string();
+        }
+        let hist2 = HistoryForReplay::new(hist_proto, "fake".to_string());
+        init_core_replay_preloaded("fail_wf_task", [hist, hist2])
     } else {
         let mut starter = init_core_and_create_wf("fail_wf_task").await;
         starter.get_worker().await

data/bridge/sdk-core/tests/main.rs

@@ -20,6 +20,7 @@ mod integ_tests {
     mod client_tests;
     mod ephemeral_server_tests;
     mod heartbeat_tests;
+    mod metrics_tests;
     mod polling_tests;
     mod queries_tests;
     mod visibility_tests;
@@ -28,10 +29,10 @@ mod integ_tests {
     use std::str::FromStr;
     use temporal_client::WorkflowService;
     use temporal_sdk_core::{
-        init_worker, telemetry_init, ClientOptionsBuilder, ClientTlsConfig, TlsConfig,
+        init_worker, ClientOptionsBuilder, ClientTlsConfig, CoreRuntime, TlsConfig,
         WorkflowClientTrait,
     };
-    use temporal_sdk_core_api::{worker::WorkerConfigBuilder, CoreTelemetry};
+    use temporal_sdk_core_api::worker::WorkerConfigBuilder;
     use temporal_sdk_core_protos::temporal::api::workflowservice::v1::ListNamespacesRequest;
     use temporal_sdk_core_test_utils::{
         get_integ_server_options, get_integ_telem_options, NAMESPACE,
@@ -43,13 +44,14 @@ mod integ_tests {
     #[ignore] // Really a compile time check more than anything
     async fn lang_bridge_example() {
         let opts = get_integ_server_options();
-        let telem_d = telemetry_init(&get_integ_telem_options()).unwrap();
+        let runtime = CoreRuntime::new_assume_tokio(get_integ_telem_options()).unwrap();
         let mut retrying_client = opts
-            .connect_no_namespace(telem_d.get_metric_meter(), None)
+            .connect_no_namespace(runtime.metric_meter(), None)
             .await
             .unwrap();
 
         let _worker = init_worker(
+            &runtime,
             WorkerConfigBuilder::default()
                 .namespace("default")
                 .task_queue("Wheee!")