temporalio 0.0.0 → 0.0.2

data/bridge/sdk-core/tests/load_tests.rs

@@ -0,0 +1,191 @@
+use assert_matches::assert_matches;
+use futures::{future::join_all, sink, stream::FuturesUnordered, StreamExt};
+use std::time::{Duration, Instant};
+use temporal_client::{WfClientExt, WorkflowClientTrait, WorkflowOptions};
+use temporal_sdk::{ActContext, ActivityOptions, WfContext};
+use temporal_sdk_core_protos::coresdk::{
+    activity_result::ActivityExecutionResult, activity_task::activity_task as act_task,
+    workflow_commands::ActivityCancellationType, ActivityTaskCompletion, AsJsonPayloadExt,
+};
+use temporal_sdk_core_test_utils::CoreWfStarter;
+
+#[tokio::test]
+async fn activity_load() {
+    const CONCURRENCY: usize = 1000;
+
+    let mut starter = CoreWfStarter::new("activity_load");
+    starter
+        .max_wft(CONCURRENCY)
+        .max_cached_workflows(CONCURRENCY)
+        .max_at_polls(10)
+        .max_at(CONCURRENCY);
+    let mut worker = starter.worker().await;
+
+    let activity_id = "act-1";
+    let activity_timeout = Duration::from_secs(8);
+    let payload_dat = b"hello".to_vec();
+    let task_queue = starter.get_task_queue().to_owned();
+
+    let pd = payload_dat.clone();
+    let wf_fn = move |ctx: WfContext| {
+        let task_queue = task_queue.clone();
+        let payload_dat = pd.clone();
+
+        async move {
+            let activity = ActivityOptions {
+                activity_id: Some(activity_id.to_string()),
+                activity_type: "test_activity".to_string(),
+                task_queue,
+                schedule_to_start_timeout: Some(activity_timeout),
+                start_to_close_timeout: Some(activity_timeout),
+                schedule_to_close_timeout: Some(activity_timeout),
+                heartbeat_timeout: Some(activity_timeout),
+                cancellation_type: ActivityCancellationType::TryCancel,
+                ..Default::default()
+            };
+            let res = ctx.activity(activity).await.unwrap_ok_payload();
+            assert_eq!(res.data, payload_dat);
+            Ok(().into())
+        }
+    };
+
+    let starting = Instant::now();
+    let wf_type = "activity_load";
+    worker.register_wf(wf_type.to_owned(), wf_fn);
+    join_all((0..CONCURRENCY).map(|i| {
+        let worker = &worker;
+        let wf_id = format!("activity_load_{}", i);
+        async move {
+            worker
+                .submit_wf(
+                    wf_id,
+                    wf_type.to_owned(),
+                    vec![],
+                    WorkflowOptions::default(),
+                )
+                .await
+                .unwrap();
+        }
+    }))
+    .await;
+    dbg!(starting.elapsed());
+
+    let running = Instant::now();
+    let core = starter.get_worker().await;
+
+    // Poll for and complete all activities
+    let c2 = core.clone();
+    let all_acts = async move {
+        let mut act_complete_futs = vec![];
+        for _ in 0..CONCURRENCY {
+            let task = c2.poll_activity_task().await.unwrap();
+            assert_matches!(
+                task.variant,
+                Some(act_task::Variant::Start(ref start_activity)) => {
+                    assert_eq!(start_activity.activity_type, "test_activity")
+                }
+            );
+            let pd = payload_dat.clone();
+            let core = c2.clone();
+            act_complete_futs.push(tokio::spawn(async move {
+                core.complete_activity_task(ActivityTaskCompletion {
+                    task_token: task.task_token,
+                    result: Some(ActivityExecutionResult::ok(pd.into())),
+                })
+                .await
+                .unwrap()
+            }));
+        }
+        join_all(act_complete_futs)
+            .await
+            .into_iter()
+            .for_each(|h| h.unwrap());
+    };
+    tokio::join! {
+        async {
+            worker.run_until_done().await.unwrap();
+        },
+        all_acts
+    };
+    dbg!(running.elapsed());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn workflow_load() {
+    const SIGNAME: &str = "signame";
+    let num_workflows = 200;
+    let wf_name = "workflow_load";
+    let mut starter = CoreWfStarter::new("workflow_load");
+    starter
+        .max_wft(5)
+        .max_cached_workflows(5)
+        .max_at_polls(10)
+        .max_at(100);
+    let mut worker = starter.worker().await;
+    worker.register_wf(wf_name.to_owned(), |ctx: WfContext| async move {
+        let sigchan = ctx.make_signal_channel(SIGNAME).map(Ok);
+        let drained_fut = sigchan.forward(sink::drain());
+
+        let real_stuff = async move {
+            for _ in 0..20 {
+                ctx.activity(ActivityOptions {
+                    activity_type: "echo_activity".to_string(),
+                    start_to_close_timeout: Some(Duration::from_secs(5)),
+                    input: "hi!".as_json_payload().expect("serializes fine"),
+                    ..Default::default()
+                })
+                .await;
+                ctx.timer(Duration::from_secs(1)).await;
+            }
+        };
+        tokio::select! {
+            _ = drained_fut => {}
+            _ = real_stuff => {}
+        }
+
+        Ok(().into())
+    });
+    worker.register_activity(
+        "echo_activity",
+        |_ctx: ActContext, echo_me: String| async move { Ok(echo_me) },
+    );
+    let client = starter.get_client().await;
+
+    let mut workflow_handles = vec![];
+    for i in 0..num_workflows {
+        let wfid = format!("{}_{}", wf_name, i);
+        let rid = worker
+            .submit_wf(
+                wfid.clone(),
+                wf_name.to_owned(),
+                vec![],
+                WorkflowOptions::default(),
+            )
+            .await
+            .unwrap();
+        workflow_handles.push(client.get_untyped_workflow_handle(wfid, rid));
+    }
+
+    let sig_sender = async {
+        loop {
+            let sends: FuturesUnordered<_> = (0..num_workflows)
+                .map(|i| {
+                    client.signal_workflow_execution(
+                        format!("{}_{}", wf_name, i),
+                        "".to_string(),
+                        SIGNAME.to_string(),
+                        None,
+                        None,
+                    )
+                })
+                .collect();
+            sends
+                .map(|_| Ok(()))
+                .forward(sink::drain())
+                .await
+                .expect("Sending signals works");
+            tokio::time::sleep(Duration::from_secs(2)).await;
+        }
+    };
+    tokio::select! { r1 = worker.run_until_done() => {r1.unwrap()}, _ = sig_sender => {}};
+}

data/bridge/sdk-core/tests/main.rs

@@ -0,0 +1,113 @@
+//! Integration tests
+//!
+//! Note that integ tests which want to use the server (nearly all of them) *need* to use the
+//! `#[rstest]` macro and accept the TODO fixture to support auto setup & teardown of ephemeral
+//! local servers.
+
+#[macro_use]
+extern crate rstest;
+
+#[cfg(test)]
+mod integ_tests {
+    #[macro_export]
+    macro_rules! prost_dur {
+        ($dur_call:ident $args:tt) => {
+            std::time::Duration::$dur_call$args
+                .try_into()
+                .expect("test duration fits")
+        };
+    }
+    mod client_tests;
+    mod ephemeral_server_tests;
+    mod heartbeat_tests;
+    mod metrics_tests;
+    mod polling_tests;
+    mod queries_tests;
+    mod visibility_tests;
+    mod workflow_tests;
+
+    use std::str::FromStr;
+    use temporal_client::WorkflowService;
+    use temporal_sdk_core::{
+        init_worker, ClientOptionsBuilder, ClientTlsConfig, CoreRuntime, TlsConfig,
+        WorkflowClientTrait,
+    };
+    use temporal_sdk_core_api::worker::WorkerConfigBuilder;
+    use temporal_sdk_core_protos::temporal::api::workflowservice::v1::ListNamespacesRequest;
+    use temporal_sdk_core_test_utils::{
+        get_integ_server_options, get_integ_telem_options, NAMESPACE,
+    };
+    use url::Url;
+
+    // Create a worker like a bridge would (unwraps aside)
+    #[tokio::test]
+    #[ignore] // Really a compile time check more than anything
+    async fn lang_bridge_example() {
+        let opts = get_integ_server_options();
+        let runtime = CoreRuntime::new_assume_tokio(get_integ_telem_options()).unwrap();
+        let mut retrying_client = opts
+            .connect_no_namespace(runtime.metric_meter(), None)
+            .await
+            .unwrap();
+
+        let _worker = init_worker(
+            &runtime,
+            WorkerConfigBuilder::default()
+                .namespace("default")
+                .task_queue("Wheee!")
+                .build()
+                .unwrap(),
+            // clone the client if you intend to use it later. Strip off the retry wrapper since
+            // worker will assert its own
+            retrying_client.clone(),
+        );
+
+        // Do things with worker or client
+        let _ = retrying_client
+            .list_namespaces(ListNamespacesRequest::default())
+            .await;
+    }
+
+    // TODO: Currently ignored because starting up the docker image with TLS requires some hoop
+    //  jumping. We should upgrade CI to be able to do that but this was manually run against
+    //  https://github.com/temporalio/customization-samples/tree/master/tls/tls-simple
+    #[tokio::test]
+    #[ignore]
+    async fn tls_test() {
+        // Load certs/keys
+        let root = tokio::fs::read(
+            "/home/sushi/dev/temporal/customization-samples/tls/tls-simple/certs/ca.cert",
+        )
+        .await
+        .unwrap();
+        let client_cert = tokio::fs::read(
+            "/home/sushi/dev/temporal/customization-samples/tls/tls-simple/certs/client.pem",
+        )
+        .await
+        .unwrap();
+        let client_private_key = tokio::fs::read(
+            "/home/sushi/dev/temporal/customization-samples/tls/tls-simple/certs/client.key",
+        )
+        .await
+        .unwrap();
+        let sgo = ClientOptionsBuilder::default()
+            .target_url(Url::from_str("https://localhost:7233").unwrap())
+            .client_name("tls_tester")
+            .client_version("clientver")
+            .tls_cfg(TlsConfig {
+                server_root_ca_cert: Some(root),
+                domain: Some("tls-sample".to_string()),
+                client_tls_config: Some(ClientTlsConfig {
+                    client_cert,
+                    client_private_key,
+                }),
+            })
+            .build()
+            .unwrap();
+        let con = sgo
+            .connect(NAMESPACE.to_string(), None, None)
+            .await
+            .unwrap();
+        con.list_namespaces().await.unwrap();
+    }
+}

data/bridge/sdk-core/tests/runner.rs

@@ -0,0 +1,93 @@
+use anyhow::{anyhow, bail};
+use std::{
+    env,
+    env::args,
+    path::{Path, PathBuf},
+    process::Stdio,
+};
+use temporal_sdk_core::ephemeral_server::{TemporaliteConfigBuilder, TestServerConfigBuilder};
+use temporal_sdk_core_test_utils::{
+    default_cached_download, INTEG_SERVER_TARGET_ENV_VAR, INTEG_TEMPORALITE_USED_ENV_VAR,
+    INTEG_TEST_SERVER_USED_ENV_VAR,
+};
+use tokio::{self, process::Command};
+
+#[tokio::main]
+async fn main() -> Result<(), anyhow::Error> {
+    let cargo = env::var("CARGO").unwrap_or_else(|_| "cargo".to_string());
+    let server_type = env::var("INTEG_SERVER_TYPE")
+        .unwrap_or_else(|_| "temporalite".to_string())
+        .to_lowercase();
+    // Try building first, so that we error early on build failures & don't start server
+    let status = Command::new(&cargo)
+        .args(["test", "--test", "integ_tests", "--no-run"])
+        .status()
+        .await?;
+    if !status.success() {
+        bail!("Building integration tests failed!");
+    }
+
+    // Move to clap if we start doing any more complicated input
+    let (server, envs) = if server_type == "test-server" {
+        let config = TestServerConfigBuilder::default()
+            .exe(default_cached_download())
+            .build()?;
+        println!("Using java test server");
+        (
+            Some(config.start_server_with_output(Stdio::null()).await?),
+            vec![(INTEG_TEST_SERVER_USED_ENV_VAR, "true")],
+        )
+    } else if server_type == "temporalite" {
+        let config = TemporaliteConfigBuilder::default()
+            .exe(default_cached_download())
+            .build()?;
+        println!("Using temporalite");
+        (
+            Some(config.start_server_with_output(Stdio::null()).await?),
+            vec![(INTEG_TEMPORALITE_USED_ENV_VAR, "true")],
+        )
+    } else {
+        println!("Not starting up a server. One should be running already.");
+        (None, vec![])
+    };
+
+    // Run the integ tests, passing through arguments
+    let mut args = args();
+    // Shift off binary name
+    args.next();
+    let mut cmd = Command::new(&cargo);
+    if let Some(srv) = server.as_ref() {
+        cmd.env(
+            INTEG_SERVER_TARGET_ENV_VAR,
+            format!("http://{}", &srv.target),
+        );
+    }
+    let status = cmd
+        .envs(envs)
+        .current_dir(project_root())
+        .args(
+            ["test", "--test", "integ_tests"]
+                .into_iter()
+                .map(ToString::to_string)
+                .chain(args),
+        )
+        .status()
+        .await?;
+
+    if let Some(mut srv) = server {
+        srv.shutdown().await?;
+    }
+    if status.success() {
+        Ok(())
+    } else {
+        Err(anyhow!("Integ tests failed!"))
+    }
+}
+
+fn project_root() -> PathBuf {
+    Path::new(&env!("CARGO_MANIFEST_DIR"))
+        .ancestors()
+        .nth(1)
+        .unwrap()
+        .to_path_buf()
+}

data/bridge/src/connection.rs

@@ -0,0 +1,186 @@
+use std::collections::HashMap;
+use std::convert::TryFrom;
+use std::str::FromStr;
+use std::sync::Arc;
+use std::time::Duration;
+use temporal_client::{
+    ClientInitError, ClientOptionsBuilder, ClientOptionsBuilderError, WorkflowService, RetryClient,
+    ConfiguredClient, TemporalServiceClientWithMetrics
+};
+use thiserror::Error;
+use tokio::{select};
+use tokio::runtime::{Runtime};
+use tokio_util::sync::CancellationToken;
+use tonic::metadata::{MetadataKey,MetadataValue};
+use url::Url;
+
+pub type Client = RetryClient<ConfiguredClient<TemporalServiceClientWithMetrics>>;
+
+#[derive(Error, Debug)]
+pub enum ConnectionError {
+    #[error(transparent)]
+    InvalidUrl(#[from] url::ParseError),
+
+    #[error(transparent)]
+    InvalidProtobuf(#[from] prost::DecodeError),
+
+    #[error(transparent)]
+    InvalidConnectionOptions(#[from] ClientOptionsBuilderError),
+
+    #[error("`{0}` RPC call is not supported by the API")]
+    InvalidRpc(String),
+
+    #[error(transparent)]
+    InvalidRpcMetadataKey(#[from] tonic::metadata::errors::InvalidMetadataKey),
+
+    #[error(transparent)]
+    InvalidRpcMetadataValue(#[from] tonic::metadata::errors::InvalidMetadataValue),
+
+    #[error(transparent)]
+    UnableToConnect(#[from] ClientInitError),
+
+    #[error(transparent)]
+    UnableToInitializeRuntime(#[from] std::io::Error),
+
+    #[error(transparent)]
+    RequestError(#[from] tonic::Status),
+
+    #[error("RPC call was cancelled")]
+    RequestCancelled,
+}
+
+pub type RpcResult = Result<Vec<u8>, ConnectionError>;
+
+fn rpc_req<P: prost::Message + Default>(params: RpcParams) -> Result<tonic::Request<P>, ConnectionError> {
+    let proto = P::decode(&*params.request)?;
+    let mut req = tonic::Request::new(proto);
+
+    for (k, v) in &params.metadata {
+        req.metadata_mut().insert(
+            MetadataKey::from_str(k.as_str())?,
+            MetadataValue::try_from(v.as_str())?
+        );
+    }
+
+    if let Some(timeout_millis) = params.timeout_millis {
+        req.set_timeout(Duration::from_millis(timeout_millis));
+    }
+
+    Ok(req)
+}
+
+fn rpc_resp<P: prost::Message + Default>(res: Result<tonic::Response<P>, tonic::Status>) -> RpcResult {
+    Ok(res?.get_ref().encode_to_vec())
+}
+
+macro_rules! rpc_call {
+    ($client:expr, $rpc:ident, $params:expr) => {
+        rpc_resp($client.$rpc(rpc_req($params)?).await)
+    };
+}
+
+async fn make_rpc_call(mut client: Client, params: RpcParams) -> RpcResult {
+    match params.rpc.as_str() {
+        "register_namespace" => rpc_call!(client, register_namespace, params),
+        "describe_namespace" => rpc_call!(client, describe_namespace, params),
+        "list_namespaces" => rpc_call!(client, list_namespaces, params),
+        "update_namespace" => rpc_call!(client, update_namespace, params),
+        "deprecate_namespace" => rpc_call!(client, deprecate_namespace, params),
+        "start_workflow_execution" => rpc_call!(client, start_workflow_execution, params),
+        "get_workflow_execution_history" => rpc_call!(client, get_workflow_execution_history, params),
+        // "get_workflow_execution_history_reverse" => rpc_call!(client, get_workflow_execution_history_reverse, params),
+        "poll_workflow_task_queue" => rpc_call!(client, poll_workflow_task_queue, params),
+        "respond_workflow_task_completed" => rpc_call!(client, respond_workflow_task_completed, params),
+        "respond_workflow_task_failed" => rpc_call!(client, respond_workflow_task_failed, params),
+        "poll_activity_task_queue" => rpc_call!(client, poll_activity_task_queue, params),
+        "record_activity_task_heartbeat" => rpc_call!(client, record_activity_task_heartbeat, params),
+        "record_activity_task_heartbeat_by_id" => rpc_call!(client, record_activity_task_heartbeat_by_id, params),
+        "respond_activity_task_completed" => rpc_call!(client, respond_activity_task_completed, params),
+        "respond_activity_task_completed_by_id" => rpc_call!(client, respond_activity_task_completed_by_id, params),
+        "respond_activity_task_failed" => rpc_call!(client, respond_activity_task_failed, params),
+        "respond_activity_task_failed_by_id" => rpc_call!(client, respond_activity_task_failed_by_id, params),
+        "respond_activity_task_canceled" => rpc_call!(client, respond_activity_task_canceled, params),
+        "respond_activity_task_canceled_by_id" => rpc_call!(client, respond_activity_task_canceled_by_id, params),
+        "request_cancel_workflow_execution" => rpc_call!(client, request_cancel_workflow_execution, params),
+        "signal_workflow_execution" => rpc_call!(client, signal_workflow_execution, params),
+        "signal_with_start_workflow_execution" => rpc_call!(client, signal_with_start_workflow_execution, params),
+        "reset_workflow_execution" => rpc_call!(client, reset_workflow_execution, params),
+        "terminate_workflow_execution" => rpc_call!(client, terminate_workflow_execution, params),
+        "delete_workflow_execution" => rpc_call!(client, delete_workflow_execution, params),
+        "list_open_workflow_executions" => rpc_call!(client, list_open_workflow_executions, params),
+        "list_closed_workflow_executions" => rpc_call!(client, list_closed_workflow_executions, params),
+        "list_workflow_executions" => rpc_call!(client, list_workflow_executions, params),
+        "list_archived_workflow_executions" => rpc_call!(client, list_archived_workflow_executions, params),
+        "scan_workflow_executions" => rpc_call!(client, scan_workflow_executions, params),
+        "count_workflow_executions" => rpc_call!(client, count_workflow_executions, params),
+        "get_search_attributes" => rpc_call!(client, get_search_attributes, params),
+        "respond_query_task_completed" => rpc_call!(client, respond_query_task_completed, params),
+        "reset_sticky_task_queue" => rpc_call!(client, reset_sticky_task_queue, params),
+        "query_workflow" => rpc_call!(client, query_workflow, params),
+        "describe_workflow_execution" => rpc_call!(client, describe_workflow_execution, params),
+        "describe_task_queue" => rpc_call!(client, describe_task_queue, params),
+        "get_cluster_info" => rpc_call!(client, get_cluster_info, params),
+        "get_system_info" => rpc_call!(client, get_system_info, params),
+        "list_task_queue_partitions" => rpc_call!(client, list_task_queue_partitions, params),
+        "create_schedule" => rpc_call!(client, create_schedule, params),
+        "describe_schedule" => rpc_call!(client, describe_schedule, params),
+        "update_schedule" => rpc_call!(client, update_schedule, params),
+        "patch_schedule" => rpc_call!(client, patch_schedule, params),
+        "list_schedule_matching_times" => rpc_call!(client, list_schedule_matching_times, params),
+        "delete_schedule" => rpc_call!(client, delete_schedule, params),
+        "list_schedules" => rpc_call!(client, list_schedules, params),
+        "update_worker_build_id_ordering" => rpc_call!(client, update_worker_build_id_ordering, params),
+        "get_worker_build_id_ordering" => rpc_call!(client, get_worker_build_id_ordering, params),
+        "update_workflow" => rpc_call!(client, update_workflow, params),
+        "start_batch_operation" => rpc_call!(client, start_batch_operation, params),
+        "stop_batch_operation" => rpc_call!(client, stop_batch_operation, params),
+        "describe_batch_operation" => rpc_call!(client, describe_batch_operation, params),
+        "list_batch_operations" => rpc_call!(client, list_batch_operations, params),
+        _ => Err(ConnectionError::InvalidRpc(params.rpc.to_string()))
+    }
+}
+
+// A Connection is a Client wrapper for making RPC calls
+pub struct Connection {
+    pub client: Client,
+    runtime: Arc<Runtime>
+}
+
+pub struct RpcParams {
+    pub rpc: String,
+    pub request: Vec<u8>,
+    pub metadata: HashMap<String, String>,
+    pub timeout_millis: Option<u64>
+}
+
+async fn create_client(host: String) -> Result<Client, ConnectionError> {
+    let url = Url::try_from(&*host)?;
+    let options = ClientOptionsBuilder::default()
+        .identity("testtest".to_string())
+        .target_url(url)
+        .client_name("ruby-sdk".to_string())
+        .client_version("0.0.1".to_string())
+        .build()?;
+
+    Ok(options.connect_no_namespace(None, None).await?)
+}
+
+impl Connection {
+    // TODO: Change the interface to accept a full configuration
+    pub fn connect(runtime: Arc<Runtime>, host: String) -> Result<Self, ConnectionError> {
+        let client = runtime.block_on(create_client(host))?;
+
+        Ok(Connection { client, runtime })
+    }
+
+    pub fn call(&self, params: RpcParams, cancel_token: CancellationToken) -> RpcResult {
+        let core_client = self.client.clone();
+
+        self.runtime.block_on(async move {
+            select! {
+                _ = cancel_token.cancelled() => Err(ConnectionError::RequestCancelled),
+                result = make_rpc_call(core_client, params) => result
+            }
+        })
+    }
+}