@rivetkit/engine 1.0.0

package/packages/engine/tests/common/ctx.rs

@@ -0,0 +1,201 @@
+use anyhow::*;
+use gas::prelude::*;
+use rivet_service_manager::{Service, ServiceKind};
+use std::time::Duration;
+
+pub struct TestOpts {
+	pub datacenters: usize,
+}
+
+impl TestOpts {
+	pub fn new(datacenters: usize) -> Self {
+		Self { datacenters }
+	}
+}
+
+impl Default for TestOpts {
+	fn default() -> Self {
+		Self { datacenters: 1 }
+	}
+}
+
+pub struct TestCtx {
+	dcs: Vec<TestDatacenter>,
+	pub opts: TestOpts,
+}
+
+pub struct TestDatacenter {
+	pub config: rivet_config::Config,
+	pub pools: rivet_pools::Pools,
+	pub test_deps: rivet_test_deps::TestDeps,
+	pub workflow_ctx: StandaloneCtx,
+	engine_handle: tokio::task::JoinHandle<Result<()>>,
+}
+
+impl TestCtx {
+	/// Creates a test context with multiple datacenters
+	pub async fn new_multi(dc_count: usize) -> Result<Self> {
+		Self::new_with_opts(TestOpts::new(dc_count)).await
+	}
+
+	/// Creates a test context with custom options
+	pub async fn new_with_opts(opts: TestOpts) -> Result<Self> {
+		// Set up logging
+		let _ = tracing_subscriber::fmt()
+			.with_env_filter("info")
+			.with_ansi(false)
+			.with_test_writer()
+			.try_init();
+
+		// Initialize test dependencies for all DCs
+		assert!(opts.datacenters >= 1, "datacenters must be at least 1");
+		let dc_count = opts.datacenters;
+		tracing::info!("setting up test dependencies for {} DCs", dc_count);
+		let dc_labels: Vec<u16> = (1..=dc_count as u16).collect();
+		let test_deps_list = rivet_test_deps::TestDeps::new_multi(&dc_labels)
+			.await?
+			.into_iter();
+
+		// Setup all datacenters
+		let mut dcs = Vec::new();
+		for test_deps in test_deps_list {
+			let dc = Self::setup_instance(test_deps).await?;
+			dcs.push(dc);
+		}
+
+		Ok(Self { dcs, opts })
+	}
+
+	async fn setup_instance(test_deps: rivet_test_deps::TestDeps) -> Result<TestDatacenter> {
+		let config = test_deps.config().clone();
+		let pools = test_deps.pools().clone();
+
+		// Start the service manager with all required services
+		let dc_label = config.dc_label();
+		tracing::info!(dc_label, "starting engine services for DC");
+		let engine_handle = tokio::spawn({
+			let config = config.clone();
+			let pools = pools.clone();
+			async move {
+				let services = vec![
+					Service::new(
+						"api-peer",
+						ServiceKind::ApiPeer,
+						|config, pools| Box::pin(rivet_api_peer::start(config, pools)),
+						false,
+					),
+					Service::new(
+						"guard",
+						ServiceKind::Standalone,
+						|config, pools| Box::pin(rivet_guard::start(config, pools)),
+						true,
+					),
+					Service::new(
+						"workflow-worker",
+						ServiceKind::Standalone,
+						|config, pools| Box::pin(rivet_workflow_worker::start(config, pools)),
+						true,
+					),
+					Service::new(
+						"bootstrap",
+						ServiceKind::Oneshot,
+						|config, pools| Box::pin(rivet_bootstrap::start(config, pools)),
+						false,
+					),
+				];
+
+				rivet_service_manager::start(config, pools, services).await
+			}
+		});
+
+		// Wait for ports to open
+		tracing::info!(dc_label, "waiting for services to be ready");
+		tokio::join!(
+			wait_for_port("api-peer", test_deps.api_peer_port()),
+			wait_for_port("guard", test_deps.guard_port()),
+		);
+
+		// Create workflow context for assertions
+		let cache = rivet_cache::CacheInner::from_env(&config, pools.clone())?;
+		let workflow_ctx = StandaloneCtx::new(
+			db::DatabaseKv::from_pools(pools.clone()).await?,
+			config.clone(),
+			pools.clone(),
+			cache,
+			"test",
+			Id::new_v1(config.dc_label()),
+			Id::new_v1(config.dc_label()),
+		)?;
+
+		Ok(TestDatacenter {
+			config,
+			pools,
+			test_deps,
+			workflow_ctx,
+			engine_handle,
+		})
+	}
+
+	pub fn leader_dc(&self) -> &TestDatacenter {
+		&self.dcs[0]
+	}
+
+	pub fn get_dc(&self, label: u16) -> &TestDatacenter {
+		self.dcs
+			.iter()
+			.find(|dc| dc.config.dc_label() == label)
+			.unwrap_or_else(|| panic!("No datacenter found with label {}", label))
+	}
+
+	pub async fn shutdown(self) {
+		tracing::info!("shutting down multi-DC test context");
+		for dc in self.dcs {
+			dc.shutdown().await;
+		}
+	}
+}
+
+impl TestDatacenter {
+	pub fn api_peer_port(&self) -> u16 {
+		self.test_deps.api_peer_port()
+	}
+
+	pub fn guard_port(&self) -> u16 {
+		self.test_deps.guard_port()
+	}
+
+	async fn shutdown(self) {
+		tracing::info!(
+			dc_label = self.config.dc_label(),
+			"shutting down test instance"
+		);
+		self.engine_handle.abort();
+	}
+}
+
+pub async fn wait_for_port(service_name: &str, port: u16) {
+	let addr = format!("127.0.0.1:{}", port);
+	let start = std::time::Instant::now();
+	let timeout = Duration::from_secs(30);
+
+	tracing::info!("waiting for {} on port {}", service_name, port);
+
+	loop {
+		match tokio::net::TcpStream::connect(&addr).await {
+			std::result::Result::Ok(_) => {
+				tracing::info!("{} is ready on port {}", service_name, port);
+				return;
+			}
+			std::result::Result::Err(e) => {
+				if start.elapsed() > timeout {
+					panic!(
+						"Timeout waiting for {} on port {} after {:?}: {}",
+						service_name, port, timeout, e
+					);
+				}
+				// Check less frequently to avoid spamming
+				tokio::time::sleep(Duration::from_millis(100)).await;
+			}
+		}
+	}
+}

package/packages/engine/tests/common/mod.rs

@@ -0,0 +1,30 @@
+#![allow(dead_code, unused_variables, unused_imports)]
+
+pub mod actors;
+pub mod ctx;
+pub mod ns;
+pub mod runner;
+pub mod test_helpers;
+
+pub use actors::*;
+pub use ctx::*;
+pub use ns::*;
+pub use test_helpers::*;
+
+use std::future::Future;
+use std::time::Duration;
+
+pub fn run<F, Fut>(opts: TestOpts, test_fn: F)
+where
+	F: FnOnce(TestCtx) -> Fut,
+	Fut: Future<Output = ()>,
+{
+	let runtime = tokio::runtime::Runtime::new().expect("failed to build runtime");
+
+	runtime.block_on(async {
+		let ctx = TestCtx::new_with_opts(opts).await.expect("build testctx");
+		tokio::time::timeout(Duration::from_secs(10), test_fn(ctx))
+			.await
+			.expect("test timed out");
+	});
+}

package/packages/engine/tests/common/ns.rs

@@ -0,0 +1,36 @@
+use std::str::FromStr;
+
+pub async fn create_namespace(name: &str, guard_port: u16) -> rivet_util::Id {
+	tracing::info!(?name, ?guard_port, "creating test namespace");
+
+	let client = reqwest::Client::new();
+	let response = client
+		.post(format!("http://127.0.0.1:{}/namespaces", guard_port))
+		.json(&serde_json::json!({
+			"name": name,
+			"display_name": "Test Namespace",
+		}))
+		.send()
+		.await
+		.expect("Failed to send namespace creation request");
+
+	if !response.status().is_success() {
+		let text = response.text().await.expect("Failed to read response text");
+		panic!("Failed to create namespace: {}", text);
+	}
+
+	let body: serde_json::Value = response
+		.json()
+		.await
+		.expect("Failed to parse JSON response");
+	let namespace_id = body["namespace"]["namespace_id"]
+		.as_str()
+		.expect("Missing namespace_id in response");
+
+	let namespace_id =
+		rivet_util::Id::from_str(namespace_id).expect("Failed to parse namespace ID");
+
+	tracing::info!(?namespace_id, ?name, "namespace created");
+
+	namespace_id
+}

package/packages/engine/tests/common/runner.rs

@@ -0,0 +1,134 @@
+use std::{path::Path, time::Duration};
+
+use rivet_util::Id;
+use tokio::process::{Child, Command};
+
+pub struct TestRunner {
+	pub runner_id: Id,
+	internal_port: u16,
+	pub port: u16,
+	handle: Child,
+}
+
+impl TestRunner {
+	pub async fn new(
+		port: u16,
+		namespace_name: &str,
+		key: &str,
+		version: u32,
+		total_slots: u32,
+	) -> Self {
+		let internal_server_port = portpicker::pick_unused_port().expect("runner http server port");
+		let http_server_port = portpicker::pick_unused_port().expect("runner http server port");
+
+		tracing::info!(?internal_server_port, ?http_server_port, "starting runner");
+
+		let manifest_dir = env!("CARGO_MANIFEST_DIR");
+		let runner_script_path =
+			Path::new(manifest_dir).join("../../../sdks/typescript/test-runner/dist/index.js");
+
+		if !runner_script_path.exists() {
+			panic!(
+				"Runner script not found at '{}'. Build it first with `pnpm install && pnpm build -F @rivetkit/engine-test-runner`.",
+				runner_script_path.display(),
+			);
+		}
+
+		let handle = Command::new("node")
+			.arg(runner_script_path)
+			.env("INTERNAL_SERVER_PORT", internal_server_port.to_string())
+			.env("RIVET_NAMESPACE", namespace_name)
+			.env("RIVET_RUNNER_KEY", key.to_string())
+			.env("RIVET_RUNNER_VERSION", version.to_string())
+			.env("RIVET_RUNNER_TOTAL_SLOTS", total_slots.to_string())
+			.env("RIVET_ENDPOINT", format!("http://127.0.0.1:{port}"))
+			.kill_on_drop(true)
+			.spawn()
+			.expect("Failed to execute runner js file, node not installed");
+
+		let runner_id = Self::wait_ready(internal_server_port).await;
+
+		TestRunner {
+			runner_id,
+			internal_port: internal_server_port,
+			port: http_server_port,
+			handle,
+		}
+	}
+
+	async fn wait_ready(port: u16) -> Id {
+		let client = reqwest::Client::new();
+		let mut attempts = 0;
+
+		loop {
+			let res = client
+				.get(format!("http://127.0.0.1:{port}/wait-ready"))
+				.send()
+				.await;
+
+			let response = match res {
+				Ok(x) => x,
+				Err(err) => {
+					if attempts < 10 {
+						attempts += 1;
+						tokio::time::sleep(Duration::from_millis(150)).await;
+						continue;
+					} else {
+						Err(err).expect("Failed to send wait ready request to runner")
+					}
+				}
+			};
+
+			if !response.status().is_success() {
+				if attempts < 10 {
+					attempts += 1;
+					tokio::time::sleep(Duration::from_millis(150)).await;
+					continue;
+				}
+
+				let text = response.text().await.expect("Failed to read response text");
+				panic!("Failed to wait ready for runner: {text}");
+			}
+
+			return response
+				.json()
+				.await
+				.expect("Failed to parse JSON response");
+		}
+	}
+
+	pub async fn has_actor(&self, actor_id: &str) -> bool {
+		let client = reqwest::Client::new();
+		let response = client
+			.get(format!("http://127.0.0.1:{}/has-actor", self.internal_port))
+			.query(&[("actor", actor_id)])
+			.send()
+			.await
+			.expect("Failed to send request has-actor to runner");
+
+		if response.status() == reqwest::StatusCode::NOT_FOUND {
+			return false;
+		}
+
+		if response.status().is_success() {
+			return true;
+		}
+
+		let text = response.text().await.expect("Failed to fetch has-actor");
+		panic!("Failed to fetch has-actor: {text}");
+	}
+
+	pub async fn shutdown(&self) {
+		let client = reqwest::Client::new();
+		let response = client
+			.get(format!("http://127.0.0.1:{}/shutdown", self.internal_port))
+			.send()
+			.await
+			.expect("Failed to send shutdown request to runner");
+
+		if !response.status().is_success() {
+			let text = response.text().await.expect("Failed to read response text");
+			panic!("Failed to shutdown runner: {text}");
+		}
+	}
+}

package/packages/engine/tests/common/test_helpers.rs

@@ -0,0 +1,226 @@
+use std::time::Duration;
+
+use serde_json::json;
+
+// Namespace helpers
+pub async fn setup_test_namespace(guard_port: u16) -> (String, rivet_util::Id) {
+	let random_suffix = rand::random::<u16>();
+	let namespace_name = format!("test-{random_suffix}");
+	let namespace_id = super::create_namespace(&namespace_name, guard_port).await;
+	(namespace_name, namespace_id)
+}
+
+// Setup namespace with runner
+pub async fn setup_test_namespace_with_runner(
+	dc: &super::TestDatacenter,
+) -> (String, rivet_util::Id, super::runner::TestRunner) {
+	let (namespace_name, namespace_id) = setup_test_namespace(dc.guard_port()).await;
+
+	let runner = super::runner::TestRunner::new(
+		dc.guard_port(),
+		&namespace_name,
+		&format!("key-{:012x}", rand::random::<u64>()),
+		1,
+		20,
+	)
+	.await;
+
+	(namespace_name, namespace_id, runner)
+}
+
+pub async fn setup_runner(
+	dc: &super::TestDatacenter,
+	namespace_name: &str,
+	key: &str,
+	version: u32,
+	total_slots: u32,
+) -> super::runner::TestRunner {
+	super::runner::TestRunner::new(dc.guard_port(), &namespace_name, key, version, total_slots)
+		.await
+}
+
+pub async fn cleanup_test_namespace(namespace_id: rivet_util::Id, _guard_port: u16) {
+	// TODO: implement namespace deletion when available
+	tracing::info!(?namespace_id, "namespace cleanup (not implemented)");
+}
+
+// Data generation helpers
+pub fn generate_test_input_data() -> String {
+	base64::Engine::encode(
+		&base64::engine::general_purpose::STANDARD,
+		json!({
+			"test": true,
+			"timestamp": chrono::Utc::now().timestamp_millis(),
+			"data": "test input data"
+		})
+		.to_string(),
+	)
+}
+
+pub fn generate_large_input_data(size_mb: usize) -> String {
+	let size_bytes = size_mb * 1024 * 1024;
+	let data = "x".repeat(size_bytes);
+	base64::Engine::encode(&base64::engine::general_purpose::STANDARD, &data)
+}
+
+pub fn generate_unicode_string() -> String {
+	"test-🦀-🚀-✨-你好-世界".to_string()
+}
+
+pub fn generate_special_chars_string() -> String {
+	"test-!@#$%^&*()_+-=[]{}|;':\",./<>?".to_string()
+}
+
+// Wait helpers
+pub async fn wait_for_actor_propagation(actor_id: &str, timeout_secs: u64) {
+	tracing::info!(?actor_id, ?timeout_secs, "waiting for actor propagation");
+	tokio::time::sleep(Duration::from_secs(timeout_secs)).await;
+}
+
+pub async fn wait_for_eventual_consistency() {
+	tracing::info!("waiting for eventual consistency");
+	tokio::time::sleep(Duration::from_millis(500)).await;
+}
+
+// Actor verification helpers
+pub async fn assert_actor_exists(
+	actor_id: &str,
+	namespace: &str,
+	guard_port: u16,
+) -> serde_json::Value {
+	let response = super::get_actor(actor_id, Some(namespace), guard_port).await;
+	assert!(
+		response.status().is_success(),
+		"Actor {} should exist in namespace {}",
+		actor_id,
+		namespace
+	);
+	response
+		.json()
+		.await
+		.expect("Failed to parse actor response")
+}
+
+pub async fn assert_actor_not_exists(actor_id: &str, guard_port: u16) {
+	let response = super::get_actor(actor_id, None, guard_port).await;
+	assert_eq!(
+		response.status(),
+		400,
+		"Actor {} should not exist (expecting 400 for Actor::NotFound)",
+		actor_id
+	);
+}
+
+pub async fn assert_actor_returns_bad_request(actor_id: &str, guard_port: u16) {
+	let response = super::get_actor(actor_id, None, guard_port).await;
+	assert_eq!(
+		response.status(),
+		400,
+		"Actor {} should return 400 BAD_REQUEST",
+		actor_id
+	);
+}
+
+pub async fn assert_actor_is_destroyed(actor_id: &str, namespace: Option<&str>, guard_port: u16) {
+	let response = super::get_actor(actor_id, namespace, guard_port).await;
+	assert!(
+		response.status().is_success(),
+		"Actor {} should still be retrievable after destroy",
+		actor_id
+	);
+
+	let body: serde_json::Value = response
+		.json()
+		.await
+		.expect("Failed to parse actor response");
+
+	tracing::info!(?body, ?actor_id, "assert_actor_is_destroyed response body");
+
+	assert!(
+		body["actor"]["destroy_ts"].as_i64().is_some(),
+		"Actor {} should have destroy_ts set. Response: {:?}",
+		actor_id,
+		body
+	);
+}
+
+pub async fn assert_actor_in_dc(actor_id_str: &str, expected_dc_label: u16) {
+	// Parse the actor ID to get the datacenter label
+	let actor_id: rivet_util::Id = actor_id_str.parse().expect("Failed to parse actor ID");
+	let actual_dc_label = actor_id.label();
+
+	assert_eq!(
+		actual_dc_label, expected_dc_label,
+		"Actor should be in datacenter {} but is in {}",
+		expected_dc_label, actual_dc_label
+	);
+}
+
+pub async fn assert_actor_in_runner(
+	dc: &super::TestDatacenter,
+	actor_id_str: &str,
+	expected_runner_id: &str,
+) {
+	let actor_id: rivet_util::Id = actor_id_str.parse().expect("Failed to parse actor ID");
+
+	let actors_res = dc
+		.workflow_ctx
+		.op(pegboard::ops::actor::get_runner::Input {
+			actor_ids: vec![actor_id],
+		})
+		.await
+		.unwrap();
+	let runner_id = actors_res.actors.first().map(|x| x.runner_id.to_string());
+
+	assert_eq!(
+		runner_id,
+		Some(expected_runner_id.to_string()),
+		"Actor {actor_id} should be in runner {expected_runner_id} (actually in runner {runner_id:?})",
+	);
+}
+
+pub fn assert_actors_equal(actor1: &serde_json::Value, actor2: &serde_json::Value) {
+	assert_eq!(
+		actor1["actor"]["actor_id"], actor2["actor"]["actor_id"],
+		"Actor IDs should match"
+	);
+	assert_eq!(
+		actor1["actor"]["namespace_id"], actor2["actor"]["namespace_id"],
+		"Namespace IDs should match"
+	);
+	assert_eq!(
+		actor1["actor"]["name"], actor2["actor"]["name"],
+		"Actor names should match"
+	);
+}
+
+// Response assertion helpers
+pub async fn assert_created_response(response: &serde_json::Value, expected_created: bool) {
+	let created = response["created"]
+		.as_bool()
+		.expect("Missing created field in response");
+	assert_eq!(
+		created, expected_created,
+		"Expected created to be {}",
+		expected_created
+	);
+}
+
+pub fn assert_pagination_response(response: &serde_json::Value) {
+	assert!(
+		response.get("actors").is_some() || response.get("names").is_some(),
+		"Response should have actors or names array"
+	);
+	// cursor is optional in pagination responses
+}
+
+// Datacenter helpers
+pub fn get_test_datacenter_names(_ctx: &super::TestCtx) -> Vec<String> {
+	vec!["dc-1".to_string(), "dc-2".to_string()] // Adjust based on actual DC names
+}
+
+pub async fn setup_multi_datacenter_test() -> super::TestCtx {
+	super::TestCtx::new_multi(2)
+		.await
+		.expect("Failed to setup multi-datacenter test")
+}

package/packages/engine/tests/runners_dupe_key.rs

@@ -0,0 +1,27 @@
+mod common;
+
+#[test]
+fn runner_dupe_key() {
+	common::run(common::TestOpts::new(1), |ctx| async move {
+		let (namespace, _) = common::setup_test_namespace(ctx.leader_dc().guard_port()).await;
+
+		let runner1 = common::setup_runner(ctx.leader_dc(), &namespace, "key-1", 1, 1).await;
+		let runner2 = common::setup_runner(ctx.leader_dc(), &namespace, "key-1", 1, 1).await;
+
+		let res = ctx
+			.leader_dc()
+			.workflow_ctx
+			.op(pegboard::ops::runner::get::Input {
+				runner_ids: vec![runner1.runner_id, runner2.runner_id],
+			})
+			.await
+			.unwrap();
+		let mut runners = res.runners.into_iter();
+
+		let runner1 = runners.next().unwrap();
+		let runner2 = runners.next().unwrap();
+
+		assert!(runner1.drain_ts.is_some(), "runner1 not draining");
+		assert!(runner2.drain_ts.is_none(), "runner2 is draining");
+	});
+}

package/packages/engine/tests/runners_version.rs

@@ -0,0 +1,50 @@
+mod common;
+
+#[test]
+fn runner_version_upgrade() {
+	common::run(common::TestOpts::new(1), |ctx| async move {
+		let (namespace, _) = common::setup_test_namespace(ctx.leader_dc().guard_port()).await;
+		let _runner1 = common::setup_runner(ctx.leader_dc(), &namespace, "key-1", 1, 1).await;
+		let runner2 = common::setup_runner(ctx.leader_dc(), &namespace, "key-2", 2, 2).await;
+
+		// Create actor to fill a single slot. This forces the second allocation to either be in the other
+		// runner, or in the same runner IF runner versions are implemented correctly.
+		let actor_id1 = common::create_actor_with_options(
+			common::CreateActorOptions {
+				namespace: namespace.clone(),
+				name: "actor1".to_string(),
+				key: None,
+				datacenter: None,
+				..Default::default()
+			},
+			ctx.leader_dc().guard_port(),
+		)
+		.await;
+
+		assert!(!actor_id1.is_empty(), "Actor ID should not be empty");
+
+		common::assert_actor_exists(&actor_id1, &namespace, ctx.leader_dc().guard_port()).await;
+
+		common::assert_actor_in_runner(ctx.leader_dc(), &actor_id1, &runner2.runner_id.to_string())
+			.await;
+
+		let actor_id2 = common::create_actor_with_options(
+			common::CreateActorOptions {
+				namespace: namespace.clone(),
+				name: "actor2".to_string(),
+				key: None,
+				datacenter: None,
+				..Default::default()
+			},
+			ctx.leader_dc().guard_port(),
+		)
+		.await;
+
+		assert!(!actor_id2.is_empty(), "Actor ID should not be empty");
+
+		common::assert_actor_exists(&actor_id2, &namespace, ctx.leader_dc().guard_port()).await;
+
+		common::assert_actor_in_runner(ctx.leader_dc(), &actor_id2, &runner2.runner_id.to_string())
+			.await;
+	});
+}