@rivetkit/engine 1.0.0 → 2.2.1-pr.4600.b74ff3b

package/packages/epoxy/tests/backfill.rs

@@ -0,0 +1,65 @@
+mod common;
+
+use common::{
+	TestCtx,
+	utils::{
+		read_changelog_entries, read_v2_committed_value, write_legacy_v2_value, write_legacy_value,
+	},
+};
+
+#[tokio::test(flavor = "multi_thread")]
+async fn backfill_migrates_legacy_values_into_v2_and_changelog() {
+	let mut test_ctx = TestCtx::new_with(&[1_u64]).await.unwrap();
+	let replica_id = test_ctx.leader_id;
+	let ctx = test_ctx.get_ctx(replica_id);
+
+	write_legacy_value(ctx, replica_id, b"legacy-key-a", b"legacy-value-a")
+		.await
+		.unwrap();
+	write_legacy_v2_value(ctx, replica_id, b"legacy-key-b", b"legacy-value-b")
+		.await
+		.unwrap();
+
+	let workflow_id = ctx
+		.workflow(epoxy::workflows::backfill::Input {
+			chunk_size: Some(1),
+		})
+		.tag("replica", replica_id)
+		.dispatch()
+		.await
+		.unwrap();
+	let migrated_keys = ctx
+		.workflow::<epoxy::workflows::backfill::Input>(workflow_id)
+		.output()
+		.await
+		.unwrap();
+
+	assert_eq!(migrated_keys, 2);
+
+	let committed_a = read_v2_committed_value(ctx, replica_id, b"legacy-key-a")
+		.await
+		.unwrap()
+		.unwrap();
+	assert_eq!(committed_a.value, b"legacy-value-a");
+	assert_eq!(committed_a.version, 0);
+	assert!(!committed_a.mutable);
+
+	let committed_b = read_v2_committed_value(ctx, replica_id, b"legacy-key-b")
+		.await
+		.unwrap()
+		.unwrap();
+	assert_eq!(committed_b.value, b"legacy-value-b");
+	assert_eq!(committed_b.version, 0);
+	assert!(!committed_b.mutable);
+
+	let changelog_entries = read_changelog_entries(ctx, replica_id).await.unwrap();
+	assert_eq!(changelog_entries.len(), 2);
+	assert!(changelog_entries.iter().any(|entry| {
+		entry.key == b"legacy-key-a" && entry.value == b"legacy-value-a" && entry.version == 0
+	}));
+	assert!(changelog_entries.iter().any(|entry| {
+		entry.key == b"legacy-key-b" && entry.value == b"legacy-value-b" && entry.version == 0
+	}));
+
+	test_ctx.shutdown().await.unwrap();
+}

package/packages/epoxy/tests/backfill_snapshot.rs

@@ -0,0 +1,233 @@
+mod common;
+
+use common::utils::{
+	get_local, read_changelog_entries, read_legacy_value, read_v2_committed_value,
+	write_legacy_value,
+};
+use epoxy::ops::propose::{
+	self, Command, CommandError, CommandKind, Proposal, ProposalResult, SetCommand,
+};
+use test_snapshot::SnapshotTestCtx;
+
+const TEST_KEYS: &[(&[u8], &[u8])] = &[
+	(b"actor:abc123", b"running"),
+	(b"actor:def456", b"stopped"),
+	(b"config:version", b"42"),
+];
+
+/// Propose a key scoped to a single replica.
+async fn propose_local(
+	ctx: &gas::prelude::TestCtx,
+	replica_id: epoxy_protocol::protocol::ReplicaId,
+	key: &[u8],
+	value: &[u8],
+	mutable: bool,
+) -> anyhow::Result<ProposalResult> {
+	ctx.op(propose::Input {
+		proposal: Proposal {
+			commands: vec![Command {
+				kind: CommandKind::SetCommand(SetCommand {
+					key: key.to_vec(),
+					value: Some(value.to_vec()),
+				}),
+			}],
+		},
+		mutable,
+		purge_cache: false,
+		target_replicas: Some(vec![replica_id]),
+	})
+	.await
+}
+
+/// Load a v1 snapshot and verify dual reads, proposals, and backfill across
+/// both replicas.
+#[tokio::test(flavor = "multi_thread")]
+async fn v1_snapshot_dual_read_mutate_and_backfill() {
+	let mut test_ctx = SnapshotTestCtx::from_snapshot_with_coordinator("epoxy-v1")
+		.await
+		.unwrap();
+
+	for &replica_id in &test_ctx.replica_ids() {
+		let ctx = test_ctx.get_ctx(replica_id);
+
+		// -- Phase 1: Verify v1 data exists and dual reads work --
+
+		for &(key, expected_value) in TEST_KEYS {
+			// Legacy subspace should have the v1 data.
+			let legacy_val = read_legacy_value(ctx, replica_id, key).await.unwrap();
+			assert!(
+				legacy_val.is_some(),
+				"replica {replica_id}: legacy value for {:?} should exist",
+				String::from_utf8_lossy(key),
+			);
+
+			// V2 subspace should be empty before backfill.
+			let v2_val = read_v2_committed_value(ctx, replica_id, key).await.unwrap();
+			assert!(
+				v2_val.is_none(),
+				"replica {replica_id}: v2 value for {:?} should not exist before backfill",
+				String::from_utf8_lossy(key),
+			);
+
+			// Dual-read (get_local) should see the v1 data through fallback.
+			let dual_val = get_local(ctx, replica_id, key).await.unwrap();
+			assert_eq!(
+				dual_val.as_deref(),
+				Some(expected_value),
+				"replica {replica_id}: dual read for {:?} should return v1 value",
+				String::from_utf8_lossy(key),
+			);
+		}
+
+		// -- Phase 2: Immutable re-proposal on v1 keys --
+
+		// Re-proposing the same value is idempotent (no-op, no changelog write).
+		let result = propose_local(ctx, replica_id, b"actor:abc123", b"running", false)
+			.await
+			.unwrap();
+		assert!(
+			matches!(result, ProposalResult::Committed),
+			"replica {replica_id}: idempotent re-proposal should succeed: {result:?}",
+		);
+
+		// Proposing a different value for an immutable key returns CommandError.
+		let result = propose_local(ctx, replica_id, b"actor:abc123", b"different", false)
+			.await
+			.unwrap();
+		assert!(
+			matches!(
+				result,
+				ProposalResult::CommandError(CommandError::ExpectedValueDoesNotMatch { .. })
+			),
+			"replica {replica_id}: conflicting immutable proposal should return CommandError: {result:?}",
+		);
+
+		// -- Phase 3: Propose a new mutable key alongside v1 data --
+
+		let result = propose_local(ctx, replica_id, b"new-key", b"new-value", true)
+			.await
+			.unwrap();
+		assert!(
+			matches!(result, ProposalResult::Committed),
+			"replica {replica_id}: new key proposal should commit: {result:?}",
+		);
+
+		let new_val = get_local(ctx, replica_id, b"new-key").await.unwrap();
+		assert_eq!(
+			new_val.as_deref(),
+			Some(b"new-value".as_slice()),
+			"replica {replica_id}: new key should be readable via dual-read",
+		);
+
+		// Mutate it.
+		let result = propose_local(ctx, replica_id, b"new-key", b"updated-value", true)
+			.await
+			.unwrap();
+		assert!(
+			matches!(result, ProposalResult::Committed),
+			"replica {replica_id}: mutation should commit: {result:?}",
+		);
+		let updated = get_local(ctx, replica_id, b"new-key").await.unwrap();
+		assert_eq!(
+			updated.as_deref(),
+			Some(b"updated-value".as_slice()),
+			"replica {replica_id}: mutated key should reflect new value",
+		);
+
+		// -- Phase 4: Run backfill and verify migration --
+
+		let workflow_id = ctx
+			.workflow(epoxy::workflows::backfill::Input {
+				chunk_size: Some(10),
+			})
+			.tag("replica", replica_id)
+			.dispatch()
+			.await
+			.unwrap();
+		let migrated_keys = ctx
+			.workflow::<epoxy::workflows::backfill::Input>(workflow_id)
+			.output()
+			.await
+			.unwrap();
+
+		assert_eq!(
+			migrated_keys, 3,
+			"replica {replica_id}: backfill should migrate all 3 legacy keys",
+		);
+
+		// All original v1 keys should now have v2 committed values.
+		for &(key, expected_value) in TEST_KEYS {
+			let committed = read_v2_committed_value(ctx, replica_id, key)
+				.await
+				.unwrap()
+				.unwrap_or_else(|| {
+					panic!(
+						"replica {replica_id}: v2 value for {:?} should exist after backfill",
+						String::from_utf8_lossy(key),
+					)
+				});
+			assert_eq!(committed.value, expected_value);
+			assert_eq!(committed.version, 0);
+			assert!(!committed.mutable);
+		}
+
+		// Verify changelog has entries for all 3 backfilled keys. Additional
+		// entries from phase-3 proposals and commit propagation may also exist.
+		let changelog = read_changelog_entries(ctx, replica_id).await.unwrap();
+		assert!(
+			changelog.len() >= 5,
+			"replica {replica_id}: expected at least 5 changelog entries (3 backfill + 2 proposals), got {}",
+			changelog.len(),
+		);
+		for &(key, expected_value) in TEST_KEYS {
+			assert!(
+				changelog
+					.iter()
+					.any(|e| e.key == key && e.value == expected_value),
+				"replica {replica_id}: changelog should contain backfill entry for {:?}",
+				String::from_utf8_lossy(key),
+			);
+		}
+
+		// -- Phase 5: Dual read prefers v2 after backfill --
+
+		// Overwrite the legacy value with stale data. Dual read should still
+		// return the v2 value since v2 is checked first in the read order.
+		write_legacy_value(ctx, replica_id, b"actor:abc123", b"stale-legacy")
+			.await
+			.unwrap();
+
+		let dual_val = get_local(ctx, replica_id, b"actor:abc123").await.unwrap();
+		assert_eq!(
+			dual_val.as_deref(),
+			Some(b"running".as_slice()),
+			"replica {replica_id}: dual read should prefer v2 over stale legacy",
+		);
+
+		// -- Phase 6: Propose on a backfilled key --
+
+		// The backfilled keys are immutable (version 0, mutable=false). A same-
+		// value re-proposal should be idempotent. A new-value proposal should
+		// be rejected since the v2 committed value now exists.
+		let result = propose_local(ctx, replica_id, b"actor:def456", b"stopped", false)
+			.await
+			.unwrap();
+		assert!(
+			matches!(result, ProposalResult::Committed),
+			"replica {replica_id}: idempotent proposal on backfilled key should succeed: {result:?}",
+		);
+
+		let result = propose_local(ctx, replica_id, b"actor:def456", b"changed", false)
+			.await
+			.unwrap();
+		assert!(
+			matches!(
+				result,
+				ProposalResult::CommandError(CommandError::ExpectedValueDoesNotMatch { .. })
+			),
+			"replica {replica_id}: conflicting proposal on backfilled key should fail: {result:?}",
+		);
+	}
+
+	test_ctx.shutdown().await.unwrap();
+}

package/packages/epoxy/tests/common/mod.rs

@@ -4,7 +4,6 @@ use epoxy_protocol::protocol::ReplicaId;
 use gas::prelude::{TestCtx as WorkflowTestCtx, *};
 // Note: workflows module no longer exposed in tests
 use rivet_util::Id;
-use serde_json::json;
 use std::collections::HashMap;
 use url::Url;
 
@@ -53,6 +52,33 @@ impl TestCtx {
 		Self::new_with(DEFAULT_REPLICA_IDS).await
 	}
 
+	#[allow(dead_code)]
+	pub async fn new_replica_only_with(replica_ids: &[ReplicaId]) -> anyhow::Result<Self> {
+		gas::ctx::test::setup_logging();
+
+		let leader_id = replica_ids[0];
+
+		let mut test_ctx = TestCtx {
+			test_id: Uuid::new_v4(),
+			leader_id,
+			coordinator_workflow_id: Id::new_v1(leader_id as u16),
+			replica_contexts: HashMap::new(),
+			replica_metadata: HashMap::new(),
+		};
+
+		tracing::info!("adding replicas");
+		for &replica_id in replica_ids {
+			test_ctx.add_replica(replica_id).await?;
+		}
+
+		tracing::info!("starting replicas");
+		for &replica_id in replica_ids {
+			test_ctx.start_replica(replica_id).await?;
+		}
+
+		Ok(test_ctx)
+	}
+
 	pub async fn new_with(replica_ids: &[ReplicaId]) -> anyhow::Result<Self> {
 		gas::ctx::test::setup_logging();
 
@@ -82,9 +108,7 @@ impl TestCtx {
 		// Start coordinator workflow on leader
 		let mut config_sub = test_ctx
 			.get_ctx(leader_id)
-			.subscribe::<epoxy::workflows::coordinator::ConfigChangeMessage>(
-				json!({ "replica": leader_id }),
-			)
+			.subscribe::<epoxy::workflows::coordinator::ConfigChangeMessage>(("replica", leader_id))
 			.await
 			.unwrap();
 
@@ -117,6 +141,15 @@ impl TestCtx {
 			.wf_ctx
 	}
 
+	#[allow(dead_code)]
+	pub fn api_peer_url(&self, replica_id: ReplicaId) -> String {
+		let metadata = self
+			.replica_metadata
+			.get(&replica_id)
+			.expect("replica metadata not found");
+		format!("http://127.0.0.1:{}", metadata.api_peer_port)
+	}
+
 	pub async fn add_replica(&mut self, replica_id: ReplicaId) -> anyhow::Result<()> {
 		tracing::info!(?replica_id, "adding replica");
 
@@ -188,6 +221,22 @@ impl TestCtx {
 		Ok(())
 	}
 
+	#[allow(dead_code)]
+	pub async fn shutdown(&mut self) -> anyhow::Result<()> {
+		// Epoxy commit propagation is fire-and-forget in production. Give local test
+		// broadcasts a moment to finish before tearing replicas down.
+		tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+
+		let mut replica_ids = self.replica_contexts.keys().copied().collect::<Vec<_>>();
+		replica_ids.sort_unstable();
+
+		for replica_id in replica_ids {
+			self.stop_replica(replica_id, false).await?;
+		}
+
+		Ok(())
+	}
+
 	#[allow(dead_code)]
 	pub async fn stop_replica(
 		&mut self,
@@ -219,22 +268,29 @@ impl TestCtx {
 		Ok(())
 	}
 
-	fn build_datacenters(&self) -> Result<Vec<rivet_config::config::topology::Datacenter>> {
-		// Build datacenters using our predetermined metadata
-		let mut datacenters = Vec::new();
-		let replica_ids: Vec<_> = self.replica_metadata.keys().copied().collect();
+	fn build_datacenters(
+		&self,
+	) -> Result<HashMap<String, rivet_config::config::topology::Datacenter>> {
+		// Build datacenters using our predetermined metadata.
+		let mut datacenters = HashMap::new();
+		let mut replica_ids: Vec<_> = self.replica_metadata.keys().copied().collect();
+		replica_ids.sort_unstable();
 
 		for &other_replica_id in &replica_ids {
 			let metadata = &self.replica_metadata[&other_replica_id];
-			datacenters.push(rivet_config::config::topology::Datacenter {
-				name: format!("dc-{}", other_replica_id),
-				datacenter_label: other_replica_id as u16,
-				is_leader: other_replica_id == self.leader_id,
-				peer_url: Url::parse(&format!("http://127.0.0.1:{}", metadata.api_peer_port))?,
-				public_url: Url::parse(&format!("http://127.0.0.1:{}", metadata.guard_port))?,
-				proxy_url: None,
-				valid_hosts: None,
-			});
+			let name = format!("dc-{}", other_replica_id);
+			datacenters.insert(
+				name.clone(),
+				rivet_config::config::topology::Datacenter {
+					name: format!("dc-{}", other_replica_id),
+					datacenter_label: other_replica_id as u16,
+					is_leader: other_replica_id == self.leader_id,
+					peer_url: Url::parse(&format!("http://127.0.0.1:{}", metadata.api_peer_port))?,
+					public_url: Url::parse(&format!("http://127.0.0.1:{}", metadata.guard_port))?,
+					proxy_url: None,
+					valid_hosts: None,
+				},
+			);
 		}
 
 		Ok(datacenters)
@@ -242,7 +298,9 @@ impl TestCtx {
 
 	#[allow(dead_code)]
 	pub fn replica_ids(&self) -> Vec<ReplicaId> {
-		self.replica_metadata.keys().cloned().collect()
+		let mut replica_ids = self.replica_metadata.keys().cloned().collect::<Vec<_>>();
+		replica_ids.sort_unstable();
+		replica_ids
 	}
 }
 
@@ -284,9 +342,7 @@ async fn setup_epoxy_coordinator_wf(
 
 	// Trigger reconfiguration
 	let mut sub = leader_ctx
-		.subscribe::<epoxy::workflows::coordinator::ConfigChangeMessage>(
-			json!({ "replica": leader_id }),
-		)
+		.subscribe::<epoxy::workflows::coordinator::ConfigChangeMessage>(("replica", leader_id))
 		.await?;
 	leader_ctx
 		.signal(epoxy::workflows::coordinator::Reconfigure {})