@rivetkit/engine 1.0.0

package/packages/env/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "rivet-env"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+edition.workspace = true
+
+[dependencies]
+lazy_static.workspace = true
+uuid.workspace = true
+
+[build-dependencies]
+anyhow.workspace = true
+vergen.workspace = true

package/packages/env/build.rs

@@ -0,0 +1,8 @@
+use anyhow::Result;
+use vergen::Emitter;
+
+fn main() -> Result<()> {
+	Emitter::default().emit()?;
+
+	Ok(())
+}

package/packages/env/src/lib.rs

@@ -0,0 +1,30 @@
+use lazy_static::lazy_static;
+use std::env;
+use uuid::Uuid;
+
+lazy_static! {
+	static ref SERVICE_NAME: String =
+		env::var("RIVET_SERVICE_NAME").unwrap_or_else(|_| "rivet".to_string());
+	static ref SERVICE_INSTANCE: String =
+		env::var("RIVET_SERVICE_INSTANCE").unwrap_or_else(|_| Uuid::new_v4().to_string());
+	static ref SOURCE_HASH: String =
+		env::var("RIVET_SOURCE_HASH").unwrap_or_else(|_| "unknown".to_string());
+}
+
+/// Generic name used to differentiate pools of servers.
+pub fn service_name() -> &'static str {
+	&SERVICE_NAME
+}
+
+/// Unique string for this instance of the service.
+pub fn service_instance() -> &'static str {
+	&SERVICE_INSTANCE
+}
+
+/// Returns the source hash. Used to identify if the source code has changed in order to purge
+/// caches.
+///
+/// This will use the `RIVET_SOURCE_HASH` and fall back to the Git hash otherwise.
+pub fn source_hash() -> &'static str {
+	&SOURCE_HASH
+}

package/packages/epoxy/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "epoxy"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+edition.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+axum.workspace = true
+base64.workspace = true
+bytes.workspace = true
+gas.workspace = true
+epoxy-protocol.workspace = true
+futures-util.workspace = true
+lazy_static.workspace = true
+protobuf.workspace = true
+reqwest.workspace = true
+rivet-api-builder.workspace = true
+rivet-config.workspace = true
+rivet-error.workspace = true
+rivet-pools.workspace = true
+rivet-util.workspace = true
+serde.workspace = true
+serde_bare.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+slog-async.workspace = true
+slog-term.workspace = true
+slog.workspace = true
+tokio.workspace = true
+tracing-slog.workspace = true
+tracing.workspace = true
+universaldb.workspace = true
+url.workspace = true
+uuid.workspace = true
+vbare.workspace = true
+
+[dev-dependencies]
+gas.workspace = true
+rivet-runtime.workspace = true
+rivet-test-deps.workspace = true
+portpicker.workspace = true
+tokio.workspace = true
+libc.workspace = true

package/packages/epoxy/README.md

@@ -0,0 +1,158 @@
+# Epoxy
+
+Epoxy is a geo-distributed, strongly-consistent KV store based on the EPaxos protocol.
+
+## EPaxos Overview
+
+### Resources
+
+Paper:
+
+https://www.cs.cmu.edu/~dga/papers/epaxos-sosp2013.pdf
+
+Reference implementations:
+
+- https://github.com/efficient/epaxos/tree/master/src/epaxos
+- https://github.com/pisimulation/epaxos/tree/master
+
+TODO: Correctness issue: https://github.com/otrack/on-epaxos-correctness
+
+Talks:
+
+- Original paper: https://www.youtube.com/watch?v=KxoWlUZNKn8&t=357s
+- Optimization: https://www.youtube.com/watch?v=aysKrS_SAXo
+
+Other resources:
+
+- Atlas follow up: https://software.imdea.org/~gotsman/papers/atlas-eurosys20.pdf
+
+### Glossary
+
+- **Replica** — one of the datacenters running epaxos
+- **Coordinator** — the workflow that manages adding and removing replicas from the cluster
+- **Instance** — per-replica slot (R.1, R.2, …); at most one command is chosen per slot. **This is not the same as a replica.**
+- **Command** — an operation to be ran on the KV store
+- **Command leader** — the replica that received the client’s command (not a fixed global leader)
+- **Interference (interf)** — two commands "interfere" if they conflict (i.e., same key)
+- **Dependencies (deps)** — set of interfering instances that a command depends on.
+- **Sequence number (seq)** — integer used (with deps) to break cycles and order execution.
+- **Log** — each replica’s private log of instances and their attributes/status.
+- **Ballot number** — monotonic proposal id; includes an epoch prefix to order configurations.
+- **Epoch** — configuration/version used to order ballots across reconfigurations.
+
+### Interference Overview
+
+Interference is at the core of EPaxos. Interference is used to determine if a command can be committed on the fast path without executing the full slow-path Paxos protocol.
+
+Some notes related to Rivet's workload requirements relating to interference:
+
+- Any write to the same key is an interference, regardless of the operation kind
+	- The only case where we could optimize this more is if commands are communicative (i.e. simila to UDB's atomic operations)
+- Even though we use CAS for actor keys:
+	- This counts as a write regardless of the previous value, since CAS commands are non-communicative
+	- CAS get written to the log even though it may have no effect on the value
+	- If there is no conflict, the fast-path will return a success since CAS is equivalent to a write
+	- The CAS will be written to the log regardless, but we will determine if the write was a success when committed
+- Consistent reads also count as interference, which is not good for read-heavy operations
+	- Rivet is designed to ensure that values do not change after they are set so we can use optimistic reads instead (see dev-docs/ACTOR_KEY_RESERVATION.md)
+	- This pattern is similar to the lease pattern that the paper proposes
+
+## Workloads Depending On Epoxy
+
+- `pegboard::workflows::actor::actor_keys::Propose`: Uses check or set operation for the actor key's reservation ID
+- `pegboard::ops::get_reservation_for_key`: Uses an optimistic read to find the reservation ID to resolve the actual actor ID
+
+## Architecture
+
+### Coordinator Workflow
+
+The leader datacenter holds a coordinator workflow. This workflow detect config changes and handle:
+
+- Gracefully adding replicas
+- Propagating cluster config changes
+- Epoch number
+
+See `spec/RECONFIGURE.md` for more information.
+
+### Replcia Workflow
+
+Each datacenter has its own replica workflow that is responsible for:
+
+- Downloading instances from other replicas
+- Notifying coordinator when joined cluster
+
+### Messages
+
+All peer-to-peer communication is done via the `POST /v{version}/epoxy/message` using the versioned BARE epoxy protocol.
+
+### Proposals
+
+Proposals are the mechanism for establishing consensus when making a chance to the KV store.
+
+Proposals are not implemented as a workflow since they're:
+
+- Inherently designed to be fallable so they don't benefit from the overhead of workflows
+- They can operate in paralle, while a single replica workflow cannot — though a workflow-per-proposal would technically work fine
+
+See `spec/PROPOSAL.md` for more information.
+
+## Coordinator & Dynamic Topology Configuration
+
+**Do not use `rivet_config::config::Topology` (i.e. `ctx.config().topology()`) from this service.** Instead, read the cluster config propagated from the coordinator with `epoxy::ops::read_cluster_config`.
+
+The leader datacenter runs a coordinator workflow in charge of coordinating config changes across all peers.
+
+This ensures that peers receive config changes at the exact same time.
+
+## Joining Replicas
+
+### Reconfigure Abort
+
+The coordinator will attempt to propagate the config change to all new replicas when a config change is detected.
+
+If the config is invalid (i.e. the replica cannot be reached), the workflow will keep retrying indefinitely.
+
+Reconfigure will be aborted if there is a config change detected. In this case, all changes will be abandoned and the coordinator will attempt to reconfigure with the new config.
+
+### Error in EPaxos Join Specification
+
+The paper has an incorrect explanation of adding new replicas. It describes the join process as:
+
+1. Broadcast _Join_ with new epoch
+2. Download instances from other replicas
+3. Replica begins voting
+
+The issue with this is that the quorum count will have changed, but the replica cannot vote yet. This means that if you add too many replicas at the same time you will cause a complete outage until the new replicas have downloaded all instances and began voting.
+
+Instead, we opt to:
+
+1. Add replica as a learner (starts receiving commits)
+2. Download instances from other replicas
+3. Corodinator increments voting once node is active
+4. Replica begins voting
+
+This enables clusters to add many new replicas at once without causing an outage.
+
+### Misc Notes
+
+- Downloading replicas receive commits from other replicas even though they are not part of the quorum while they are downloading. This ensures that they have received all messages, even though they are not part of the quorum.
+
+## Future Optimizations
+
+## Integrate Explicit Prepare
+
+Explicit Prepare is	implemented but not integrated anywhere.
+
+### Avoiding Execution Livelock (§4.9)
+
+The paper recommends prioritizing earlier commands. We don't have functionality to do this right now, but the current design has almost no contention so this is not a concern.
+
+### Batching (§7.6)
+
+The paper recommends batching requests for a "9x" improvement in throughput. We don't need to do this since we (a) execute requests in parallel, (b) assume low contention, and (c) built the system to be stateless + scale horizontally with UDB.
+
+### Optimized Epaxos (§4.4)
+
+The paper recommends an optimize recovery mechanism that requires less replicas for consensus.
+
+

package/packages/epoxy/spec/KEYS.md

@@ -0,0 +1,33 @@
+# Keys
+
+## Key Hierarchy
+
+```
+/rivet/epoxy/
+	replica/{replica_id}/
+		config = {Config}                            # Replica configuration with epoch and peer info
+		instance_number = {u32}                      # Track instance number for next instance
+		current_ballot = {Ballot}                    # Current ballot number for this replica
+		instance_ballot/{replica}/{slot} = {Ballot}  # Highest ballot seen for each instance
+		log/{replica}/{slot}/entry = {LogEntry}
+		key_instance/{key}/{replica}/{slot} = ()     # Used for checking interference
+		key_value/{key}/
+			committed_value = Vec<u8>                # Value written on commit
+			optimistic_cached_value = Vec<u8>        # Value read from remote datacenters in kv::get_optimistic
+```
+
+### Design Notes
+
+- `key_instances/{key}` is stored separately from `keys/{key}` because we need to efficiently be able to do a range scan over key_instances in order to find interference, so we need to minimize keys in that subspace
+- `instance_number` should be replaced with `version_stamp` to reduce contention
+
+### Contention
+
+- `lead_consensus`
+	- `instance_number`
+	- `current_ballot`
+- `find_interference` & `update_log`
+	- `key_instances/{key}/({replica,slot})`
+- `validate_ballot_for_instance`
+	- `instance_ballot/{replica}/{slot}`
+

package/packages/epoxy/spec/PROPOSAL.md

@@ -0,0 +1,125 @@
+# EPaxos Proposal Specification
+
+Based on Figure 2 & 3.
+
+## Phase 1: Establish ordering constraints
+
+**Replica L on receiving Request(γ) from a client becomes the designated leader for command γ (steps 2, 3 and 4 executed atomically):**
+
+1: increment instance number i_L ← i_L + 1
+   {Interf_L,γ is the set of instances Q.j such that the command recorded in cmds_L[Q][j] interferes w/ γ}
+   -> src/replica/messages/lead_consensus.rs
+2: seq_γ ← 1+ max ({cmds_L[Q][j].seq | Q.j ∈ Interf_L,γ} ∪ {0})
+   -> src/replica/messages/lead_consensus.rs
+3: deps_γ ← Interf_L,γ
+   -> src/replica/messages/lead_consensus.rs
+4: cmds_L[L][i_L] ← (γ, seq_γ, deps_γ, pre-accepted)
+   -> src/replica/messages/lead_consensus.rs
+5: send PreAccept(γ, seq_γ, deps_γ, L.i_L) to all other replicas in F, where F is a fast quorum that includes L
+   -> src/replica/propose.rs
+
+**Any replica R, on receiving PreAccept(γ, seq_γ, deps_γ, L.i) (steps 6, 7 and 8 executed atomically):**
+
+6: update seq_γ ← max({seq_γ} ∪ {1 + cmds_R[Q][j].seq | Q.j ∈ Interf_R,γ})
+   -> src/replica/messages/pre_accept.rs
+7: update deps_γ ← deps_γ ∪ Interf_R,γ
+   -> src/replica/messages/pre_accept.rs
+8: cmds_R[L][i] ← (γ, seq_γ, deps_γ, pre-accepted)
+   -> src/replica/messages/pre_accept.rs
+9: reply PreAcceptOK(γ, seq_γ, deps_γ, L.i) to L
+   -> src/replica/messages/pre_accept.rs
+
+**Replica L (command leader for γ), on receiving at least ⌊N/2⌋ PreAcceptOK responses:**
+
+10: if received PreAcceptOK's from all replicas in F \ {L}, with seq_γ and deps_γ the same in all replies (for some fast quorum F) then
+    -> src/replica/messages/decide_path.rs
+11:    run Commit phase for (γ, seq_γ, deps_γ) at L.i
+       -> src/replica/messages/decide_path.rs
+12: else
+    -> src/replica/messages/decide_path.rs
+13:    update deps_γ ← Union(deps_γ from all replies)
+       -> src/replica/messages/decide_path.rs
+14:    update seq_γ ← max({seq_γ of all replies})
+       -> src/replica/messages/decide_path.rs
+15:    run Paxos-Accept phase for (γ, seq_γ, deps_γ) at L.i
+       -> src/replica/messages/decide_path.rs
+
+## Phase 2: Paxos-Accept
+
+**Command leader L, for (γ, seq_γ, deps_γ) at instance L.i:**
+
+16: cmds_L[L][i] ← (γ, seq_γ, deps_γ, accepted)
+    -> src/replica/messages/accepted.rs
+17: send Accept(γ, seq_γ, deps_γ, L.i) to at least ⌊N/2⌋ other replicas
+    -> src/replica/propose.rs
+
+**Any replica R, on receiving Accept(γ, seq_γ, deps_γ, L.i):**
+
+18: cmds_R[L][i] ← (γ, seq_γ, deps_γ, accepted)
+    -> src/replica/messages/accept.rs
+19: reply AcceptOK(γ, L.i) to L
+    -> src/replica/messages/accept.rs
+
+**Command leader L, on receiving at least ⌊N/2⌋ AcceptOK's:**
+
+20: run Commit phase for (γ, seq_γ, deps_γ) at L.i
+    -> src/replica/propose.rs
+
+## Commit
+
+**Command leader L, for (γ, seq_γ, deps_γ) at instance L.i:**
+
+21: cmds_L[L][i] ← (γ, seq_γ, deps_γ, committed)
+    -> src/replica/messages/committed.rs
+22: send commit notification for γ to client
+    -> src/replica/propose.rs (via return value)
+23: send Commit(γ, seq_γ, deps_γ, L.i) to all other replicas
+    -> src/replica/propose.rs
+
+**Any replica R, on receiving Commit(γ, seq_γ, deps_γ, L.i):**
+
+24: cmds_R[L][i] ← (γ, seq_γ, deps_γ, committed)
+    -> src/replica/messages/commit.rs
+
+## Explicit Prepare
+
+**Replica Q for instance L.i of potentially failed replica L**
+
+25: increment ballot number to epoch.(b+1).Q, (where epoch.b.R is the highest ballot number Q is aware of in instance L.i)
+    -> src/ops/explicit_prepare.rs
+26: send Prepare(epoch.(b+1).Q, L.i) to all replicas (including self) and wait for at least ⌊N/2⌋ + 1 replies
+    -> src/ops/explicit_prepare.rs
+27: let R be the set of replies w/ the highest ballot number
+    -> src/ops/explicit_prepare.rs
+28: if R contains a (γ, seq_γ, deps_γ, committed) then
+    -> src/ops/explicit_prepare.rs
+29:    run Commit phase for (γ, seq_γ, deps_γ) at L.i
+    -> src/ops/explicit_prepare.rs
+30: else if R contains an (γ, seq_γ, deps_γ, accepted) then
+    -> src/ops/explicit_prepare.rs
+31:    run Paxos-Accept phase for (γ, seq_γ, deps_γ) at L.i
+    -> src/ops/explicit_prepare.rs
+32: else if R contains at least ⌊N/2⌋ identical replies (γ, seq_γ, deps_γ, pre-accepted) for the default ballot epoch.0.L of instance L.i, and none of those replies is from L then
+    -> src/ops/explicit_prepare.rs
+33:    run Paxos-Accept phase for (γ, seq_γ, deps_γ) at L.i
+    -> src/ops/explicit_prepare.rs
+34: else if R contains at least one (γ, seq_γ, deps_γ, pre-accepted) then
+    -> src/ops/explicit_prepare.rs
+35:    start Phase 1 (at line 2) for γ at L.i, avoid fast path
+    -> src/ops/explicit_prepare.rs
+36: else
+    -> src/ops/explicit_prepare.rs
+37:    start Phase 1 (at line 2) for no-op at L.i, avoid fast path
+    -> src/ops/explicit_prepare.rs
+
+**Replica R, on receiving Prepare(epoch.b.Q, L.i) from Q**
+
+38: if epoch.b.Q is larger than the most recent ballot number epoch.x.Y accepted for instance L.i then
+    -> src/replica/messages/prepare.rs
+39:    reply PrepareOK(cmds_R[L][i], epoch.x.Y, L.i)
+    -> src/replica/messages/prepare.rs
+40: else
+    -> src/replica/messages/prepare.rs
+41:    reply NACK
+    -> src/replica/messages/prepare.rs
+

package/packages/epoxy/spec/RECONFIGURE.md

@@ -0,0 +1,40 @@
+# EPaxos Reconfiguration Specification
+
+This specification describes the reconfiguration process for EPaxos, specifically how new nodes join the cluster.
+
+## Process
+
+**Coordinator checks for configuration changes:**
+
+1: Read current topology from rivet_config
+2: Compare with existing cluster state to find new replicas
+3: if new replicas found then
+4:    Perform health checks on all new replicas
+9:    Add new replicas to cluster config with status=Joining
+10:   Send BeginLearningRequest to each new replica with current cluster config
+
+**New replica R on receiving BeginLearningSignal from coordinator:**
+
+14: for each existing replica E in config (excluding self) do
+15:    Download log entries from replica E
+16:    for each log entry in downloaded chunk do
+17:       Replay log entry (without committing to KV)
+18: Recover key-value state from replayed log entries
+19:    for each unique key in log entries do
+20:       Find all committed instances that touched this key
+21:       Determine final value using highest sequence leaf entry
+22:       Commit recovered value to KV store
+23: Send CoordinatorUpdateReplicaStatusRequest(status=Active) to coordinator
+
+**Coordinator on receiving replica status update to Active:**
+
+24: Update replica status in cluster config
+25: if status changed from non-Active to Active then
+26:    increment epoch
+27:    Send UpdateConfigRequest(new_config) to all replicas
+
+**Any replica R on receiving UpdateConfigRequest(new_config):**
+
+28: Update local cluster configuration to new_config
+29: Recognize new active members in cluster
+

package/packages/epoxy/src/consts.rs

@@ -0,0 +1,42 @@
+use std::time::Duration;
+
+/// Timeout for HTTP request to a peer datacenter.
+pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
+
+/// Response is `list<Instance>
+///
+/// ## Based on size
+///
+/// Instance = 250 bytes for an avg key size (bare minimum 115 bytes)
+///
+/// Reasonable total download size = 64 MiB
+///
+/// That's ~268k instances
+///
+/// ## Based on FDB limitation
+///
+/// FDB starts having performance issues between 10k-100k keys in a single range read based on
+/// rudimentary tests.
+///
+/// Can likely parallelize FDB transactions to return larger chunks.
+///
+/// ## Main limitation
+///
+/// FDB is the main limitation here, so we opt to use an FDB-friendly chunk size.
+///
+/// ## Estimated download time
+///
+/// Our current impl is incredily slow since we do not parallelize downloads.
+///
+/// Assume we have 10M keys with a 5k chunk size. This results in 2000 round trips.
+///
+/// If we have a latency of 200 ms (worst case), this takes 400 seconds total to propagate.
+pub const DOWNLOAD_INSTANCE_COUNT: u64 = 5_000;
+
+/// Number of keys to recover in a single chunk during the recovery process.
+///
+/// Keys are recovered in chunks to avoid memory issues and provide progress updates.
+/// The recovery process scans all key-instance markers and rebuilds the committed values.
+///
+/// This number is relatively small since we have to do a lot of operations per key.
+pub const RECOVER_KEY_CHUNK_SIZE: u64 = 500;

package/packages/epoxy/src/errors.rs

@@ -0,0 +1,21 @@
+use rivet_error::*;
+use serde::{Deserialize, Serialize};
+
+#[derive(RivetError, Debug, Deserialize, Serialize)]
+#[error("kv")]
+pub enum Kv {
+	#[error("no_leader_elected", "No leader has been elected yet.")]
+	NoLeaderElected,
+
+	#[error("leader_forwarding_failed", "Failed to forward request to leader.")]
+	LeaderForwardingFailed,
+
+	#[error(
+		"response_channel_closed",
+		"Failed to receive KV response, channel closed."
+	)]
+	ResponseChannelClosed,
+
+	#[error("not_leader", "Current node is not the leader.")]
+	NotLeader,
+}