@rivetkit/engine 1.0.0 → 2.2.1-pr.4600.b74ff3b

package/packages/gasoline/src/db/kv/mod.rs

@@ -5,11 +5,11 @@ use std::{
 	collections::{HashMap, HashSet},
 	hash::{DefaultHasher, Hash, Hasher},
 	sync::Arc,
-	time::Instant,
+	time::{Duration, Instant},
 };
 
 use anyhow::{Context, Result};
-use futures_util::{StreamExt, TryStreamExt, stream::BoxStream};
+use futures_util::{StreamExt, TryStreamExt, future::try_join_all, stream::BoxStream};
 use rivet_util::Id;
 use rivet_util::future::CustomInstrumentExt;
 use serde_json::json;
@@ -23,20 +23,20 @@ use universaldb::{
 	value::Value,
 };
 
-use rivet_metrics::KeyValue;
-
 use super::{BumpSubSubject, Database, PulledWorkflowData, SignalData, WorkflowData};
 use crate::{
 	error::{WorkflowError, WorkflowResult},
 	history::{
 		event::{
 			ActivityEvent, Event, EventData, EventType, LoopEvent, MessageSendEvent, RemovedEvent,
-			SignalEvent, SignalSendEvent, SleepEvent, SleepState, SubWorkflowEvent,
+			SignalSendEvent, SignalsEvent, SleepEvent, SleepState, SubWorkflowEvent,
+			VersionCheckEvent,
 		},
 		location::Location,
 	},
 	metrics,
 	worker::PING_INTERVAL,
+	workflow::PruneVariant,
 };
 
 mod debug;
@@ -48,11 +48,13 @@ mod system;
 const WORKER_LOST_THRESHOLD_MS: i64 = rivet_util::duration::seconds(30);
 /// How long before overwriting an existing metrics lock.
 const METRICS_LOCK_TIMEOUT_MS: i64 = rivet_util::duration::seconds(30);
+const EARLY_TXN_TIMEOUT: Duration = Duration::from_millis(2500);
 
 pub struct DatabaseKv {
+	config: rivet_config::Config,
 	pools: rivet_pools::Pools,
 	subspace: universaldb::utils::Subspace,
-	system: Mutex<system::SystemInfo>,
+	system: Arc<Mutex<system::SystemInfo>>,
 }
 
 impl DatabaseKv {
@@ -142,13 +144,17 @@ impl DatabaseKv {
 		);
 
 		let workflow_name_key = keys::workflow::NameKey::new(workflow_id);
+		let wake_signal_key =
+			keys::workflow::WakeSignalKey::new(workflow_id, signal_name.to_string());
+
+		let (workflow_name_entry, wake_signal_entry) = tokio::try_join!(
+			tx.get(&self.subspace.pack(&workflow_name_key), Serializable),
+			tx.get(&self.subspace.pack(&wake_signal_key), Serializable),
+		)?;
 
 		// TODO: This does not check if the workflow is silenced
 		// Check if the workflow exists
-		let Some(workflow_name_entry) = tx
-			.get(&self.subspace.pack(&workflow_name_key), Serializable)
-			.await?
-		else {
+		let Some(workflow_name_entry) = workflow_name_entry else {
 			return Err(WorkflowError::WorkflowNotFound.into());
 		};
 
@@ -200,15 +206,8 @@ impl DatabaseKv {
 			&workflow_id_key.serialize(workflow_id)?,
 		);
 
-		let wake_signal_key =
-			keys::workflow::WakeSignalKey::new(workflow_id, signal_name.to_string());
-
 		// If the workflow currently has a wake signal key for this signal, wake it
-		if tx
-			.get(&self.subspace.pack(&wake_signal_key), Serializable)
-			.await?
-			.is_some()
-		{
+		if wake_signal_entry.is_some() {
 			let mut wake_condition_key = keys::wake::WorkflowWakeConditionKey::new(
 				workflow_name,
 				workflow_id,
@@ -226,7 +225,15 @@ impl DatabaseKv {
 		update_metric(
 			&tx.with_subspace(self.subspace.clone()),
 			None,
-			Some(keys::metric::GaugeMetric::SignalPending(
+			Some(keys::metric::Metric::SignalPending2(
+				signal_name.to_string(),
+			)),
+		);
+		update_wf_metric(
+			&tx.with_subspace(self.subspace.clone()),
+			workflow_id,
+			None,
+			Some(keys::workflow::Metric::SignalPending(
 				signal_name.to_string(),
 			)),
 		);
@@ -338,7 +345,7 @@ impl DatabaseKv {
 		update_metric(
 			&tx,
 			None,
-			Some(keys::metric::GaugeMetric::WorkflowSleeping(
+			Some(keys::metric::Metric::WorkflowSleeping(
 				workflow_name.to_string(),
 			)),
 		);
@@ -420,11 +427,16 @@ impl Database for DatabaseKv {
 		std::time::Duration::from_secs(4)
 	}
 
-	async fn from_pools(pools: rivet_pools::Pools) -> anyhow::Result<Arc<Self>> {
+	async fn new(
+		config: rivet_config::Config,
+		pools: rivet_pools::Pools,
+	) -> anyhow::Result<Arc<Self>> {
+		metrics::DB_INSTANCE.inc();
 		Ok(Arc::new(DatabaseKv {
+			config,
 			pools,
 			subspace: universaldb::utils::Subspace::new(&(RIVET, GASOLINE, KV)),
-			system: Mutex::new(system::SystemInfo::new()),
+			system: system::SystemInfo::get(),
 		}))
 	}
 
@@ -439,6 +451,7 @@ impl Database for DatabaseKv {
 			.map_err(WorkflowError::PoolsGeneric)?
 			.subscribe(&subjects::convert(subject))
 			.await
+			.context("failed to subscribe to bump sub")
 			.map_err(|x| WorkflowError::CreateSubscription(x.into()))?;
 
 		let stream = async_stream::stream! {
@@ -466,9 +479,10 @@ impl Database for DatabaseKv {
 			.map_err(WorkflowError::PoolsGeneric)?
 			.run(|tx| {
 				async move {
+					let start = Instant::now();
 					let now = rivet_util::timestamp::now();
 
-					let mut last_ping_cache: Vec<(Id, i64)> = Vec::new();
+					let mut last_ping_cache = HashMap::<Id, i64>::new();
 					let mut lost_worker_ids = HashSet::new();
 					let mut expired_workflow_count = 0;
 
@@ -487,7 +501,16 @@ impl Database for DatabaseKv {
 						Snapshot,
 					);
 
-					while let Some(lease_key_entry) = stream.try_next().await? {
+					loop {
+						if start.elapsed() > EARLY_TXN_TIMEOUT {
+							tracing::warn!("timed out processing expired leases");
+							break;
+						}
+
+						let Some(lease_key_entry) = stream.try_next().await? else {
+							break;
+						};
+
 						let lease_key = self
 							.subspace
 							.unpack::<keys::workflow::LeaseKey>(lease_key_entry.key())?;
@@ -496,8 +519,8 @@ impl Database for DatabaseKv {
 						let last_ping_ts_key = keys::worker::LastPingTsKey::new(worker_id);
 
 						// Get last ping of worker for this lease
-						let last_ping_ts = if let Some((_, last_ping_ts)) =
-							last_ping_cache.iter().find(|(k, _)| k == &worker_id)
+						let last_ping_ts = if let Some(last_ping_ts) =
+							last_ping_cache.get(&worker_id)
 						{
 							*last_ping_ts
 						} else if let Some(last_ping_entry) = tx
@@ -512,12 +535,12 @@ impl Database for DatabaseKv {
 							let last_ping_ts = last_ping_ts_key.deserialize(&last_ping_entry)?;
 
 							// Update cache
-							last_ping_cache.push((worker_id, last_ping_ts));
+							last_ping_cache.insert(worker_id, last_ping_ts);
 
 							last_ping_ts
 						} else {
 							// Update cache
-							last_ping_cache.push((worker_id, 0));
+							last_ping_cache.insert(worker_id, 0);
 
 							0
 						};
@@ -563,10 +586,10 @@ impl Database for DatabaseKv {
 
 							update_metric(
 								&tx.with_subspace(self.subspace.clone()),
-								Some(keys::metric::GaugeMetric::WorkflowActive(
+								Some(keys::metric::Metric::WorkflowActive(
 									workflow_name.to_string(),
 								)),
-								Some(keys::metric::GaugeMetric::WorkflowSleeping(
+								Some(keys::metric::Metric::WorkflowSleeping(
 									workflow_name.to_string(),
 								)),
 							);
@@ -583,6 +606,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("clear_expired_leases_tx"))
 			.await
+			.context("failed to clear expired leases")
 			.map_err(WorkflowError::Udb)?;
 
 		if expired_workflow_count != 0 {
@@ -633,9 +657,12 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("acquire_lock_tx"))
 			.await
+			.context("failed to acquire metrics lock")
 			.map_err(WorkflowError::Udb)?;
 
 		if acquired_lock {
+			metrics::WORKER_LAST_METRICS_PUBLISH.set(rivet_util::timestamp::now());
+
 			let entries = self
 				.pools
 				.udb()
@@ -643,9 +670,8 @@ impl Database for DatabaseKv {
 				.run(|tx| async move {
 					let tx = tx.with_subspace(self.subspace.clone());
 
-					let metrics_subspace = self
-						.subspace
-						.subspace(&keys::metric::GaugeMetricKey::subspace());
+					let metrics_subspace =
+						self.subspace.subspace(&keys::metric::MetricKey::subspace());
 					tx.get_ranges_keyvalues(
 						universaldb::RangeOption {
 							mode: StreamingMode::WantAll,
@@ -654,7 +680,7 @@ impl Database for DatabaseKv {
 						Serializable,
 					)
 					.map(|res| match res {
-						Ok(entry) => tx.read_entry::<keys::metric::GaugeMetricKey>(&entry),
+						Ok(entry) => tx.read_entry::<keys::metric::MetricKey>(&entry),
 						Err(err) => Err(err.into()),
 					})
 					.try_collect::<Vec<_>>()
@@ -662,82 +688,78 @@ impl Database for DatabaseKv {
 				})
 				.custom_instrument(tracing::info_span!("read_metrics_tx"))
 				.await
+				.context("failed to read metrics")
 				.map_err(WorkflowError::Udb)?;
 
-			let mut total_workflow_counts: Vec<(String, usize)> = Vec::new();
+			let mut total_workflow_counts: Vec<(String, i64)> = Vec::new();
 
 			for (key, count) in entries {
 				match key.metric {
-					keys::metric::GaugeMetric::WorkflowActive(workflow_name) => {
-						metrics::WORKFLOW_ACTIVE.record(
-							count as u64,
-							&[KeyValue::new("workflow_name", workflow_name.clone())],
-						);
+					keys::metric::Metric::WorkflowActive(workflow_name) => {
+						metrics::WORKFLOW_ACTIVE
+							.with_label_values(&[workflow_name.as_str()])
+							.set(count);
 
 						if let Some(entry) = total_workflow_counts
 							.iter_mut()
 							.find(|(name, _)| name == &workflow_name)
 						{
-							entry.1 += 1;
+							entry.1 += count;
 						} else {
-							total_workflow_counts.push((workflow_name, 1));
+							total_workflow_counts.push((workflow_name, count));
 						}
 					}
-					keys::metric::GaugeMetric::WorkflowSleeping(workflow_name) => {
-						metrics::WORKFLOW_SLEEPING.record(
-							count as u64,
-							&[KeyValue::new("workflow_name", workflow_name.clone())],
-						);
+					keys::metric::Metric::WorkflowSleeping(workflow_name) => {
+						metrics::WORKFLOW_SLEEPING
+							.with_label_values(&[workflow_name.as_str()])
+							.set(count);
 
 						if let Some(entry) = total_workflow_counts
 							.iter_mut()
 							.find(|(name, _)| name == &workflow_name)
 						{
-							entry.1 += 1;
+							entry.1 += count;
 						} else {
-							total_workflow_counts.push((workflow_name, 1));
+							total_workflow_counts.push((workflow_name, count));
 						}
 					}
-					keys::metric::GaugeMetric::WorkflowDead(workflow_name, error) => {
-						metrics::WORKFLOW_DEAD.record(
-							count as u64,
-							&[
-								KeyValue::new("workflow_name", workflow_name.clone()),
-								KeyValue::new("error", error),
-							],
-						);
+					keys::metric::Metric::WorkflowDead(workflow_name, error) => {
+						metrics::WORKFLOW_DEAD
+							.with_label_values(&[workflow_name.as_str(), error.as_str()])
+							.set(count);
 
 						if let Some(entry) = total_workflow_counts
 							.iter_mut()
 							.find(|(name, _)| name == &workflow_name)
 						{
-							entry.1 += 1;
+							entry.1 += count;
 						} else {
-							total_workflow_counts.push((workflow_name, 1));
+							total_workflow_counts.push((workflow_name, count));
 						}
 					}
-					keys::metric::GaugeMetric::WorkflowComplete(workflow_name) => {
+					keys::metric::Metric::WorkflowComplete(workflow_name) => {
 						if let Some(entry) = total_workflow_counts
 							.iter_mut()
 							.find(|(name, _)| name == &workflow_name)
 						{
-							entry.1 += 1;
+							entry.1 += count;
 						} else {
-							total_workflow_counts.push((workflow_name, 1));
+							total_workflow_counts.push((workflow_name, count));
 						}
 					}
-					keys::metric::GaugeMetric::SignalPending(signal_name) => {
+					keys::metric::Metric::SignalPending(_) => {}
+					keys::metric::Metric::SignalPending2(signal_name) => {
 						metrics::SIGNAL_PENDING
-							.record(count as u64, &[KeyValue::new("signal_name", signal_name)]);
+							.with_label_values(&[signal_name.as_str()])
+							.set(count);
 					}
 				}
 			}
 
 			for (workflow_name, count) in total_workflow_counts {
-				metrics::WORKFLOW_TOTAL.record(
-					count as u64,
-					&[KeyValue::new("workflow_name", workflow_name.clone())],
-				);
+				metrics::WORKFLOW_TOTAL
+					.with_label_values(&[workflow_name.as_str()])
+					.set(count);
 			}
 
 			// Clear lock
@@ -752,6 +774,7 @@ impl Database for DatabaseKv {
 				})
 				.custom_instrument(tracing::info_span!("clear_lock_tx"))
 				.await
+				.context("failed to release metrics lock")
 				.map_err(WorkflowError::Udb)?;
 		}
 
@@ -759,11 +782,14 @@ impl Database for DatabaseKv {
 	}
 
 	#[tracing::instrument(skip_all)]
-	async fn update_worker_ping(&self, worker_id: Id) -> WorkflowResult<()> {
-		metrics::WORKER_LAST_PING.record(
-			rivet_util::timestamp::now() as u64,
-			&[KeyValue::new("worker_id", worker_id.to_string())],
-		);
+	async fn update_worker_ping(
+		&self,
+		worker_id: Id,
+		update_active_idx: bool,
+	) -> WorkflowResult<()> {
+		metrics::WORKER_LAST_PING
+			.with_label_values(&[&worker_id.to_string()])
+			.set(rivet_util::timestamp::now());
 
 		self.pools
 			.udb()
@@ -774,24 +800,27 @@ impl Database for DatabaseKv {
 				let last_ping_ts = rivet_util::timestamp::now();
 				let last_ping_ts_key = keys::worker::LastPingTsKey::new(worker_id);
 
-				if let Some(last_last_ping_ts) =
-					tx.read_opt(&last_ping_ts_key, Serializable).await?
-				{
-					let active_worker_idx_key =
-						keys::worker::ActiveWorkerIdxKey::new(last_last_ping_ts, worker_id);
-					tx.delete(&active_worker_idx_key);
-				}
-
 				tx.write(&last_ping_ts_key, last_ping_ts)?;
 
-				let active_worker_idx_key =
-					keys::worker::ActiveWorkerIdxKey::new(last_ping_ts, worker_id);
-				tx.write(&active_worker_idx_key, ())?;
+				if update_active_idx {
+					if let Some(last_last_ping_ts) =
+						tx.read_opt(&last_ping_ts_key, Serializable).await?
+					{
+						let active_worker_idx_key =
+							keys::worker::ActiveWorkerIdxKey::new(last_last_ping_ts, worker_id);
+						tx.delete(&active_worker_idx_key);
+					}
+
+					let active_worker_idx_key =
+						keys::worker::ActiveWorkerIdxKey::new(last_ping_ts, worker_id);
+					tx.write(&active_worker_idx_key, ())?;
+				}
 
 				Ok(())
 			})
 			.custom_instrument(tracing::info_span!("update_worker_ping_tx"))
 			.await
+			.context("failed to update worker ping")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -817,6 +846,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("mark_worker_inactive_tx"))
 			.await
+			.context("failed to mark worker inactive")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -850,6 +880,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("dispatch_workflow_tx"))
 			.await
+			.context("failed to dispatch workflow")
 			.map_err(WorkflowError::Udb)?;
 
 		self.bump(BumpSubSubject::Worker);
@@ -869,6 +900,7 @@ impl Database for DatabaseKv {
 						.map(|workflow_id| {
 							let tx = tx.clone();
 							async move {
+								let name_key = keys::workflow::NameKey::new(workflow_id);
 								let input_key = keys::workflow::InputKey::new(workflow_id);
 								let input_subspace = self.subspace.subspace(&input_key);
 								let state_key = keys::workflow::StateKey::new(workflow_id);
@@ -880,11 +912,13 @@ impl Database for DatabaseKv {
 
 								// Read input and output
 								let (
+									name_entry,
 									input_chunks,
 									state_chunks,
 									output_chunks,
 									has_wake_condition_entry,
 								) = tokio::try_join!(
+									tx.get(&self.subspace.pack(&name_key), Serializable),
 									tx.get_ranges_keyvalues(
 										universaldb::RangeOption {
 											mode: StreamingMode::WantAll,
@@ -934,6 +968,9 @@ impl Database for DatabaseKv {
 
 									Ok(Some(WorkflowData {
 										workflow_id,
+										name: name_key.deserialize(
+											&name_entry.context("name key should exist")?,
+										)?,
 										input,
 										state,
 										output,
@@ -951,11 +988,12 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("get_workflow_tx"))
 			.await
+			.context("failed to get workflows")
 			.map_err(WorkflowError::Udb)
 	}
 
 	/// Returns the first incomplete workflow with the given name and tags, first meaning the one with the
-	/// lowest id value (interpreted as u128) because its in a KV store. There is no way to get any other
+	/// lowest id value (by internal representation) because its in a KV store. There is no way to get any other
 	/// workflow besides the first.
 	#[tracing::instrument(skip_all, fields(%workflow_name))]
 	async fn find_workflow(
@@ -972,17 +1010,45 @@ impl Database for DatabaseKv {
 			.run(|tx| async move { self.find_workflow_inner(workflow_name, tags, &tx).await })
 			.custom_instrument(tracing::info_span!("find_workflow_tx"))
 			.await
+			.context("failed to find workflow")
 			.map_err(WorkflowError::Udb)?;
 
 		let dt = start_instant.elapsed().as_secs_f64();
-		metrics::FIND_WORKFLOWS_DURATION.record(
-			dt,
-			&[KeyValue::new("workflow_name", workflow_name.to_string())],
-		);
+		metrics::FIND_WORKFLOWS_DURATION
+			.with_label_values(&[workflow_name])
+			.observe(dt);
 
 		Ok(workflow_id)
 	}
 
+	#[tracing::instrument(skip_all)]
+	async fn find_workflows(
+		&self,
+		queries: &[(&str, serde_json::Value)],
+	) -> WorkflowResult<Vec<Option<Id>>> {
+		let start_instant = Instant::now();
+
+		let workflow_ids = self
+			.pools
+			.udb()
+			.map_err(WorkflowError::PoolsGeneric)?
+			.run(|tx| async move {
+				let futures = queries.iter().map(|(workflow_name, tags)| {
+					self.find_workflow_inner(workflow_name, tags, &tx)
+				});
+				try_join_all(futures).await
+			})
+			.custom_instrument(tracing::info_span!("find_workflows_batch_tx"))
+			.await
+			.context("failed to find workflows")
+			.map_err(WorkflowError::Udb)?;
+
+		let dt = start_instant.elapsed().as_secs_f64();
+		metrics::FIND_WORKFLOWS_BATCH_DURATION.observe(dt);
+
+		Ok(workflow_ids)
+	}
+
 	#[tracing::instrument(skip_all)]
 	async fn pull_workflows(
 		&self,
@@ -1006,16 +1072,28 @@ impl Database for DatabaseKv {
 					let tx = tx.with_subspace(self.subspace.clone());
 					let now = rivet_util::timestamp::now();
 
-					// All wake conditions with a timestamp after this timestamp will be pulled
+					// All wake conditions with a timestamp before this timestamp will be pulled
 					let pull_before = now + i64::try_from(self.worker_poll_interval().as_millis())?;
 					// Only consider workers that have pinged within 2 ping intervals ago
 					let active_workers_after = now - i64::try_from(PING_INTERVAL.as_millis() * 2)?;
 
-					// Determine load shedding ratio based on linear mapping on cpu usage. We will gradually
-					// pull less workflows as the cpu usage increases
-					let cpu_usage = { self.system.lock().await.cpu_usage() };
-					let load_shed_ratio_x1000 =
-						calc_pull_ratio((cpu_usage.max(100.0) * 10.0) as u64, 500, 1000, 800, 100);
+					let cpu_usage_ratio = {
+						self.system
+							.lock()
+							.await
+							.cpu_usage_ratio(self.config.runtime.worker_cpu_max)
+					};
+					let load_shed_curve = self.config.runtime.worker_load_shedding_curve();
+					let load_shed_ratio_x1000 = calc_pull_ratio(
+						(cpu_usage_ratio * 1000.0) as u64,
+						load_shed_curve[0].0,
+						load_shed_curve[0].1,
+						load_shed_curve[1].0,
+						load_shed_curve[1].1,
+					);
+
+					// Record load shedding ratio metric
+					metrics::LOAD_SHEDDING_RATIO.observe(load_shed_ratio_x1000 as f64 / 1000.0);
 
 					let active_worker_subspace_start = tx.pack(
 						&keys::worker::ActiveWorkerIdxKey::subspace(active_workers_after),
@@ -1042,32 +1120,53 @@ impl Database for DatabaseKv {
 							Ok(key.worker_id)
 						})
 						.try_collect::<Vec<_>>(),
-						futures_util::stream::iter(owned_filter)
-							.map(|wf_name| {
-								let wake_subspace_start = end_of_key_range(&tx.pack(
-									&keys::wake::WorkflowWakeConditionKey::subspace_without_ts(
-										wf_name.clone(),
-									),
-								));
-								let wake_subspace_end =
-									tx.pack(&keys::wake::WorkflowWakeConditionKey::subspace(
-										wf_name,
-										pull_before,
+						async {
+							let start = Instant::now();
+							let mut buffer = Vec::new();
+							let mut stream = futures_util::stream::iter(owned_filter)
+								.map(|wf_name| {
+									let wake_subspace_start = end_of_key_range(&tx.pack(
+										&keys::wake::WorkflowWakeConditionKey::subspace_without_ts(
+											wf_name.clone(),
+										),
 									));
+									let wake_subspace_end =
+										tx.pack(&keys::wake::WorkflowWakeConditionKey::subspace(
+											wf_name,
+											pull_before,
+										));
 
-								tx.get_ranges_keyvalues(
-									universaldb::RangeOption {
-										mode: StreamingMode::WantAll,
-										..(wake_subspace_start, wake_subspace_end).into()
-									},
-									// This is Snapshot to reduce contention with any new wake conditions
-									// being inserted. Conflicts are handled by workflow leases.
-									Snapshot,
-								)
-							})
-							.flatten()
-							.map(|res| tx.unpack::<keys::wake::WorkflowWakeConditionKey>(res?.key()))
-							.try_collect::<Vec<_>>(),
+									tx.get_ranges_keyvalues(
+										universaldb::RangeOption {
+											mode: StreamingMode::WantAll,
+											..(wake_subspace_start, wake_subspace_end).into()
+										},
+										// This is Snapshot to reduce contention with any new wake conditions
+										// being inserted. Conflicts are handled by workflow leases.
+										Snapshot,
+									)
+								})
+								.flatten()
+								.map(|res| {
+									tx.unpack::<keys::wake::WorkflowWakeConditionKey>(res?.key())
+								});
+
+							loop {
+								if start.elapsed() > EARLY_TXN_TIMEOUT {
+									tracing::warn!("timed out pulling wake conditions");
+									break;
+								}
+
+								let Some(wake_key) = stream.try_next().await? else {
+									break;
+								};
+
+								buffer.push(wake_key);
+							}
+
+							anyhow::Ok(buffer)
+						}
+						.custom_instrument(tracing::debug_span!("read_wake_conditions"))
 					)?;
 
 					// Sort for consistency across all workers
@@ -1089,70 +1188,94 @@ impl Database for DatabaseKv {
 
 						0
 					};
-					let active_worker_count = active_worker_ids.len() as u64;
+					let active_worker_count = active_worker_ids.len().max(1) as u64;
 
 					// Collect name and deadline ts for each wf id
-					let mut dedup_workflows: Vec<MinimalPulledWorkflow> = Vec::new();
+					let mut dedup_workflows = HashMap::<Id, MinimalPulledWorkflow>::new();
+					let now = rivet_util::timestamp::now(); // More up to date now than prev var
 					for wake_key in &wake_keys {
-						if let Some(wf) = dedup_workflows
-							.iter_mut()
-							.find(|wf| wf.workflow_id == wake_key.workflow_id)
-						{
-							let key_wake_deadline_ts = wake_key.condition.deadline_ts();
+						// Record time difference between when the wake condition was created and when it was
+						// pulled (here). We ignore deadline wake conditions because their ts value is not
+						// representative of when it was created, but rather when it should wake.
+						if !matches!(
+							wake_key.condition,
+							keys::wake::WakeCondition::Deadline { .. }
+						) {
+							// TODO: This will record metrics even if the txn fails, which is wrong
+							metrics::WORKFLOW_WAKE_DELTA_DURATION
+								.with_label_values(&[&wake_key.workflow_name])
+								.observe(now.saturating_sub(wake_key.ts).max(0) as f64 / 1000.0);
+						}
 
-							// Update wake condition ts if earlier
-							if wake_key.ts < wf.earliest_wake_condition_ts {
-								wf.earliest_wake_condition_ts = wake_key.ts;
-							}
+						let Some(wf) = dedup_workflows.get_mut(&wake_key.workflow_id) else {
+							dedup_workflows.insert(
+								wake_key.workflow_id,
+								MinimalPulledWorkflow {
+									workflow_id: wake_key.workflow_id,
+									workflow_name: wake_key.workflow_name.clone(),
+									wake_deadline_ts: wake_key.condition.deadline_ts(),
+									earliest_wake_condition_ts: wake_key.ts,
+								},
+							);
 
-							// Update wake deadline ts if earlier
-							if wf.wake_deadline_ts.is_none()
-								|| key_wake_deadline_ts < wf.wake_deadline_ts
-							{
-								wf.wake_deadline_ts = key_wake_deadline_ts;
+							// Hard limit of 10k deduped workflows, this gets further limited to 1000 at
+							// `assigned_workflows`
+							if dedup_workflows.len() >= 10000 {
+								break;
 							}
 
 							continue;
+						};
+
+						let key_wake_deadline_ts = wake_key.condition.deadline_ts();
+
+						// Update wake condition ts if earlier
+						if wake_key.ts < wf.earliest_wake_condition_ts {
+							wf.earliest_wake_condition_ts = wake_key.ts;
 						}
 
-						dedup_workflows.push(MinimalPulledWorkflow {
-							workflow_id: wake_key.workflow_id,
-							workflow_name: wake_key.workflow_name.clone(),
-							wake_deadline_ts: wake_key.condition.deadline_ts(),
-							earliest_wake_condition_ts: wake_key.ts,
-						});
+						// Update wake deadline ts if earlier
+						if wf.wake_deadline_ts.is_none()
+							|| key_wake_deadline_ts < wf.wake_deadline_ts
+						{
+							wf.wake_deadline_ts = key_wake_deadline_ts;
+						}
 					}
 
 					// Filter workflows in a way that spreads all current pending workflows across all active
 					// workers evenly
-					let assigned_workflows = dedup_workflows.into_iter().filter(|wf| {
-						let mut hasher = DefaultHasher::new();
-
-						// Earliest wake condition ts is consistent for hashing purposes because when it
-						// changes it means a worker has leased it
-						wf.earliest_wake_condition_ts.hash(&mut hasher);
-						let wf_hash = hasher.finish();
-
-						let pseudorandom_value_x1000 = {
-							// Add a little pizazz to the hash so its a different number than wf_hash but
-							// still consistent
-							1234i32.hash(&mut hasher);
-							hasher.finish() % 1000 // 0-1000
-						};
-
-						if pseudorandom_value_x1000 < load_shed_ratio_x1000 {
-							return false;
-						}
+					let assigned_workflows = dedup_workflows
+						.into_values()
+						.filter(|wf| {
+							let mut hasher = DefaultHasher::new();
+
+							// Earliest wake condition ts is consistent for hashing purposes because when it
+							// changes it means a worker has leased it
+							wf.earliest_wake_condition_ts.hash(&mut hasher);
+							let wf_hash = hasher.finish();
+
+							let pseudorandom_value_x1000 = {
+								// Add a little pizazz to the hash so its a different number than wf_hash but
+								// still consistent
+								1234i32.hash(&mut hasher);
+								hasher.finish() % 1000 // 0-1000
+							};
+
+							if pseudorandom_value_x1000 > load_shed_ratio_x1000 {
+								return false;
+							}
 
-						let wf_worker_idx = wf_hash % active_worker_count;
+							let wf_worker_idx = wf_hash % active_worker_count;
 
-						// Every worker pulls workflows that has to the current worker as well as the next
-						// worker for redundancy. this results in increased txn conflicts but less chance of
-						// orphaned workflows
-						let next_worker_idx = (current_worker_idx + 1) % active_worker_count;
+							// Every worker pulls workflows that match the current worker idx as well as the next
+							// worker for redundancy. this results in increased txn conflicts but less chance of
+							// orphaned workflows
+							let next_worker_idx = (current_worker_idx + 1) % active_worker_count;
 
-						wf_worker_idx == current_worker_idx || wf_worker_idx == next_worker_idx
-					});
+							wf_worker_idx == current_worker_idx || wf_worker_idx == next_worker_idx
+						})
+						// Hard limit of 1000 workflows per pull
+						.take(1000);
 
 					// Check leases
 					let leased_workflows = futures_util::stream::iter(assigned_workflows)
@@ -1174,10 +1297,10 @@ impl Database for DatabaseKv {
 
 									update_metric(
 										&tx,
-										Some(keys::metric::GaugeMetric::WorkflowSleeping(
+										Some(keys::metric::Metric::WorkflowSleeping(
 											wf.workflow_name.clone(),
 										)),
-										Some(keys::metric::GaugeMetric::WorkflowActive(
+										Some(keys::metric::Metric::WorkflowActive(
 											wf.workflow_name.clone(),
 										)),
 									);
@@ -1190,7 +1313,7 @@ impl Database for DatabaseKv {
 						.buffer_unordered(1024)
 						.try_filter_map(|x| std::future::ready(Ok(x)))
 						.try_collect::<Vec<_>>()
-						.instrument(tracing::trace_span!("map_to_leased_workflows"))
+						.custom_instrument(tracing::debug_span!("map_to_leased_workflows"))
 						.await?;
 
 					// Clear all wake conditions from workflows that we have leased
@@ -1205,49 +1328,6 @@ impl Database for DatabaseKv {
 						tx.delete(wake_key);
 					}
 
-					let leased_workflow_ids = leased_workflows
-						.iter()
-						.map(|wf| wf.workflow_id)
-						.collect::<Vec<_>>();
-
-					// Clear secondary indexes so that we don't get any new wake conditions inserted while
-					// the workflow is running
-					futures_util::stream::iter(leased_workflow_ids)
-						.map(|workflow_id| {
-							let tx = tx.clone();
-							async move {
-								// Clear sub workflow secondary idx
-								let wake_sub_workflow_key =
-									keys::workflow::WakeSubWorkflowKey::new(workflow_id);
-								if let Some(entry) = tx
-									.get(&self.subspace.pack(&wake_sub_workflow_key), Serializable)
-									.await?
-								{
-									let sub_workflow_id =
-										wake_sub_workflow_key.deserialize(&entry)?;
-
-									let sub_workflow_wake_key = keys::wake::SubWorkflowWakeKey::new(
-										sub_workflow_id,
-										workflow_id,
-									);
-
-									tx.clear(&self.subspace.pack(&sub_workflow_wake_key));
-								}
-
-								// Clear signals secondary index
-								let wake_signals_subspace = self.subspace.subspace(
-									&keys::workflow::WakeSignalKey::subspace(workflow_id),
-								);
-								tx.clear_subspace_range(&wake_signals_subspace);
-
-								anyhow::Ok(())
-							}
-						})
-						// TODO: How to get rid of this buffer?
-						.buffer_unordered(1024)
-						.try_collect::<()>()
-						.await?;
-
 					// NOTE: We don't read any workflow data in this txn since its only for acquiring leases.
 					// The less operations we do in this txn the less contention there is with other workers.
 					Ok(leased_workflows)
@@ -1255,14 +1335,17 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("pull_workflows_tx"))
 			.await
+			.context("failed to lease workflows")
 			.map_err(WorkflowError::Udb)?;
 
 		let worker_id_str = worker_id.to_string();
 		let dt = start_instant.elapsed().as_secs_f64();
 		metrics::LAST_PULL_WORKFLOWS_DURATION
-			.record(dt, &[KeyValue::new("worker_id", worker_id_str.clone())]);
+			.with_label_values(&[worker_id_str.as_str()])
+			.set(dt);
 		metrics::PULL_WORKFLOWS_DURATION
-			.record(dt, &[KeyValue::new("worker_id", worker_id_str.clone())]);
+			.with_label_values(&[worker_id_str.as_str()])
+			.observe(dt);
 
 		if leased_workflows.is_empty() {
 			return Ok(Vec::new());
@@ -1270,7 +1353,81 @@ impl Database for DatabaseKv {
 
 		let start_instant2 = Instant::now();
 
-		let pulled_workflows = self
+		// Collect metrics on lease counts
+		let mut leased_metrics = HashMap::<&str, usize>::new();
+		for leased_workflow in &leased_workflows {
+			*leased_metrics
+				.entry(leased_workflow.workflow_name.as_str())
+				.or_default() += 1;
+		}
+
+		for (workflow_name, count) in &leased_metrics {
+			metrics::WORKFLOW_LEASED
+				.with_label_values(&[*workflow_name])
+				.observe(*count as f64);
+		}
+
+		let leased_workflow_ids = leased_workflows
+			.iter()
+			.map(|wf| wf.workflow_id)
+			.collect::<Vec<_>>();
+		let clear_secondary_idx_fut = async move {
+			self.pools
+				.udb()
+				.map_err(WorkflowError::PoolsGeneric)?
+				.run(|tx| {
+					let leased_workflow_ids = leased_workflow_ids.clone();
+
+					async move {
+						// Clear secondary indexes so that we don't get any new wake conditions inserted while
+						// the workflow is running
+						futures_util::stream::iter(leased_workflow_ids)
+							.map(|workflow_id| {
+								let tx = tx.clone();
+								async move {
+									// Clear sub workflow secondary idx
+									let wake_sub_workflow_key =
+										keys::workflow::WakeSubWorkflowKey::new(workflow_id);
+									// NOTE: Snapshot read because we prefer having this txn not conflict rather
+									// than unnecessarily insert wake conditions
+									if let Some(entry) = tx
+										.get(&self.subspace.pack(&wake_sub_workflow_key), Snapshot)
+										.await?
+									{
+										let sub_workflow_id =
+											wake_sub_workflow_key.deserialize(&entry)?;
+
+										let sub_workflow_wake_key =
+											keys::wake::SubWorkflowWakeKey::new(
+												sub_workflow_id,
+												workflow_id,
+											);
+
+										tx.clear(&self.subspace.pack(&sub_workflow_wake_key));
+									}
+
+									// Clear signals secondary index
+									let wake_signals_subspace = self.subspace.subspace(
+										&keys::workflow::WakeSignalKey::subspace(workflow_id),
+									);
+									tx.clear_subspace_range(&wake_signals_subspace);
+
+									anyhow::Ok(())
+								}
+							})
+							// TODO: How to get rid of this buffer?
+							.buffer_unordered(1024)
+							.try_collect::<()>()
+							.await
+					}
+				})
+				.custom_instrument(tracing::info_span!("clear_workflow_secondary_idx_tx"))
+				.await
+				.context("failed to clear workflow secondary indexes")
+		};
+
+		let pulled_workflows_fut = async move {
+			self
 			.pools
 			.udb()
 			.map_err(WorkflowError::PoolsGeneric)?
@@ -1288,8 +1445,10 @@ impl Database for DatabaseKv {
 								let ray_id_key = keys::workflow::RayIdKey::new(wf.workflow_id);
 								let input_key = keys::workflow::InputKey::new(wf.workflow_id);
 								let state_key = keys::workflow::StateKey::new(wf.workflow_id);
+								let output_key = keys::workflow::OutputKey::new(wf.workflow_id);
 								let input_subspace = self.subspace.subspace(&input_key);
 								let state_subspace = self.subspace.subspace(&state_key);
+								let output_subspace = self.subspace.subspace(&output_key);
 								let active_history_subspace = self.subspace.subspace(
 									&keys::history::HistorySubspaceKey::new(
 										wf.workflow_id,
@@ -1302,36 +1461,39 @@ impl Database for DatabaseKv {
 									ray_id_entry,
 									input_chunks,
 									state_chunks,
+									has_output,
 									events,
 								) = tokio::try_join!(
-									async {
-										tx.get(&self.subspace.pack(&create_ts_key), Serializable)
-											.await
-									},
-									async {
-										tx.get(&self.subspace.pack(&ray_id_key), Serializable).await
-									},
-									async {
-										tx.get_ranges_keyvalues(
-											universaldb::RangeOption {
-												mode: StreamingMode::WantAll,
-												..(&input_subspace).into()
-											},
-											Serializable,
-										)
-										.try_collect::<Vec<_>>()
-										.await
-									},
+									tx.get(&self.subspace.pack(&create_ts_key), Serializable),
+									tx.get(&self.subspace.pack(&ray_id_key), Serializable),
+									tx.get_ranges_keyvalues(
+										universaldb::RangeOption {
+											mode: StreamingMode::WantAll,
+											..(&input_subspace).into()
+										},
+										Serializable,
+									)
+									.try_collect::<Vec<_>>(),
+									tx.get_ranges_keyvalues(
+										universaldb::RangeOption {
+											mode: StreamingMode::WantAll,
+											..(&state_subspace).into()
+										},
+										Serializable,
+									)
+									.try_collect::<Vec<_>>(),
 									async {
 										tx.get_ranges_keyvalues(
 											universaldb::RangeOption {
-												mode: StreamingMode::WantAll,
-												..(&state_subspace).into()
+												mode: StreamingMode::Exact,
+												limit: Some(1),
+												..(&output_subspace).into()
 											},
 											Serializable,
 										)
-										.try_collect::<Vec<_>>()
+										.try_next()
 										.await
+										.map(|entry| entry.is_some())
 									},
 									async {
 										let mut events_by_location: HashMap<Location, Vec<Event>> =
@@ -1363,7 +1525,7 @@ impl Database for DatabaseKv {
 												if current_event.location.is_empty() {
 													current_event =
 														WorkflowHistoryEventBuilder::new(
-															partial_key.location,
+															partial_key.location.clone(),
 														);
 												} else {
 													// Insert current event builder to into wf events and
@@ -1371,13 +1533,24 @@ impl Database for DatabaseKv {
 													let previous_event = std::mem::replace(
 														&mut current_event,
 														WorkflowHistoryEventBuilder::new(
-															partial_key.location,
+															partial_key.location.clone(),
 														),
 													);
-													events_by_location
-														.entry(previous_event.location.root())
-														.or_default()
-														.push(Event::try_from(previous_event)?);
+
+													let loc = previous_event.location.clone();
+
+													match Event::try_from(previous_event) {
+														Ok(event) => {
+															events_by_location
+																.entry(loc.root())
+																.or_default()
+																.push(event);
+														}
+														Err(err) => {
+															tracing::error!(workflow_id=?wf.workflow_id, location=?loc, ?err, "failed building workflow history");
+															return Ok(None);
+														}
+													}
 												}
 											}
 
@@ -1477,26 +1650,111 @@ impl Database for DatabaseKv {
 
 												current_event.inner_event_type =
 													Some(inner_event_type);
+											} else if let Ok(key) = self
+												.subspace
+												.unpack::<keys::history::InnerVersionKey>(
+												entry.key(),
+											) {
+												let inner_version =
+													key.deserialize(entry.value())?;
+
+												current_event.inner_version = Some(inner_version);
+											} else if let Ok(key) = self
+												.subspace
+												.unpack::<keys::history::IndexedNameKey>(
+												entry.key(),
+											) {
+												if current_event.indexed_names.len() != key.index {
+													tracing::error!(
+														?wf,
+														location=?partial_key.location,
+														expected=%current_event.indexed_names.len(),
+														got=%key.index,
+														"corrupt history, indexed name doesn't exist yet or is out of order"
+													);
+													return Ok(None);
+												}
+
+												let name = key.deserialize(entry.value())?;
+												current_event.indexed_names.push(name);
+											} else if let Ok(key) =
+												self.subspace
+													.unpack::<keys::history::IndexedInputChunkKey>(
+														entry.key(),
+													) {
+												if let Some(input_chunks) = current_event
+													.indexed_input_chunks
+													.get_mut(key.index)
+												{
+													input_chunks.push(entry);
+												} else if current_event.indexed_input_chunks.len()
+													== key.index
+												{
+													current_event
+														.indexed_input_chunks
+														.push(vec![entry]);
+												} else {
+													tracing::error!(
+														?wf,
+														location=?partial_key.location,
+														expected=%current_event.indexed_input_chunks.len(),
+														got=%key.index,
+														"corrupt history, indexed chunk doesn't exist yet or is out of order"
+													);
+													return Ok(None);
+												}
 											}
 
 											// We ignore keys we don't need (like tags)
 										}
 										// Insert final event
 										if !current_event.location.is_empty() {
-											events_by_location
-												.entry(current_event.location.root())
-												.or_default()
-												.push(Event::try_from(current_event)?);
+											let loc = current_event.location.clone();
+
+											match Event::try_from(current_event) {
+												Ok(event) => {
+													events_by_location
+														.entry(loc.root())
+														.or_default()
+														.push(event);
+												}
+												Err(err) => {
+													tracing::error!(workflow_id=?wf.workflow_id, location=?loc, ?err, "failed building workflow history");
+													return Ok(None);
+												}
+											}
 										}
 
-										Ok(events_by_location)
+										Ok(Some(events_by_location))
 									}
 								)?;
 
+								if has_output {
+									tracing::warn!(workflow_id=?wf.workflow_id, "workflow already completed, ignoring");
+
+									// Clear lease
+									let lease_key = keys::workflow::LeaseKey::new(wf.workflow_id);
+									tx.clear(&self.subspace.pack(&lease_key));
+									let worker_id_key =
+										keys::workflow::WorkerIdKey::new(wf.workflow_id);
+									tx.clear(&self.subspace.pack(&worker_id_key));
+
+									return Ok(None);
+								}
+
+								let Some(events) = events else {
+									return Ok(None);
+								};
+
+								let (Some(create_ts_entry), Some(ray_id_entry)) = (create_ts_entry, ray_id_entry) else {
+									tracing::error!(workflow_id=?wf.workflow_id, "create_ts and ray_id keys should exist");
+									return Ok(None);
+								};
+
 								let create_ts = create_ts_key
-									.deserialize(&create_ts_entry.context("key should exist")?)?;
+									.deserialize(&create_ts_entry)?;
 								let ray_id = ray_id_key
-									.deserialize(&ray_id_entry.context("key should exist")?)?;
+									.deserialize(&ray_id_entry)?;
 								let input = input_key.combine(input_chunks)?;
 								let state = if state_chunks.is_empty() {
 									serde_json::value::RawValue::NULL.to_owned()
@@ -1504,7 +1762,7 @@ impl Database for DatabaseKv {
 									state_key.combine(state_chunks)?
 								};
 
-								Result::<_>::Ok(PulledWorkflowData {
+								anyhow::Ok(Some(PulledWorkflowData {
 									workflow_id: wf.workflow_id,
 									workflow_name: wf.workflow_name,
 									create_ts,
@@ -1513,11 +1771,12 @@ impl Database for DatabaseKv {
 									state,
 									wake_deadline_ts: wf.wake_deadline_ts,
 									events,
-								})
+								}))
 							}
 						})
 						// TODO: How to get rid of this buffer?
 						.buffer_unordered(512)
+						.try_filter_map(|x| std::future::ready(Ok(x)))
 						.try_collect::<Vec<_>>()
 						.instrument(tracing::trace_span!("map_to_partial_workflow"))
 						.await
@@ -1525,20 +1784,27 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("pull_workflow_history_tx"))
 			.await
-			.map_err(WorkflowError::Udb)?;
+			.context("failed to pull workflow history")
+		};
+
+		let (_, pulled_workflows) =
+			tokio::try_join!(clear_secondary_idx_fut, pulled_workflows_fut,)
+				.map_err(WorkflowError::Udb)?;
 
 		let dt2 = start_instant2.elapsed().as_secs_f64();
 		let dt = start_instant.elapsed().as_secs_f64();
 		metrics::LAST_PULL_WORKFLOWS_FULL_DURATION
-			.record(dt, &[KeyValue::new("worker_id", worker_id_str.clone())]);
+			.with_label_values(&[worker_id_str.as_str()])
+			.set(dt);
 		metrics::PULL_WORKFLOWS_FULL_DURATION
-			.record(dt, &[KeyValue::new("worker_id", worker_id_str.clone())]);
-		metrics::LAST_PULL_WORKFLOWS_HISTORY_DURATION.record(
-			dt2 as u64,
-			&[KeyValue::new("worker_id", worker_id_str.clone())],
-		);
+			.with_label_values(&[worker_id_str.as_str()])
+			.observe(dt);
+		metrics::LAST_PULL_WORKFLOWS_HISTORY_DURATION
+			.with_label_values(&[worker_id_str.as_str()])
+			.set(dt2);
 		metrics::PULL_WORKFLOWS_HISTORY_DURATION
-			.record(dt2, &[KeyValue::new("worker_id", worker_id_str.clone())]);
+			.with_label_values(&[worker_id_str.as_str()])
+			.observe(dt2);
 
 		Ok(pulled_workflows)
 	}
@@ -1549,15 +1815,18 @@ impl Database for DatabaseKv {
 		workflow_id: Id,
 		workflow_name: &str,
 		output: &serde_json::value::RawValue,
+		prune_variant: PruneVariant,
 	) -> WorkflowResult<()> {
 		let start_instant = Instant::now();
 
-		let wrote_to_wake_idx = self
+		let (wrote_to_wake_idx, pending_signal_cleared_count) = self
 			.pools
 			.udb()
 			.map_err(WorkflowError::PoolsGeneric)?
 			.run(|tx| {
 				async move {
+					let tx = tx.with_subspace(self.subspace.clone());
+
 					let sub_workflow_wake_subspace = self
 						.subspace
 						.subspace(&keys::wake::SubWorkflowWakeKey::subspace(workflow_id));
@@ -1575,34 +1844,29 @@ impl Database for DatabaseKv {
 						Serializable,
 					);
 
-					let (wrote_to_wake_idx, tag_keys, wake_deadline_entry) = tokio::try_join!(
+					let (wrote_to_wake_idx, tag_keys, wake_deadline) = tokio::try_join!(
 						// Check for other workflows waiting on this one, wake all
 						async {
 							let mut wrote_to_wake_idx = false;
 
 							while let Some(entry) = stream.try_next().await? {
-								let sub_workflow_wake_key =
-									self.subspace
-										.unpack::<keys::wake::SubWorkflowWakeKey>(&entry.key())?;
-								let workflow_name =
-									sub_workflow_wake_key.deserialize(entry.value())?;
-
-								let wake_condition_key = keys::wake::WorkflowWakeConditionKey::new(
-									workflow_name,
-									sub_workflow_wake_key.workflow_id,
-									keys::wake::WakeCondition::SubWorkflow {
-										sub_workflow_id: workflow_id,
-									},
-								);
+								let (sub_workflow_wake_key, workflow_name) =
+									tx.read_entry::<keys::wake::SubWorkflowWakeKey>(&entry)?;
 
 								// Add wake condition for workflow
-								tx.set(
-									&self.subspace.pack(&wake_condition_key),
-									&wake_condition_key.serialize(())?,
-								);
+								tx.write(
+									&keys::wake::WorkflowWakeConditionKey::new(
+										workflow_name,
+										sub_workflow_wake_key.workflow_id,
+										keys::wake::WakeCondition::SubWorkflow {
+											sub_workflow_id: workflow_id,
+										},
+									),
+									(),
+								)?;
 
 								// Clear secondary index
-								tx.clear(entry.key());
+								tx.delete(&sub_workflow_wake_key);
 
 								wrote_to_wake_idx = true;
 							}
@@ -1618,32 +1882,27 @@ impl Database for DatabaseKv {
 							Serializable,
 						)
 						.map(|res| {
-							self.subspace
-								.unpack::<keys::workflow::TagKey>(res?.key())
+							tx.unpack::<keys::workflow::TagKey>(res?.key())
 								.map_err(anyhow::Error::from)
 						})
 						.try_collect::<Vec<_>>(),
-						tx.get(&self.subspace.pack(&wake_deadline_key), Serializable),
+						tx.read_opt(&wake_deadline_key, Serializable),
 					)?;
 
 					for key in tag_keys {
-						let by_name_and_tag_key = keys::workflow::ByNameAndTagKey::new(
+						tx.delete(&keys::workflow::ByNameAndTagKey::new(
 							workflow_name.to_string(),
 							key.k,
 							key.v,
 							workflow_id,
-						);
-						tx.clear(&self.subspace.pack(&by_name_and_tag_key));
+						));
 					}
 
 					// Clear null key
-					{
-						let by_name_and_tag_key = keys::workflow::ByNameAndTagKey::null(
-							workflow_name.to_string(),
-							workflow_id,
-						);
-						tx.clear(&self.subspace.pack(&by_name_and_tag_key));
-					}
+					tx.delete(&keys::workflow::ByNameAndTagKey::null(
+						workflow_name.to_string(),
+						workflow_id,
+					));
 
 					// Get and clear the pending deadline wake condition, if any. This could be put in the
 					// `pull_workflows` function (where we clear secondary indexes) but we chose to clear it
@@ -1651,22 +1910,16 @@ impl Database for DatabaseKv {
 					// it inserting more wake conditions. This reduces the load on `pull_workflows`. The
 					// reason this isn't immediately cleared in `pull_workflows` along with the rest of the
 					// wake conditions is because it might be in the future.
-					if let Some(raw) = wake_deadline_entry {
-						let deadline_ts = wake_deadline_key.deserialize(&raw)?;
-
-						let wake_condition_key = keys::wake::WorkflowWakeConditionKey::new(
+					if let Some(deadline_ts) = wake_deadline {
+						tx.delete(&keys::wake::WorkflowWakeConditionKey::new(
 							workflow_name.to_string(),
 							workflow_id,
 							keys::wake::WakeCondition::Deadline { deadline_ts },
-						);
-
-						tx.clear(&self.subspace.pack(&wake_condition_key));
+						));
 					}
 
 					// Clear "has wake condition"
-					let has_wake_condition_key =
-						keys::workflow::HasWakeConditionKey::new(workflow_id);
-					tx.clear(&self.subspace.pack(&has_wake_condition_key));
+					tx.delete(&keys::workflow::HasWakeConditionKey::new(workflow_id));
 
 					// Write output
 					let output_key = keys::workflow::OutputKey::new(workflow_id);
@@ -1674,30 +1927,84 @@ impl Database for DatabaseKv {
 					for (i, chunk) in output_key.split_ref(output)?.into_iter().enumerate() {
 						let chunk_key = output_key.chunk(i);
 
-						tx.set(&self.subspace.pack(&chunk_key), &chunk);
+						tx.set(&tx.pack(&chunk_key), &chunk);
 					}
 
 					// Clear lease
-					let lease_key = keys::workflow::LeaseKey::new(workflow_id);
-					tx.clear(&self.subspace.pack(&lease_key));
-					let worker_id_key = keys::workflow::WorkerIdKey::new(workflow_id);
-					tx.clear(&self.subspace.pack(&worker_id_key));
+					tx.delete(&keys::workflow::LeaseKey::new(workflow_id));
+					tx.delete(&keys::workflow::WorkerIdKey::new(workflow_id));
+
+					// Clear pending signals metric for observability
+					let metrics_subspace = self
+						.subspace
+						.subspace(&keys::workflow::MetricKey::subspace(workflow_id));
+					let mut stream = tx.get_ranges_keyvalues(
+						universaldb::RangeOption {
+							mode: StreamingMode::WantAll,
+							..(&metrics_subspace).into()
+						},
+						Serializable,
+					);
+
+					let mut pending_signal_cleared_count = 0;
+					loop {
+						let Some(entry) = stream.try_next().await? else {
+							break;
+						};
+
+						let (key, metric_count) =
+							tx.read_entry::<keys::workflow::MetricKey>(&entry)?;
+
+						// Ignore negatives and zero
+						if metric_count as isize <= 0 {
+							continue;
+						}
+
+						match key.metric {
+							keys::workflow::Metric::SignalPending(signal_name) => {
+								update_metric_by(
+									&tx,
+									Some(keys::metric::Metric::SignalPending2(signal_name)),
+									None,
+									metric_count,
+								);
+								pending_signal_cleared_count += metric_count;
+							}
+						}
+					}
+
+					// Insert into prune idx if applicable
+					match prune_variant {
+						PruneVariant::All | PruneVariant::History => {
+							tx.write(
+								&keys::workflow::PruneIdxKey::new(workflow_id, prune_variant),
+								(),
+							)?;
+						}
+						PruneVariant::None => {}
+					}
+
+					tx.write(
+						&keys::workflow::CompleteTsKey::new(workflow_id),
+						rivet_util::timestamp::now(),
+					)?;
 
 					update_metric(
-						&tx.with_subspace(self.subspace.clone()),
-						Some(keys::metric::GaugeMetric::WorkflowActive(
+						&tx,
+						Some(keys::metric::Metric::WorkflowActive(
 							workflow_name.to_string(),
 						)),
-						Some(keys::metric::GaugeMetric::WorkflowComplete(
+						Some(keys::metric::Metric::WorkflowComplete(
 							workflow_name.to_string(),
 						)),
 					);
 
-					Ok(wrote_to_wake_idx)
+					Ok((wrote_to_wake_idx, pending_signal_cleared_count))
 				}
 			})
 			.custom_instrument(tracing::info_span!("complete_workflows_tx"))
 			.await
+			.context("failed to complete workflow")
 			.map_err(WorkflowError::Udb)?;
 
 		// Wake worker again in case some other workflow was waiting for this one to complete
@@ -1706,11 +2013,14 @@ impl Database for DatabaseKv {
 			self.bump(BumpSubSubject::Worker);
 		}
 
+		if pending_signal_cleared_count != 0 {
+			tracing::debug!(count=%pending_signal_cleared_count, "cleared pending signals after workflow completed");
+		}
+
 		let dt = start_instant.elapsed().as_secs_f64();
-		metrics::COMPLETE_WORKFLOW_DURATION.record(
-			dt,
-			&[KeyValue::new("workflow_name", workflow_name.to_string())],
-		);
+		metrics::COMPLETE_WORKFLOW_DURATION
+			.with_label_values(&[workflow_name])
+			.observe(dt);
 
 		Ok(())
 	}
@@ -1834,13 +2144,13 @@ impl Database for DatabaseKv {
 
 					update_metric(
 						&tx.with_subspace(self.subspace.clone()),
-						Some(keys::metric::GaugeMetric::WorkflowActive(
+						Some(keys::metric::Metric::WorkflowActive(
 							workflow_name.to_string(),
 						)),
 						Some(if has_wake_condition {
-							keys::metric::GaugeMetric::WorkflowSleeping(workflow_name.to_string())
+							keys::metric::Metric::WorkflowSleeping(workflow_name.to_string())
 						} else {
-							keys::metric::GaugeMetric::WorkflowDead(
+							keys::metric::Metric::WorkflowDead(
 								workflow_name.to_string(),
 								error.to_string(),
 							)
@@ -1852,6 +2162,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("commit_workflow_tx"))
 			.await
+			.context("failed to commit workflow")
 			.map_err(WorkflowError::Udb)?;
 
 		// Always wake the worker immediately again. This is an IMPORTANT implementation detail to prevent
@@ -1874,16 +2185,15 @@ impl Database for DatabaseKv {
 		self.bump(BumpSubSubject::Worker);
 
 		let dt = start_instant.elapsed().as_secs_f64();
-		metrics::COMMIT_WORKFLOW_DURATION.record(
-			dt,
-			&[KeyValue::new("workflow_name", workflow_name.to_string())],
-		);
+		metrics::COMMIT_WORKFLOW_DURATION
+			.with_label_values(&[workflow_name])
+			.observe(dt);
 
 		Ok(())
 	}
 
 	#[tracing::instrument(skip_all)]
-	async fn pull_next_signal(
+	async fn pull_next_signals(
 		&self,
 		workflow_id: Id,
 		_workflow_name: &str,
@@ -1891,176 +2201,191 @@ impl Database for DatabaseKv {
 		location: &Location,
 		version: usize,
 		_loop_location: Option<&Location>,
-		last_try: bool,
-	) -> WorkflowResult<Option<SignalData>> {
+		limit: usize,
+		last_attempt: bool,
+	) -> WorkflowResult<Vec<SignalData>> {
 		let owned_filter = filter
 			.into_iter()
 			.map(|x| x.to_string())
 			.collect::<Vec<_>>();
 
-		// Fetch signal from UDB
-		let signal =
-			self.pools
-				.udb()
-				.map_err(WorkflowError::PoolsGeneric)?
-				.run(|tx| {
-					let owned_filter = owned_filter.clone();
+		// Fetch signals from UDB
+		let signals = self
+			.pools
+			.udb()
+			.map_err(WorkflowError::PoolsGeneric)?
+			.run(|tx| {
+				let owned_filter = owned_filter.clone();
 
-					async move {
-						let signal = {
-							// Create a stream for each signal name subspace
-							let streams = owned_filter
-								.iter()
-								.map(|signal_name| {
-									let pending_signal_subspace = self.subspace.subspace(
-										&keys::workflow::PendingSignalKey::subspace(
-											workflow_id,
-											signal_name.to_string(),
-										),
-									);
+				async move {
+					// Fetch signals from all streams at the same time
+					let mut signals = futures_util::stream::iter(owned_filter.clone())
+						.map(|signal_name| {
+							let pending_signal_subspace = self.subspace.subspace(
+								&keys::workflow::PendingSignalKey::subspace(
+									workflow_id,
+									signal_name.to_string(),
+								),
+							);
 
-									tx.get_ranges_keyvalues(
-										universaldb::RangeOption {
-											mode: StreamingMode::WantAll,
-											limit: Some(1),
-											..(&pending_signal_subspace).into()
-										},
-										// NOTE: This is Serializable because any insert into this subspace
-										// should cause a conflict and retry of this txn
-										Serializable,
-									)
-								})
-								.collect::<Vec<_>>();
-
-							// Fetch the next entry from all streams at the same time
-							let mut results = futures_util::future::try_join_all(
-								streams.into_iter().map(|mut stream| async move {
-									if let Some(entry) = stream.try_next().await? {
-										Result::<_>::Ok(Some((
-											entry.key().to_vec(),
-											self.subspace
-												.unpack::<keys::workflow::PendingSignalKey>(
-													&entry.key(),
-												)?,
-										)))
-									} else {
-										Ok(None)
-									}
-								}),
-							)
-							.instrument(tracing::trace_span!("map_signals"))
-							.await?;
-
-							// Sort by ts
-							results.sort_by_key(|res| res.as_ref().map(|(_, key)| key.ts));
-
-							results.into_iter().flatten().next().map(
-								|(raw_key, pending_signal_key)| {
-									(
-										raw_key,
-										pending_signal_key.signal_name,
-										pending_signal_key.ts,
-										pending_signal_key.signal_id,
-									)
+							tx.get_ranges_keyvalues(
+								universaldb::RangeOption {
+									mode: StreamingMode::Exact,
+									// Each individual stream is limited to our max limit, we apply this
+									// limit again after they are all aggregated further down
+									limit: Some(limit),
+									..(&pending_signal_subspace).into()
 								},
+								// NOTE: This is Serializable because any insert into this subspace
+								// should cause a conflict and retry of this txn
+								Serializable,
 							)
-						};
-
-						// Signal found
-						if let Some((raw_key, signal_name, ts, signal_id)) = signal {
-							let ack_ts_key = keys::signal::AckTsKey::new(signal_id);
+						})
+						.flatten()
+						.map(|res| {
+							let entry = res?;
 
-							// Ack signal
-							tx.add_conflict_range(
-								&raw_key,
-								&end_of_key_range(&raw_key),
-								ConflictRangeType::Read,
-							)?;
-							tx.set(
-								&self.subspace.pack(&ack_ts_key),
-								&ack_ts_key.serialize(rivet_util::timestamp::now())?,
-							);
+							anyhow::Ok(
+								self.subspace
+									.unpack::<keys::workflow::PendingSignalKey>(&entry.key())?,
+							)
+						})
+						.try_collect::<Vec<_>>()
+						.instrument(tracing::trace_span!("map_signals"))
+						.await?;
 
-							update_metric(
-								&tx.with_subspace(self.subspace.clone()),
-								Some(keys::metric::GaugeMetric::SignalPending(
-									signal_name.to_string(),
-								)),
-								None,
-							);
+					if !signals.is_empty() {
+						let now = rivet_util::timestamp::now();
 
-							// TODO: Split txn into two after acking here?
+						// Insert history event
+						keys::history::insert::signals_event(
+							&self.subspace,
+							&tx,
+							workflow_id,
+							&location,
+							version,
+							now,
+						)?;
 
-							// Clear pending signal key
-							tx.clear(&raw_key);
+						// Sort by ts after aggregating but before applying limit again. Signals are already
+						// in order by ts in their individual streams so this should be cheap
+						signals.sort_by_key(|key| key.ts);
+
+						// Read signal data in parallel
+						let signals =
+							futures_util::stream::iter(signals.into_iter().take(limit).enumerate())
+								.map(|(index, key)| {
+									let tx = tx.clone();
+									async move {
+										let ack_ts_key = keys::signal::AckTsKey::new(key.signal_id);
+
+										let packed_key = self.subspace.pack(&key);
+
+										// Ack signal
+										tx.add_conflict_range(
+											&packed_key,
+											&end_of_key_range(&packed_key),
+											ConflictRangeType::Read,
+										)?;
+										tx.set(
+											&self.subspace.pack(&ack_ts_key),
+											&ack_ts_key.serialize(now)?,
+										);
 
-							// Read signal body
-							let body_key = keys::signal::BodyKey::new(signal_id);
-							let body_subspace = self.subspace.subspace(&body_key);
+										update_metric(
+											&tx.with_subspace(self.subspace.clone()),
+											Some(keys::metric::Metric::SignalPending2(
+												key.signal_name.clone(),
+											)),
+											None,
+										);
+										update_wf_metric(
+											&tx.with_subspace(self.subspace.clone()),
+											workflow_id,
+											Some(keys::workflow::Metric::SignalPending(
+												key.signal_name.clone(),
+											)),
+											None,
+										);
 
-							let chunks = tx
-								.get_ranges_keyvalues(
-									universaldb::RangeOption {
-										mode: StreamingMode::WantAll,
-										..(&body_subspace).into()
-									},
-									Serializable,
-								)
+										// Clear pending signal key
+										tx.clear(&packed_key);
+
+										// Read signal body
+										let body_key = keys::signal::BodyKey::new(key.signal_id);
+										let body_subspace = self.subspace.subspace(&body_key);
+
+										let chunks = tx
+											.get_ranges_keyvalues(
+												universaldb::RangeOption {
+													mode: StreamingMode::WantAll,
+													..(&body_subspace).into()
+												},
+												Serializable,
+											)
+											.try_collect::<Vec<_>>()
+											.await?;
+
+										let body = body_key.combine(chunks)?;
+
+										// Insert each signal body into the signals event
+										keys::history::insert::signals_event_signal(
+											&self.subspace,
+											&tx,
+											workflow_id,
+											&location,
+											index,
+											key.signal_id,
+											&key.signal_name,
+											&body,
+										)?;
+
+										anyhow::Ok(SignalData {
+											signal_id: key.signal_id,
+											signal_name: key.signal_name,
+											create_ts: key.ts,
+											body,
+										})
+									}
+								})
+								// IMPORTANT: The signals need to stay in order
+								.buffered(1024)
 								.try_collect::<Vec<_>>()
 								.await?;
 
-							let body = body_key.combine(chunks)?;
-
-							// Insert history event
-							keys::history::insert::signal_event(
-								&self.subspace,
-								&tx,
+						Ok(signals)
+					}
+					// No signals found
+					else {
+						// Write signal wake index if no signal was received. Normally this is done in
+						// `commit_workflow` but without this code there would be a race condition if the
+						// signal is published between after this transaction and before `commit_workflow`.
+						// There is a possibility of `commit_workflow` NOT writing a signal secondary index
+						// after this in which case there might be an unnecessary wake condition inserted
+						// causing the workflow to wake up again, but this is not as big of an issue because
+						// workflow wakes should be idempotent if no events happen.
+						// It is important that this is only written on the last try to pull workflows
+						// (the workflow engine internally retries a few times) because it should only
+						// write signal wake indexes before going to sleep (with err `NoSignalFound`) and
+						// not during a retry.
+						if last_attempt {
+							self.write_signal_wake_idxs(
 								workflow_id,
-								&location,
-								version,
-								rivet_util::timestamp::now(),
-								signal_id,
-								&signal_name,
-								&body,
+								&owned_filter.iter().map(|x| x.as_str()).collect::<Vec<_>>(),
+								&tx,
 							)?;
-
-							Ok(Some(SignalData {
-								signal_id,
-								signal_name,
-								create_ts: ts,
-								body,
-							}))
 						}
-						// No signal found
-						else {
-							// Write signal wake index if no signal was received. Normally this is done in
-							// `commit_workflow` but without this code there would be a race condition if the
-							// signal is published between after this transaction and before `commit_workflow`.
-							// There is a possibility of `commit_workflow` NOT writing a signal secondary index
-							// after this in which case there might be an unnecessary wake condition inserted
-							// causing the workflow to wake up again, but this is not as big of an issue because
-							// workflow wakes should be idempotent if no events happen.
-							// It is important that this is only written on the last try to pull workflows
-							// (the workflow engine internally retries a few times) because it should only
-							// write signal wake indexes before going to sleep (with err `NoSignalFound`) and
-							// not during a retry.
-							if last_try {
-								self.write_signal_wake_idxs(
-									workflow_id,
-									&owned_filter.iter().map(|x| x.as_str()).collect::<Vec<_>>(),
-									&tx,
-								)?;
-							}
 
-							Ok(None)
-						}
+						Ok(Vec::new())
 					}
-				})
-				.custom_instrument(tracing::info_span!("pull_next_signal_tx"))
-				.await
-				.map_err(WorkflowError::Udb)?;
+				}
+			})
+			.custom_instrument(tracing::info_span!("pull_next_signals_tx"))
+			.await
+			.context("failed to pull signals")
+			.map_err(WorkflowError::Udb)?;
 
-		Ok(signal)
+		Ok(signals)
 	}
 
 	#[tracing::instrument(skip_all)]
@@ -2075,6 +2400,7 @@ impl Database for DatabaseKv {
 			.map_err(WorkflowError::PoolsGeneric)?
 			.run(|tx| {
 				async move {
+					let name_key = keys::workflow::NameKey::new(sub_workflow_id);
 					let input_key = keys::workflow::InputKey::new(sub_workflow_id);
 					let input_subspace = self.subspace.subspace(&input_key);
 					let state_key = keys::workflow::StateKey::new(sub_workflow_id);
@@ -2085,7 +2411,14 @@ impl Database for DatabaseKv {
 						keys::workflow::HasWakeConditionKey::new(sub_workflow_id);
 
 					// Read input and output
-					let (input_chunks, state_chunks, output_chunks, has_wake_condition_entry) = tokio::try_join!(
+					let (
+						name_entry,
+						input_chunks,
+						state_chunks,
+						output_chunks,
+						has_wake_condition_entry,
+					) = tokio::try_join!(
+						tx.get(&self.subspace.pack(&name_key), Serializable),
 						tx.get_ranges_keyvalues(
 							universaldb::RangeOption {
 								mode: StreamingMode::WantAll,
@@ -2147,6 +2480,8 @@ impl Database for DatabaseKv {
 
 						Ok(Some(WorkflowData {
 							workflow_id: sub_workflow_id,
+							name: name_key
+								.deserialize(&name_entry.context("name key should exist")?)?,
 							input,
 							state,
 							output,
@@ -2157,6 +2492,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("get_sub_workflow_tx"))
 			.await
+			.context("failed to get sub workflow")
 			.map_err(WorkflowError::Udb)
 	}
 
@@ -2178,6 +2514,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("publish_signal_tx"))
 			.await
+			.context("failed to publish signal")
 			.map_err(WorkflowError::Udb)?;
 
 		self.bump(BumpSubSubject::SignalPublish {
@@ -2233,6 +2570,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("publish_signal_from_workflow_tx"))
 			.await
+			.context("failed to publish signal from workflow")
 			.map_err(WorkflowError::Udb)?;
 
 		self.bump(BumpSubSubject::SignalPublish { to_workflow_id });
@@ -2290,6 +2628,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("dispatch_sub_workflow_tx"))
 			.await
+			.context("failed to dispatch sub workflow")
 			.map_err(WorkflowError::Udb)?;
 
 		self.bump(BumpSubSubject::Worker);
@@ -2379,6 +2718,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("update_workflow_tags_tx"))
 			.await
+			.context("failed to update workflow tags")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -2413,6 +2753,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("update_workflow_state_tx"))
 			.await
+			.context("failed to update workflow state")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -2450,6 +2791,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("commit_workflow_activity_event_tx"))
 			.await
+			.context("failed to commit activity event")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -2486,6 +2828,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("commit_workflow_message_send_event_tx"))
 			.await
+			.context("failed to commit message send event")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -2544,23 +2887,41 @@ impl Database for DatabaseKv {
 								keys::history::HistorySubspaceVariant::Forgotten,
 							));
 
-					let loop_events_subspace =
-						self.subspace
-							.subspace(&keys::history::EventHistorySubspaceKey::entire(
-								from_workflow_id,
-								location.clone(),
-								false,
-							));
+					// Start is {loop location, 0, ...}
+					let loop_events_subspace_start = self
+						.subspace
+						.subspace(&keys::history::EventHistorySubspaceKey::entire(
+							from_workflow_id,
+							location.clone(),
+							false,
+						))
+						.range()
+						.0;
+					// End is {loop location, iteration - 1, ...}
+					let loop_events_subspace_end = self
+						.subspace
+						.subspace(&keys::history::EventHistorySubspaceKey::new(
+							from_workflow_id,
+							location.clone(),
+							iteration.saturating_sub(1),
+							false,
+						))
+						.range()
+						.1;
 
 					let mut stream = tx.get_ranges_keyvalues(
 						universaldb::RangeOption {
 							mode: StreamingMode::WantAll,
-							..(&loop_events_subspace).into()
+							..(
+								loop_events_subspace_start.as_slice(),
+								loop_events_subspace_end.as_slice(),
+							)
+								.into()
 						},
 						Serializable,
 					);
 
-					// Move all current events under this loop to the forgotten history
+					// Move all events under this loop up to the current iteration to the forgotten history
 					loop {
 						let Some(entry) = stream.try_next().await? else {
 							break;
@@ -2570,7 +2931,7 @@ impl Database for DatabaseKv {
 							return Err(universaldb::tuple::PackError::BadPrefix.into());
 						}
 
-						// Truncate tuple up to ACTIVE and replace it with FORGOTTEN
+						// Truncate tuple up to ...ACTIVE and replace it with ...FORGOTTEN
 						let truncated_key = &entry.key()[active_history_subspace.bytes().len()..];
 						let forgotten_key =
 							[forgotten_history_subspace.bytes(), truncated_key].concat();
@@ -2578,7 +2939,7 @@ impl Database for DatabaseKv {
 						tx.set(&forgotten_key, entry.value());
 					}
 
-					tx.clear_subspace_range(&loop_events_subspace);
+					tx.clear_range(&loop_events_subspace_start, &loop_events_subspace_end);
 
 					// Only retain last 100 events in forgotten history
 					if iteration > 100 {
@@ -2607,6 +2968,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("upsert_loop_event_tx"))
 			.await
+			.context("failed to upsert loop event")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -2640,6 +3002,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("commit_workflow_sleep_event_tx"))
 			.await
+			.context("failed to commit sleep event")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -2668,6 +3031,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("update_workflow_sleep_state_tx"))
 			.await
+			.context("failed to update sleep state")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -2698,6 +3062,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("commit_workflow_branch_event_tx"))
 			.await
+			.context("failed to commit branch event")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -2731,6 +3096,7 @@ impl Database for DatabaseKv {
 			})
 			.custom_instrument(tracing::info_span!("commit_workflow_removed_event_tx"))
 			.await
+			.context("failed to commit removed event")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
@@ -2742,6 +3108,7 @@ impl Database for DatabaseKv {
 		from_workflow_id: Id,
 		location: &Location,
 		version: usize,
+		inner_version: usize,
 		_loop_location: Option<&Location>,
 	) -> WorkflowResult<()> {
 		self.pools
@@ -2754,6 +3121,7 @@ impl Database for DatabaseKv {
 					from_workflow_id,
 					location,
 					version,
+					inner_version,
 					rivet_util::timestamp::now(),
 				)?;
 
@@ -2763,12 +3131,19 @@ impl Database for DatabaseKv {
 				"commit_workflow_version_check_event_tx"
 			))
 			.await
+			.context("failed to commit version check event")
 			.map_err(WorkflowError::Udb)?;
 
 		Ok(())
 	}
 }
 
+impl Drop for DatabaseKv {
+	fn drop(&mut self) {
+		metrics::DB_INSTANCE.dec();
+	}
+}
+
 #[derive(Debug, Clone)]
 struct MinimalPulledWorkflow {
 	workflow_id: Id,
@@ -2779,8 +3154,17 @@ struct MinimalPulledWorkflow {
 
 fn update_metric(
 	tx: &universaldb::Transaction,
-	previous: Option<keys::metric::GaugeMetric>,
-	current: Option<keys::metric::GaugeMetric>,
+	previous: Option<keys::metric::Metric>,
+	current: Option<keys::metric::Metric>,
+) {
+	update_metric_by(tx, previous, current, 1)
+}
+
+fn update_metric_by(
+	tx: &universaldb::Transaction,
+	previous: Option<keys::metric::Metric>,
+	current: Option<keys::metric::Metric>,
+	by: i64,
 ) {
 	if &previous == &current {
 		return;
@@ -2788,16 +3172,43 @@ fn update_metric(
 
 	if let Some(previous) = previous {
 		tx.atomic_op(
-			&keys::metric::GaugeMetricKey::new(previous),
-			&(-1isize).to_le_bytes(),
+			&keys::metric::MetricKey::new(previous),
+			&(by * -1).to_le_bytes(),
 			MutationType::Add,
 		);
 	}
 
 	if let Some(current) = current {
 		tx.atomic_op(
-			&keys::metric::GaugeMetricKey::new(current),
-			&1usize.to_le_bytes(),
+			&keys::metric::MetricKey::new(current),
+			&by.to_le_bytes(),
+			MutationType::Add,
+		);
+	}
+}
+
+fn update_wf_metric(
+	tx: &universaldb::Transaction,
+	workflow_id: Id,
+	previous: Option<keys::workflow::Metric>,
+	current: Option<keys::workflow::Metric>,
+) {
+	if &previous == &current {
+		return;
+	}
+
+	if let Some(previous) = previous {
+		tx.atomic_op(
+			&keys::workflow::MetricKey::new(workflow_id, previous),
+			&(-1i64).to_le_bytes(),
+			MutationType::Add,
+		);
+	}
+
+	if let Some(current) = current {
+		tx.atomic_op(
+			&keys::workflow::MetricKey::new(workflow_id, current),
+			&1i64.to_le_bytes(),
 			MutationType::Add,
 		);
 	}
@@ -2818,6 +3229,10 @@ struct WorkflowHistoryEventBuilder {
 	deadline_ts: Option<i64>,
 	sleep_state: Option<SleepState>,
 	inner_event_type: Option<EventType>,
+	inner_version: Option<usize>,
+
+	indexed_names: Vec<String>,
+	indexed_input_chunks: Vec<Vec<Value>>,
 }
 
 impl WorkflowHistoryEventBuilder {
@@ -2837,6 +3252,10 @@ impl WorkflowHistoryEventBuilder {
 			deadline_ts: None,
 			sleep_state: None,
 			inner_event_type: None,
+			inner_version: None,
+
+			indexed_names: Vec::new(),
+			indexed_input_chunks: Vec::new(),
 		}
 	}
 }
@@ -2860,7 +3279,21 @@ impl TryFrom<WorkflowHistoryEventBuilder> for Event {
 				.ok_or(WorkflowError::MissingEventData("version"))?,
 			data: match event_type {
 				EventType::Activity => EventData::Activity(value.try_into()?),
-				EventType::Signal => EventData::Signal(value.try_into()?),
+				// Deprecated, manually convert to newer type
+				EventType::Signal => {
+					EventData::Signals(SignalsEvent {
+						names: vec![value.name.ok_or(WorkflowError::MissingEventData("name"))?],
+						bodies: vec![if value.input_chunks.is_empty() {
+							return Err(WorkflowError::MissingEventData("input"));
+						} else {
+							// workflow_id not needed
+							let input_key = keys::history::InputKey::new(Id::nil(), value.location);
+							input_key
+								.combine(value.input_chunks)
+								.map_err(WorkflowError::DeserializeEventData)?
+						}],
+					})
+				}
 				EventType::SignalSend => EventData::SignalSend(value.try_into()?),
 				EventType::MessageSend => EventData::MessageSend(value.try_into()?),
 				EventType::SubWorkflow => EventData::SubWorkflow(value.try_into()?),
@@ -2868,7 +3301,8 @@ impl TryFrom<WorkflowHistoryEventBuilder> for Event {
 				EventType::Sleep => EventData::Sleep(value.try_into()?),
 				EventType::Branch => EventData::Branch,
 				EventType::Removed => EventData::Removed(value.try_into()?),
-				EventType::VersionCheck => EventData::VersionCheck,
+				EventType::VersionCheck => EventData::VersionCheck(value.try_into()?),
+				EventType::Signals => EventData::Signals(value.try_into()?),
 			},
 		})
 	}
@@ -2901,27 +3335,6 @@ impl TryFrom<WorkflowHistoryEventBuilder> for ActivityEvent {
 	}
 }
 
-impl TryFrom<WorkflowHistoryEventBuilder> for SignalEvent {
-	type Error = WorkflowError;
-
-	fn try_from(value: WorkflowHistoryEventBuilder) -> WorkflowResult<Self> {
-		Ok(SignalEvent {
-			name: value.name.ok_or(WorkflowError::MissingEventData("name"))?,
-			body: {
-				if value.input_chunks.is_empty() {
-					return Err(WorkflowError::MissingEventData("input"));
-				} else {
-					// workflow_id not needed
-					let input_key = keys::history::InputKey::new(Id::nil(), value.location);
-					input_key
-						.combine(value.input_chunks)
-						.map_err(WorkflowError::DeserializeEventData)?
-				}
-			},
-		})
-	}
-}
-
 impl TryFrom<WorkflowHistoryEventBuilder> for SignalSendEvent {
 	type Error = WorkflowError;
 
@@ -3024,6 +3437,51 @@ impl TryFrom<WorkflowHistoryEventBuilder> for RemovedEvent {
 	}
 }
 
+impl TryFrom<WorkflowHistoryEventBuilder> for VersionCheckEvent {
+	type Error = WorkflowError;
+
+	fn try_from(value: WorkflowHistoryEventBuilder) -> WorkflowResult<Self> {
+		Ok(VersionCheckEvent {
+			// Fallback to event version for old events that don't have inner version
+			inner_version: value.inner_version.unwrap_or(
+				value
+					.version
+					.ok_or(WorkflowError::MissingEventData("version"))?,
+			),
+		})
+	}
+}
+
+impl TryFrom<WorkflowHistoryEventBuilder> for SignalsEvent {
+	type Error = WorkflowError;
+
+	fn try_from(value: WorkflowHistoryEventBuilder) -> WorkflowResult<Self> {
+		Ok(SignalsEvent {
+			names: if value.indexed_names.is_empty() {
+				return Err(WorkflowError::MissingEventData("name"));
+			} else {
+				value.indexed_names
+			},
+			bodies: if value.indexed_input_chunks.is_empty() {
+				return Err(WorkflowError::MissingEventData("input"));
+			} else {
+				value
+					.indexed_input_chunks
+					.into_iter()
+					.map(|input_chunks| {
+						// workflow_id not needed
+						let input_key =
+							keys::history::InputKey::new(Id::nil(), value.location.clone());
+						input_key
+							.combine(input_chunks)
+							.map_err(WorkflowError::DeserializeEventData)
+					})
+					.collect::<std::result::Result<_, _>>()?
+			},
+		})
+	}
+}
+
 fn value_to_str(v: &serde_json::Value) -> WorkflowResult<String> {
 	match v {
 		serde_json::Value::String(s) => Ok(s.clone()),
@@ -3032,13 +3490,16 @@ fn value_to_str(v: &serde_json::Value) -> WorkflowResult<String> {
 }
 
 fn calc_pull_ratio(x: u64, ax: u64, ay: u64, bx: u64, by: u64) -> u64 {
-	// must have neg slope, inversely proportional
+	// Must have neg slope, inversely proportional
 	assert!(ax < bx);
 	assert!(ay > by);
 
-	let neg_dy = ay - by;
+	// Bound domain
+	let x = x.max(ax).min(bx);
+
 	let dx = bx - ax;
-	let neg_b = ay * neg_dy / dx;
+	let neg_dy = ay - by;
+	let b = ay + ax * neg_dy / dx;
 
-	return neg_b.saturating_sub(x * neg_dy / dx);
+	return b.saturating_sub(x * neg_dy / dx);
 }