@rivetkit/engine 1.0.0 → 2.2.1-pr.4600.b74ff3b

package/packages/gasoline/src/db/kv/debug.rs

@@ -2,11 +2,13 @@ use std::{
 	collections::HashMap,
 	ops::Deref,
 	result::Result::{Err, Ok},
+	time::{Duration, Instant},
 };
 
 use anyhow::{Context, Result, ensure};
-use futures_util::{StreamExt, TryStreamExt};
+use futures_util::{StreamExt, TryStreamExt, stream::FuturesUnordered};
 use rivet_util::Id;
+use serde::Serialize;
 use tracing::Instrument;
 use universaldb::utils::{FormalChunkedKey, FormalKey, IsolationLevel::*, end_of_key_range};
 use universaldb::{
@@ -15,6 +17,7 @@ use universaldb::{
 	tuple::{PackResult, TupleDepth, TupleUnpack},
 	value::Value,
 };
+use uuid::Uuid;
 
 use super::{DatabaseKv, keys, update_metric};
 use crate::{
@@ -22,17 +25,21 @@ use crate::{
 		BumpSubSubject,
 		debug::{
 			ActivityError, ActivityEvent, DatabaseDebug, Event, EventData, HistoryData, LoopEvent,
-			MessageSendEvent, SignalData, SignalEvent, SignalSendEvent, SignalState,
+			MessageSendEvent, SignalData, SignalEvent, SignalSendEvent, SignalState, SignalsEvent,
 			SubWorkflowEvent, WorkflowData, WorkflowState,
 		},
 	},
 	error::{WorkflowError, WorkflowResult},
 	history::{
-		event::{EventType, RemovedEvent, SleepEvent, SleepState},
+		event::{EventType, RemovedEvent, SleepEvent, SleepState, VersionCheckEvent},
 		location::Location,
 	},
+	workflow::PruneVariant,
 };
 
+const EARLY_TXN_TIMEOUT: Duration = Duration::from_secs(3);
+const MAX_PRUNES_PER_TXN: usize = 1000;
+
 impl DatabaseKv {
 	#[tracing::instrument(skip_all)]
 	async fn get_workflows_inner(
@@ -518,7 +525,7 @@ impl DatabaseDebug for DatabaseKv {
 							async {
 								tx.get_ranges_keyvalues(
 									RangeOption {
-										mode: StreamingMode::WantAll,
+										mode: StreamingMode::Exact,
 										limit: Some(1),
 										..(&output_subspace).into()
 									},
@@ -616,14 +623,14 @@ impl DatabaseDebug for DatabaseKv {
 
 						// Clear metric
 						let metric = if has_output {
-							keys::metric::GaugeMetric::WorkflowComplete(workflow_name.clone())
+							keys::metric::Metric::WorkflowComplete(workflow_name.clone())
 						} else if has_wake_condition {
 							let error =
 								error_key.deserialize(&error_entry.context("key should exist")?)?;
 
-							keys::metric::GaugeMetric::WorkflowDead(workflow_name.clone(), error)
+							keys::metric::Metric::WorkflowDead(workflow_name.clone(), error)
 						} else {
-							keys::metric::GaugeMetric::WorkflowSleeping(workflow_name.clone())
+							keys::metric::Metric::WorkflowSleeping(workflow_name.clone())
 						};
 
 						update_metric(&tx.with_subspace(self.subspace.clone()), Some(metric), None);
@@ -656,21 +663,21 @@ impl DatabaseDebug for DatabaseKv {
 						let output_subspace = self.subspace.subspace(&output_key);
 
 						let (
-							workflow_name,
+							workflow_name_entry,
 							is_running,
 							has_wake_condition,
 							is_silenced,
 							has_output,
 							error,
 						) = tokio::try_join!(
-							tx.read(&name_key, Serializable),
+							tx.read_opt(&name_key, Serializable),
 							tx.exists(&worker_id_key, Serializable),
 							tx.exists(&has_wake_condition_key, Serializable),
 							tx.exists(&silence_ts_key, Serializable),
 							async {
 								tx.get_ranges_keyvalues(
 									RangeOption {
-										mode: StreamingMode::WantAll,
+										mode: StreamingMode::Exact,
 										limit: Some(1),
 										..(&output_subspace).into()
 									},
@@ -684,11 +691,19 @@ impl DatabaseDebug for DatabaseKv {
 							tx.read_opt(&error_key, Serializable),
 						)?;
 
+						let Some(workflow_name) = workflow_name_entry else {
+							tracing::warn!("workflow {workflow_id} does not exist");
+							continue;
+						};
+
 						if is_running || is_silenced {
 							continue;
 						}
 
-						ensure!(!has_output, "cannot wake a completed workflow");
+						if has_output {
+							tracing::warn!("cannot wake a completed workflow");
+							continue;
+						}
 
 						tx.write(
 							&keys::wake::WorkflowWakeConditionKey::new(
@@ -704,11 +719,11 @@ impl DatabaseDebug for DatabaseKv {
 						if !has_wake_condition {
 							update_metric(
 								&tx,
-								Some(keys::metric::GaugeMetric::WorkflowDead(
+								Some(keys::metric::Metric::WorkflowDead(
 									workflow_name.clone(),
 									error.context("key should exist")?,
 								)),
-								Some(keys::metric::GaugeMetric::WorkflowSleeping(workflow_name)),
+								Some(keys::metric::Metric::WorkflowSleeping(workflow_name)),
 							);
 						}
 					}
@@ -895,6 +910,55 @@ impl DatabaseDebug for DatabaseKv {
 									let inner_event_type = key.deserialize(entry.value())?;
 
 									current_event.inner_event_type = Some(inner_event_type);
+								} else if let Ok(key) = self
+									.subspace
+									.unpack::<keys::history::InnerVersionKey>(entry.key())
+								{
+									let inner_version = key.deserialize(entry.value())?;
+
+									current_event.inner_version = Some(inner_version);
+								} else if let Ok(key) =
+									self.subspace
+										.unpack::<keys::history::IndexedSignalIdKey>(entry.key())
+								{
+									ensure!(
+										current_event.indexed_signal_ids.len() == key.index,
+										"corrupt history, indexed signal doesn't exist yet or is out of order"
+									);
+
+									let signal_id = key.deserialize(entry.value())?;
+									current_event
+										.indexed_signal_ids
+										.insert(key.index, signal_id);
+								} else if let Ok(key) = self
+									.subspace
+									.unpack::<keys::history::IndexedNameKey>(entry.key())
+								{
+									ensure!(
+										current_event.indexed_names.len() == key.index,
+										"corrupt history, indexed name doesn't exist yet or is out of order"
+									);
+
+									let name = key.deserialize(entry.value())?;
+									current_event.indexed_names.insert(key.index, name);
+								} else if let Ok(key) =
+									self.subspace
+										.unpack::<keys::history::IndexedInputChunkKey>(entry.key())
+								{
+									ensure!(
+										current_event.indexed_input_chunks.len() == key.index,
+										"corrupt history, indexed chunk doesn't exist yet or is out of order"
+									);
+
+									if let Some(input_chunks) =
+										current_event.indexed_input_chunks.get_mut(key.index)
+									{
+										input_chunks.push(entry);
+									} else {
+										current_event
+											.indexed_input_chunks
+											.insert(key.index, vec![entry]);
+									}
 								}
 
 								// We ignore keys we don't need (like tags)
@@ -1139,7 +1203,7 @@ impl DatabaseDebug for DatabaseKv {
 						if ack_ts_entry.is_none() {
 							update_metric(
 								&tx.with_subspace(self.subspace.clone()),
-								Some(keys::metric::GaugeMetric::SignalPending(signal_name)),
+								Some(keys::metric::Metric::SignalPending(signal_name)),
 								None,
 							);
 						}
@@ -1151,6 +1215,767 @@ impl DatabaseDebug for DatabaseKv {
 			.await
 			.map_err(Into::into)
 	}
+
+	#[tracing::instrument(skip_all)]
+	async fn revive_workflows(
+		&self,
+		names: &[&str],
+		error_like: &[&str],
+		dry_run: bool,
+		parallelization: u16,
+	) -> Result<usize> {
+		ensure!(parallelization > 0);
+		ensure!(parallelization < 1024);
+
+		let chunk_size = u128::MAX / parallelization as u128;
+		let mut futs = FuturesUnordered::new();
+
+		for i in 0..parallelization as u128 {
+			let start = i * chunk_size;
+			futs.push(self.revive_workflows_inner(
+				names,
+				error_like,
+				dry_run,
+				start,
+				start + chunk_size,
+			));
+		}
+
+		let mut total = 0;
+		while let Some(res) = futs.next().await {
+			total += res?;
+		}
+
+		tracing::info!(?total, "workflows revived");
+
+		Ok(total)
+	}
+
+	#[tracing::instrument(skip_all)]
+	async fn prune_workflows(&self, before_ts: i64) -> Result<usize> {
+		let mut total = 0;
+		let mut last_key = Vec::new();
+		let dc_name = &self.config.dc_name()?;
+
+		#[derive(clickhouse::Row, Serialize)]
+		struct WorkflowTombstone<'a> {
+			datacenter: &'a str,
+			workflow_id: Id,
+			created_at: i64,
+			deleted_at: i64,
+		}
+
+		loop {
+			let (prune_count, inserter, new_last_key) = self
+				.pools
+				.udb()?
+				.run(|tx| {
+					let last_key = &last_key;
+					async move {
+						let start = Instant::now();
+						let tx = tx.with_subspace(self.subspace.clone());
+						let now = rivet_util::timestamp::now();
+						let mut prune_count = 0;
+						let mut new_last_key = Vec::new();
+						let mut inserter = if let Some(ch) = self.pools.clickhouse_option() {
+							Some(
+								ch.clone()
+									.with_database("db_gasoline")
+									.inserter::<WorkflowTombstone>("workflow_tombstones"),
+							)
+						} else {
+							None
+						};
+
+						let subspace_start = self
+							.subspace
+							.subspace(&keys::workflow::PruneIdxKey::entire_subspace())
+							.range()
+							.0;
+						let subspace_end = self
+							.subspace
+							.subspace(&keys::workflow::PruneIdxKey::subspace(before_ts))
+							.range()
+							.1;
+
+						let range_start = if last_key.is_empty() {
+							&subspace_start
+						} else {
+							last_key
+						};
+						let range_end = &subspace_end;
+
+						let mut stream = tx.get_ranges_keyvalues(
+							universaldb::RangeOption {
+								mode: StreamingMode::WantAll,
+								..(range_start.as_slice(), range_end.as_slice()).into()
+							},
+							Serializable,
+						);
+
+						loop {
+							if start.elapsed() > EARLY_TXN_TIMEOUT {
+								tracing::warn!("timed out pruning workflows");
+								break;
+							}
+
+							let Some(entry) = stream.try_next().await? else {
+								new_last_key = Vec::new();
+								break;
+							};
+
+							let prune_key =
+								tx.unpack::<keys::workflow::PruneIdxKey>(entry.key())?;
+
+							// Remove prune idx entry
+							tx.delete(&prune_key);
+
+							match prune_key.variant {
+								PruneVariant::All => {
+									let data_subspace =
+										keys::workflow::DataSubspaceKey::new_with_workflow_id(
+											prune_key.workflow_id,
+										);
+
+									tx.delete_key_subspace(&data_subspace);
+
+									if let (Some(inserter), Some(create_ts)) = (
+										&mut inserter,
+										tx.read_opt(
+											&keys::workflow::CreateTsKey::new(
+												prune_key.workflow_id,
+											),
+											Snapshot,
+										)
+										.await?,
+									) {
+										inserter
+											.write(&WorkflowTombstone {
+												datacenter: dc_name,
+												workflow_id: prune_key.workflow_id,
+												created_at: create_ts,
+												deleted_at: now,
+											})
+											.await?;
+									}
+								}
+								PruneVariant::History => {
+									let history_subspace = keys::history::HistorySubspaceKey::new(
+										prune_key.workflow_id,
+										keys::history::HistorySubspaceVariant::All,
+									);
+
+									tx.delete_key_subspace(&history_subspace);
+									tx.write(
+										&keys::workflow::PruneTsKey::new(prune_key.workflow_id),
+										now,
+									)?;
+								}
+								// Should be unreachable, we don't insert into the idx if prune variant
+								// is `none`
+								PruneVariant::None => continue,
+							}
+
+							prune_count += 1;
+							new_last_key = [entry.key(), &[0xff]].concat();
+
+							if prune_count >= MAX_PRUNES_PER_TXN {
+								break;
+							}
+						}
+
+						Ok((prune_count, inserter, new_last_key))
+					}
+				})
+				.instrument(tracing::info_span!("prune_workflows_tx"))
+				.await?;
+
+			if let Some(mut inserter) = inserter {
+				inserter.force_commit().await?;
+				inserter.end().await?;
+			}
+
+			last_key = new_last_key;
+			total += prune_count;
+
+			if last_key.is_empty() {
+				break;
+			}
+		}
+
+		Ok(total)
+	}
+
+	#[tracing::instrument(skip_all)]
+	async fn prune_complete_workflow_history(
+		&self,
+		names: &[&str],
+		before_ts: i64,
+		dry_run: bool,
+		parallelization: u16,
+	) -> Result<usize> {
+		ensure!(parallelization > 0);
+		ensure!(parallelization < 1024);
+
+		let chunk_size = u128::MAX / parallelization as u128;
+		let mut futs = FuturesUnordered::new();
+
+		for i in 0..parallelization as u128 {
+			let start = i * chunk_size;
+			futs.push(self.prune_workflow_history_inner(
+				names,
+				before_ts,
+				dry_run,
+				start,
+				start + chunk_size,
+			));
+		}
+
+		let mut total = 0;
+		while let Some(res) = futs.next().await {
+			total += res?;
+		}
+
+		tracing::info!(?total, "workflows deleted");
+
+		Ok(total)
+	}
+
+	#[tracing::instrument(skip_all)]
+	async fn prune_acked_signals(
+		&self,
+		names: &[&str],
+		before_ts: i64,
+		dry_run: bool,
+		parallelization: u16,
+		max_per_txn: Option<usize>,
+	) -> Result<usize> {
+		ensure!(parallelization > 0);
+		ensure!(parallelization < 1024);
+
+		let chunk_size = u128::MAX / parallelization as u128;
+		let mut futs = FuturesUnordered::new();
+
+		for i in 0..parallelization as u128 {
+			let start = i * chunk_size;
+			futs.push(self.prune_signals_inner(
+				names,
+				before_ts,
+				dry_run,
+				start,
+				start + chunk_size,
+				max_per_txn,
+			));
+		}
+
+		let mut total = 0;
+		while let Some(res) = futs.next().await {
+			total += res?;
+		}
+
+		tracing::info!(?total, "signals deleted");
+
+		Ok(total)
+	}
+}
+
+impl DatabaseKv {
+	pub async fn revive_workflows_inner(
+		&self,
+		names: &[&str],
+		error_like: &[&str],
+		dry_run: bool,
+		start: u128,
+		end: u128,
+	) -> Result<usize> {
+		let mut total = 0;
+		let mut current_workflow_id = Some(Id::v1(Uuid::from_u128(start), self.config.dc_label()));
+		let end_workflow_id = Id::v1(Uuid::from_u128(end), self.config.dc_label());
+
+		loop {
+			let (new_current_workflow_id, workflow_ids) = self.pools
+				.udb()?
+				.run(|tx| {
+					async move {
+						let mut workflow_ids = Vec::new();
+
+						let key_end = keys::workflow::DataSubspaceKey::new_with_workflow_id(end_workflow_id);
+						let end = self.subspace.subspace(&key_end).range().1;
+
+						let start = if let Some(current_workflow_id) = current_workflow_id {
+							let key_start = keys::workflow::DataSubspaceKey::new_with_workflow_id(current_workflow_id);
+							let mut bytes = self.subspace.subspace(&key_start).range().0;
+
+							if let Some(b) = bytes.last_mut() {
+								*b = 255;
+							}
+
+							bytes
+						} else {
+							let entire_subspace_key = keys::workflow::DataSubspaceKey::new();
+
+							self.subspace.subspace(&entire_subspace_key).range().0
+						};
+
+						let mut stream = tx.get_ranges_keyvalues(
+							RangeOption {
+								mode: StreamingMode::WantAll,
+								..(start, end).into()
+							},
+							Snapshot,
+						);
+
+						let mut workflows_processed = 0;
+						let mut current_workflow_id = None;
+						let mut name_matches = false;
+						let mut state_matches = true;
+						let mut error_matches = error_like.is_empty();
+
+						let fut = async {
+							while let Some(entry) = stream.try_next().await? {
+								let workflow_id = *self.subspace.unpack::<JustId>(entry.key())?;
+
+								if let Some(curr) = current_workflow_id {
+									if workflow_id != curr {
+										// Save if matches query
+										if name_matches && state_matches && error_matches {
+											workflow_ids.push(curr);
+										}
+
+										workflows_processed += 1;
+
+										// Reset state
+										name_matches = false;
+										state_matches = true;
+										error_matches = error_like.is_empty();
+									}
+								}
+
+								current_workflow_id = Some(workflow_id);
+
+								if let Ok(name_key) =
+									self.subspace.unpack::<keys::workflow::NameKey>(entry.key())
+								{
+									let workflow_name = name_key.deserialize(entry.value())?;
+
+									name_matches = names.iter().any(|name| &workflow_name == name);
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::workflow::OutputChunkKey>(entry.key())
+								{
+									state_matches = false;
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::workflow::WorkerIdKey>(entry.key())
+								{
+									state_matches = false;
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::workflow::HasWakeConditionKey>(entry.key())
+								{
+									state_matches = false;
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::workflow::SilenceTsKey>(entry.key())
+								{
+									state_matches = false;
+								} else if let Ok(error_key) = self
+									.subspace
+									.unpack::<keys::workflow::ErrorKey>(entry.key())
+								{
+									let error = error_key.deserialize(entry.value())?.to_lowercase();
+
+									error_matches = error_like.is_empty() || error_like.iter().any(|err| error.contains(err));
+								}
+							}
+
+							if let (Some(workflow_id), true) = (
+								current_workflow_id,
+								name_matches && state_matches && error_matches,
+							) {
+								workflow_ids.push(workflow_id);
+								current_workflow_id = None;
+							}
+
+							if current_workflow_id.is_some() {
+								workflows_processed += 1;
+							}
+
+							anyhow::Ok(())
+						};
+
+						match tokio::time::timeout(EARLY_TXN_TIMEOUT, fut).await {
+							Ok(res) => res?,
+							Err(_) => tracing::debug!("timed out reading workflows"),
+						}
+
+						tracing::info!(?workflows_processed, matching_workflows=?workflow_ids.len(), "batch processed workflows");
+
+						Ok((current_workflow_id, workflow_ids))
+					}
+				})
+				.instrument(tracing::info_span!("find_dead_workflows_tx"))
+				.await?;
+
+			current_workflow_id = new_current_workflow_id;
+			total += workflow_ids.len();
+
+			if !dry_run {
+				self.wake_workflows(workflow_ids).await?;
+			}
+
+			if current_workflow_id.is_none() {
+				tracing::info!("reached end of workflows");
+				break;
+			}
+		}
+
+		Ok(total)
+	}
+
+	pub async fn prune_workflow_history_inner(
+		&self,
+		names: &[&str],
+		before_ts: i64,
+		dry_run: bool,
+		start: u128,
+		end: u128,
+	) -> Result<usize> {
+		let mut total = 0;
+		let mut current_workflow_id = Some(Id::v1(Uuid::from_u128(start), self.config.dc_label()));
+		let end_workflow_id = Id::v1(Uuid::from_u128(end), self.config.dc_label());
+
+		loop {
+			let (new_current_workflow_id, workflow_ids) = self.pools
+				.udb()?
+				.run(|tx| {
+					async move {
+						let mut workflow_ids = Vec::new();
+
+						let key_end = keys::workflow::DataSubspaceKey::new_with_workflow_id(end_workflow_id);
+						let end = self.subspace.subspace(&key_end).range().1;
+
+						let start = if let Some(current_workflow_id) = current_workflow_id {
+							let key_start = keys::workflow::DataSubspaceKey::new_with_workflow_id(current_workflow_id);
+							let mut bytes = self.subspace.subspace(&key_start).range().0;
+
+							if let Some(b) = bytes.last_mut() {
+								*b = 255;
+							}
+
+							bytes
+						} else {
+							let entire_subspace_key = keys::workflow::DataSubspaceKey::new();
+
+							self.subspace.subspace(&entire_subspace_key).range().0
+						};
+
+						let mut stream = tx.get_ranges_keyvalues(
+							RangeOption {
+								mode: StreamingMode::WantAll,
+								..(start, end).into()
+							},
+							Snapshot,
+						);
+
+						let mut workflows_processed = 0;
+						let mut current_workflow_id = None;
+						let mut name_matches = names.is_empty();
+						let mut state_matches = false;
+						let mut ts_matches = false;
+						let mut history_matches = false;
+
+						let fut = async {
+							while let Some(entry) = stream.try_next().await? {
+								let workflow_id = *self.subspace.unpack::<JustId>(entry.key())?;
+
+								if let Some(curr) = current_workflow_id {
+									if workflow_id != curr {
+										// Save if matches query
+										if name_matches && state_matches && ts_matches && history_matches {
+											workflow_ids.push(curr);
+										}
+
+										workflows_processed += 1;
+
+										// Reset state
+										name_matches = names.is_empty();
+										state_matches = false;
+										ts_matches = false;
+										history_matches = false;
+									}
+								}
+
+								current_workflow_id = Some(workflow_id);
+
+								if let Ok(name_key) =
+									self.subspace.unpack::<keys::workflow::NameKey>(entry.key())
+								{
+									let workflow_name = name_key.deserialize(entry.value())?;
+
+									name_matches = names.is_empty() || names.iter().any(|name| &workflow_name == name);
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::workflow::OutputChunkKey>(entry.key())
+								{
+									state_matches = true;
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::workflow::WorkerIdKey>(entry.key())
+								{
+									state_matches = false;
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::workflow::HasWakeConditionKey>(entry.key())
+								{
+									state_matches = false;
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::workflow::PruneTsKey>(entry.key())
+								{
+									state_matches = false;
+								} else if let Ok(create_ts_key) = self
+									.subspace
+									.unpack::<keys::workflow::CreateTsKey>(entry.key())
+								{
+									let create_ts = create_ts_key.deserialize(entry.value())?;
+
+									ts_matches = create_ts < before_ts;
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::history::PartialEventKey>(entry.key())
+								{
+									history_matches = true;
+								}
+							}
+
+							if let (Some(workflow_id), true) = (
+								current_workflow_id,
+								name_matches && state_matches && ts_matches && history_matches
+							) {
+								workflow_ids.push(workflow_id);
+								current_workflow_id = None;
+							}
+
+							if current_workflow_id.is_some() {
+								workflows_processed += 1;
+							}
+
+							anyhow::Ok(())
+						};
+
+						match tokio::time::timeout(EARLY_TXN_TIMEOUT, fut).await {
+							Ok(res) => res?,
+							Err(_) => tracing::debug!("timed out reading workflows"),
+						}
+
+						tracing::info!(?workflows_processed, matching_workflows=?workflow_ids.len(), "batch processed workflows");
+
+						Ok((current_workflow_id, workflow_ids))
+					}
+				})
+				.instrument(tracing::info_span!("find_complete_workflows_tx"))
+				.await?;
+
+			current_workflow_id = new_current_workflow_id;
+			total += workflow_ids.len();
+
+			if !dry_run {
+				let workflow_ids = &workflow_ids;
+				self.pools
+					.udb()?
+					.run(|tx| async move {
+						let tx = tx.with_subspace(self.subspace.clone());
+						let now = rivet_util::timestamp::now();
+
+						for workflow_id in workflow_ids {
+							let history_subspace = keys::history::HistorySubspaceKey::new(
+								*workflow_id,
+								keys::history::HistorySubspaceVariant::All,
+							);
+
+							tx.delete_key_subspace(&history_subspace);
+
+							tx.write(&keys::workflow::PruneTsKey::new(*workflow_id), now)?;
+						}
+
+						Ok(())
+					})
+					.await?;
+			}
+
+			if current_workflow_id.is_none() {
+				tracing::info!("reached end of workflows");
+				break;
+			}
+		}
+
+		Ok(total)
+	}
+
+	pub async fn prune_signals_inner(
+		&self,
+		names: &[&str],
+		before_ts: i64,
+		dry_run: bool,
+		start: u128,
+		end: u128,
+		max_per_txn: Option<usize>,
+	) -> Result<usize> {
+		let mut total = 0;
+		let mut current_signal_id = Some(Id::v1(Uuid::from_u128(start), self.config.dc_label()));
+		let end_signal_id = Id::v1(Uuid::from_u128(end), self.config.dc_label());
+
+		loop {
+			let (new_current_signal_id, signal_ids) = self.pools
+				.udb()?
+				.run(|tx| {
+					async move {
+						let mut signal_ids = Vec::new();
+
+						let key_end = keys::signal::DataSubspaceKey::new_with_signal_id(end_signal_id);
+						let end = self.subspace.subspace(&key_end).range().1;
+
+						let start = if let Some(current_signal_id) = current_signal_id {
+							let key_start = keys::signal::DataSubspaceKey::new_with_signal_id(current_signal_id);
+							let mut bytes = self.subspace.subspace(&key_start).range().0;
+
+							if let Some(b) = bytes.last_mut() {
+								*b = 255;
+							}
+
+							bytes
+						} else {
+							let entire_subspace_key = keys::signal::DataSubspaceKey::new();
+
+							self.subspace.subspace(&entire_subspace_key).range().0
+						};
+
+						let mut stream = tx.get_ranges_keyvalues(
+							RangeOption {
+								mode: StreamingMode::WantAll,
+								..(start, end).into()
+							},
+							Snapshot,
+						);
+
+						let mut signals_processed = 0;
+						let mut current_signal_id = None;
+						let mut name_matches = names.is_empty();
+						let mut state_matches = false;
+						let mut ts_matches = false;
+
+						let fut = async {
+							loop {
+								let Some(entry) = stream.try_next().await? else {
+									break;
+								};
+
+								let signal_id = *self.subspace.unpack::<JustId>(entry.key())?;
+
+								if let Some(curr) = current_signal_id {
+									if signal_id != curr {
+										// Save if matches query
+										if name_matches && state_matches && ts_matches {
+											signal_ids.push(curr);
+
+											if let Some(max_per_txn) = max_per_txn {
+												if signal_ids.len() >= max_per_txn {
+													return anyhow::Ok(());
+												}
+											}
+										}
+
+										signals_processed += 1;
+
+										// Reset state
+										name_matches = names.is_empty();
+										state_matches = false;
+										ts_matches = false;
+									}
+								}
+
+								current_signal_id = Some(signal_id);
+
+								if let Ok(name_key) =
+									self.subspace.unpack::<keys::signal::NameKey>(entry.key())
+								{
+									let signal_name = name_key.deserialize(entry.value())?;
+
+									name_matches = names.is_empty() || names.iter().any(|name| &signal_name == name);
+								} else if let Ok(_) = self
+									.subspace
+									.unpack::<keys::signal::AckTsKey>(entry.key())
+								{
+									state_matches = true;
+								} else if let Ok(create_ts_key) = self
+									.subspace
+									.unpack::<keys::signal::CreateTsKey>(entry.key())
+								{
+									let create_ts = create_ts_key.deserialize(entry.value())?;
+
+									ts_matches = create_ts < before_ts;
+								}
+							}
+
+							if let (Some(signal_id), true) = (
+								current_signal_id,
+								name_matches && state_matches && ts_matches
+							) {
+								signal_ids.push(signal_id);
+								current_signal_id = None;
+							}
+
+							if current_signal_id.is_some() {
+								signals_processed += 1;
+							}
+
+							anyhow::Ok(())
+						};
+
+						match tokio::time::timeout(EARLY_TXN_TIMEOUT, fut).await {
+							Ok(res) => res?,
+							Err(_) => tracing::debug!("timed out reading signals"),
+						}
+
+						tracing::info!(?signals_processed, matching_signals=?signal_ids.len(), "batch processed signals");
+
+						Ok((current_signal_id, signal_ids))
+					}
+				})
+				.instrument(tracing::info_span!("find_acked_signals_tx"))
+				.await?;
+
+			current_signal_id = new_current_signal_id;
+			total += signal_ids.len();
+
+			if !dry_run {
+				let signal_ids = &signal_ids;
+				self.pools
+					.udb()?
+					.run(|tx| async move {
+						let tx = tx.with_subspace(self.subspace.clone());
+
+						for signal_id in signal_ids {
+							let data_subspace =
+								keys::signal::DataSubspaceKey::new_with_signal_id(*signal_id);
+
+							tx.delete_key_subspace(&data_subspace);
+						}
+
+						Ok(())
+					})
+					.await?;
+			}
+
+			if current_signal_id.is_none() {
+				tracing::info!("reached end of signals");
+				break;
+			}
+		}
+
+		Ok(total)
+	}
 }
 
 // Parses Id in third position, ignores the rest
@@ -1190,6 +2015,11 @@ struct WorkflowHistoryEventBuilder {
 	deadline_ts: Option<i64>,
 	sleep_state: Option<SleepState>,
 	inner_event_type: Option<EventType>,
+	inner_version: Option<usize>,
+
+	indexed_signal_ids: Vec<Id>,
+	indexed_names: Vec<String>,
+	indexed_input_chunks: Vec<Vec<Value>>,
 }
 
 impl WorkflowHistoryEventBuilder {
@@ -1211,6 +2041,11 @@ impl WorkflowHistoryEventBuilder {
 			deadline_ts: None,
 			sleep_state: None,
 			inner_event_type: None,
+			inner_version: None,
+
+			indexed_signal_ids: Vec::new(),
+			indexed_names: Vec::new(),
+			indexed_input_chunks: Vec::new(),
 		}
 	}
 }
@@ -1242,7 +2077,8 @@ impl TryFrom<WorkflowHistoryEventBuilder> for Event {
 				EventType::Sleep => EventData::Sleep(value.try_into()?),
 				EventType::Branch => EventData::Branch,
 				EventType::Removed => EventData::Removed(value.try_into()?),
-				EventType::VersionCheck => EventData::VersionCheck,
+				EventType::VersionCheck => EventData::VersionCheck(value.try_into()?),
+				EventType::Signals => EventData::Signals(value.try_into()?),
 			},
 		})
 	}
@@ -1491,3 +2327,57 @@ impl TryFrom<WorkflowHistoryEventBuilder> for RemovedEvent {
 		})
 	}
 }
+
+impl TryFrom<WorkflowHistoryEventBuilder> for SignalsEvent {
+	type Error = WorkflowError;
+
+	fn try_from(value: WorkflowHistoryEventBuilder) -> WorkflowResult<Self> {
+		Ok(SignalsEvent {
+			signal_ids: if value.indexed_signal_ids.is_empty() {
+				return Err(WorkflowError::MissingEventData("signal_id"));
+			} else {
+				value.indexed_signal_ids
+			},
+			names: if value.indexed_names.is_empty() {
+				return Err(WorkflowError::MissingEventData("name"));
+			} else {
+				value.indexed_names
+			},
+			bodies: if value.indexed_input_chunks.is_empty() {
+				return Err(WorkflowError::MissingEventData("input"));
+			} else {
+				value
+					.indexed_input_chunks
+					.into_iter()
+					.map(|input_chunks| {
+						// workflow_id not needed
+						let input_key =
+							keys::history::InputKey::new(Id::nil(), value.location.clone());
+						let v = input_key
+							.combine(input_chunks)
+							.map_err(WorkflowError::DeserializeEventData)?;
+
+						serde_json::from_str::<serde_json::Value>(v.get())
+							.map_err(Into::into)
+							.map_err(WorkflowError::DeserializeEventData)
+					})
+					.collect::<std::result::Result<_, _>>()?
+			},
+		})
+	}
+}
+
+impl TryFrom<WorkflowHistoryEventBuilder> for VersionCheckEvent {
+	type Error = WorkflowError;
+
+	fn try_from(value: WorkflowHistoryEventBuilder) -> WorkflowResult<Self> {
+		Ok(VersionCheckEvent {
+			// Fallback to event version for old events that don't have inner version
+			inner_version: value.inner_version.unwrap_or(
+				value
+					.version
+					.ok_or(WorkflowError::MissingEventData("version"))?,
+			),
+		})
+	}
+}