hecks 5.0.0 → 6.0.0

data/rust/host/src/{mint.rs → store/mint.rs}

@@ -1,7 +1,7 @@
 //! Mints Postgres eras, porting `lib/hecks/adapters/driven/postgres_era/
 //! lineage/*.rb`'s SQL verbatim; this crate has no bluebook parser (ADR 0012).
 
-use crate::journal::quote_ident;
+use super::journal_sql::quote_ident;
 use crate::reference_transform;
 use crate::reference_validate;
 use crate::storage_shape;
@@ -11,7 +11,7 @@ use sha2::{Digest, Sha256};
 use tokio_postgres::GenericClient;
 
 fn journal_table(domain: &str) -> String {
-    format!("hecks_journal_{}", crate::journal::snake(domain))
+    format!("hecks_journal_{}", super::journal_sql::snake(domain))
 }
 
 fn sequence(domain: &str) -> String {
@@ -25,15 +25,15 @@ fn partition(domain: &str, era: i32) -> String {
 // Domain-qualified: two domains whose aggregate names snake_case to the
 // same storage_name would otherwise collide on the same relations (docs/decisions/0059).
 fn head_snapshot(domain: &str, storage_name: &str, era: i32) -> String {
-    crate::journal::qualified_name(domain, &format!("{storage_name}_head_snapshot_{era}"))
+    super::journal_sql::qualified_name(domain, &format!("{storage_name}_head_snapshot_{era}"))
 }
 
 fn matview(domain: &str, storage_name: &str, era: i32, label: &str) -> String {
-    crate::journal::qualified_name(domain, &format!("{storage_name}_lineage_{era}_{label}"))
+    super::journal_sql::qualified_name(domain, &format!("{storage_name}_lineage_{era}_{label}"))
 }
 
 fn head_view(domain: &str, storage_name: &str) -> String {
-    crate::journal::qualified_name(domain, &format!("{storage_name}_head"))
+    super::journal_sql::qualified_name(domain, &format!("{storage_name}_head"))
 }
 
 fn text_literal(text: &str) -> String {
@@ -175,7 +175,7 @@ impl BootDecision {
 }
 
 /// `held` must already be in ordinal order (as `journal::held_eras` returns it).
-pub fn decide_boot_action(held: &[crate::journal::HeldEra], my_label: &str) -> BootDecision {
+pub fn decide_boot_action(held: &[super::journal_sql::HeldEra], my_label: &str) -> BootDecision {
     if let Some(matching) = held
         .iter()
         .find(|held_era| held_era.label.as_deref() == Some(my_label))
@@ -688,7 +688,7 @@ fn names_by_era(current_name: &str, edges: &[&Edge]) -> (Vec<String>, Vec<String
     }
     let storage = current
         .iter()
-        .map(|name| crate::journal::snake(name))
+        .map(|name| super::journal_sql::snake(name))
         .collect();
     (current, storage)
 }
@@ -1337,7 +1337,7 @@ async fn mint_era_body<C: GenericClient>(
         anyhow::bail!("era {ordinal} of {domain} is already minted — nothing to do");
     }
 
-    let watermark = crate::journal::last_ordinal(client, domain).await?;
+    let watermark = super::journal_sql::last_ordinal(client, domain).await?;
     let digest = format!("{:x}", Sha256::digest(held_text.as_bytes()));
     client
         .execute(
@@ -1351,7 +1351,7 @@ async fn mint_era_body<C: GenericClient>(
 
     // Every held era's watermark, for `ancestor_tail_sql`'s cut — includes
     // the row just inserted, which bounds era `ordinal - 1`'s tail read.
-    let held = crate::journal::held_eras(client, domain).await?;
+    let held = super::journal_sql::held_eras(client, domain).await?;
     let watermarks: std::collections::HashMap<i32, Option<i64>> = held
         .iter()
         .map(|era| (era.ordinal, era.watermark))
@@ -1618,8 +1618,8 @@ mod tests {
             "state is nullable, for a delete tombstone"
         );
     }
+    use super::super::journal_sql as journal;
     use super::*;
-    use crate::journal;
     use tokio_postgres::NoTls;
 
     fn rule(name: &str, was: Option<&str>, expression: &str) -> EdgeAggregate {
@@ -1699,9 +1699,7 @@ mod tests {
     fn held(ordinal: i32, label: Option<&str>) -> journal::HeldEra {
         journal::HeldEra {
             ordinal,
-            hash: label.map(|_| "irrelevant".to_string()),
             label: label.map(str::to_string),
-            held_text: String::new(),
             watermark: None,
         }
     }
@@ -1848,6 +1846,7 @@ mod tests {
             domain: domain.to_string(),
             era: Some(1),
             mirrored: None,
+            seal_every: None,
         };
         journal::append_lineage_mutation(
             &client,
@@ -2610,6 +2609,7 @@ mod tests {
                 domain: domain.to_string(),
                 era: Some(1),
                 mirrored: None,
+                seal_every: None,
             };
             journal::append_lineage_mutation(
                 client,

data/rust/host/src/store/postgres.rs

@@ -0,0 +1,493 @@
+//! The Postgres backend: one connection, one transaction per [`Unit`].
+//!
+//! The domain lock is a transaction-scoped advisory lock, and the `journal.rs` functions run
+//! the SQL unchanged inside the open transaction.
+
+use super::{Accepted, DeadLetter, State, StoreError};
+use crate::store::journal_sql as journal;
+use std::future::Future;
+use std::sync::Arc;
+use std::task::{Context, Poll, Waker};
+use tokio::sync::{Mutex, MutexGuard};
+use tokio_postgres::Client;
+
+/// A domain's journal, snapshot and sagas in Postgres tables, over one connection.
+pub struct PostgresStore {
+    lineage: LineageStore,
+}
+
+/// The era, head-view and member-lookup surface of a Postgres store: the part of the host that
+/// reads and writes the lineage relations directly, and so has no other backend.
+pub struct LineageStore {
+    client: Arc<Mutex<Client>>,
+}
+
+impl LineageStore {
+    /// Every live `state` document of the first head that exists, trying the head view of each of
+    /// `domains` in turn, then the plain `<storage>_head` view and the plain table, because
+    /// different runtimes have written these rows under different names. A relation that exists
+    /// but fails to read still errors; none existing answers empty.
+    pub async fn head_states(
+        &self,
+        domains: &[&str],
+        storage_name: &str,
+    ) -> anyhow::Result<Vec<serde_json::Value>> {
+        let client = self.lock().await;
+        for relation in head_relations(domains, storage_name) {
+            // `to_regclass` resolves through the connection's own `search_path`, so a same-named
+            // relation in another schema is never read.
+            let exists: bool = client
+                .query_one("SELECT to_regclass($1) IS NOT NULL", &[&relation])
+                .await?
+                .get(0);
+            if !exists {
+                continue;
+            }
+            let rows = client
+                .query(
+                    &format!(
+                        "SELECT state FROM {} ORDER BY id COLLATE \"C\"",
+                        journal::quote_ident(&relation)
+                    ),
+                    &[],
+                )
+                .await?;
+            return Ok(rows.iter().map(|row| row.get(0)).collect());
+        }
+        Ok(Vec::new())
+    }
+
+    /// The head row whose `matches` fields all hold, read from the first of `domains` whose head
+    /// view exists (then the plain `<storage>_head` view); that view decides, so a miss in it is
+    /// `None` and the later views are not asked. Each match is `(field, value)`, comparing the
+    /// field's single-attribute value object to `value`.
+    pub async fn head_find(
+        &self,
+        domains: &[&str],
+        storage_name: &str,
+        matches: &[(&str, &str)],
+    ) -> Result<Option<serde_json::Value>, StoreError> {
+        let mut relations: Vec<String> = domains
+            .iter()
+            .map(|domain| journal::head_view(domain, storage_name))
+            .collect();
+        relations.push(format!("{storage_name}_head"));
+        relations.dedup();
+        let conditions: Vec<String> = matches
+            .iter()
+            .enumerate()
+            .map(|(i, (field, _))| {
+                format!(
+                    "state->'{}'->>'value' = ${}",
+                    field.replace('\'', "''"),
+                    i + 1
+                )
+            })
+            .collect();
+        let values: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = matches
+            .iter()
+            .map(|(_, value)| value as &(dyn tokio_postgres::types::ToSql + Sync))
+            .collect();
+        let client = self.lock().await;
+        for relation in relations {
+            let sql = format!(
+                "SELECT state FROM {} WHERE {}",
+                journal::quote_ident(&relation),
+                conditions.join(" AND ")
+            );
+            match client.query_opt(&sql, &values).await {
+                Ok(row) => return Ok(row.map(|row| row.get(0))),
+                Err(e) => {
+                    let e = pg(e);
+                    if !e.is_not_provisioned() {
+                        return Err(e);
+                    }
+                }
+            }
+        }
+        Ok(None)
+    }
+
+    /// Writes one head row under the domain's ordinal lock, in a transaction of its own.
+    pub async fn put_head(
+        &self,
+        config: &journal::LineageConfig,
+        mutation: &journal::Mutation<'_>,
+    ) -> anyhow::Result<()> {
+        let mut client = self.lock().await;
+        let txn = client.transaction().await?;
+        txn.execute(
+            "SELECT pg_advisory_xact_lock(hashtext('hecks_ordinal:' || $1))",
+            &[&config.domain],
+        )
+        .await?;
+        journal::append_lineage_mutation(&txn, config, mutation).await?;
+        txn.commit().await?;
+        Ok(())
+    }
+
+    /// Reads one aggregate's head rows and lets `decide` answer with at most one `(id, state)`
+    /// to write, all under the domain's ordinal lock in one transaction; a `decide` that writes
+    /// nothing leaves the heads untouched.
+    pub async fn amend_head<T>(
+        &self,
+        config: &journal::LineageConfig,
+        domain: &str,
+        aggregate: &str,
+        storage_name: &str,
+        decide: impl FnOnce(&[(String, serde_json::Value)]) -> (T, Option<(String, serde_json::Value)>),
+    ) -> anyhow::Result<T> {
+        let mut client = self.lock().await;
+        let txn = client.transaction().await?;
+        txn.execute(
+            "SELECT pg_advisory_xact_lock(hashtext('hecks_ordinal:' || $1))",
+            &[&config.domain],
+        )
+        .await?;
+        let rows = journal::read_lineage_head_all(&txn, domain, storage_name).await?;
+        let (answer, write) = decide(&rows);
+        if let Some((id, state)) = write {
+            let mutation = journal::Mutation {
+                aggregate,
+                id: &id,
+                operation: "save",
+                state: &state,
+            };
+            journal::append_lineage_mutation(&txn, config, &mutation).await?;
+            txn.commit().await?;
+        }
+        Ok(answer)
+    }
+
+    /// The one connection, held for as long as the guard lives; its mutex serializes every use.
+    pub async fn lock(&self) -> MutexGuard<'_, Client> {
+        self.client.lock().await
+    }
+}
+
+impl PostgresStore {
+    /// A store over `client`, which every use of this store shares.
+    #[must_use]
+    pub fn new(client: Client) -> Self {
+        StoreError::teach_not_provisioned(is_missing_relation);
+        Self {
+            lineage: LineageStore {
+                client: Arc::new(Mutex::new(client)),
+            },
+        }
+    }
+
+    /// The lineage surface over the same connection.
+    pub(super) const fn lineage(&self) -> &LineageStore {
+        &self.lineage
+    }
+
+    /// Opens a transaction and takes the advisory lock for `domain` inside it.
+    ///
+    /// The lock covers the whole read-then-append sequence, not just the insert: two concurrent
+    /// invocations could otherwise both replay the same prior steps and both pass a uniqueness
+    /// check the domain thinks it is enforcing. It is scoped to the domain so unrelated domains
+    /// sharing one Postgres instance do not serialize against each other.
+    pub(super) async fn begin(&self, domain: &str) -> Result<PostgresUnit<'_>, StoreError> {
+        let guard = self.lineage.client.lock().await;
+        let mut unit = PostgresUnit { guard, open: false };
+        unit.guard.batch_execute("BEGIN").await.map_err(pg)?;
+        unit.open = true;
+        unit.guard
+            .execute(
+                "SELECT pg_advisory_xact_lock(hashtext('hecks_lambda_journal.' || $1::text))",
+                &[&domain],
+            )
+            .await
+            .map_err(pg)?;
+        Ok(unit)
+    }
+
+    /// The snapshot and the steps after it (or every step), read without the domain lock.
+    pub(super) async fn read_state(&self) -> Result<State, StoreError> {
+        let guard = self.lineage.client.lock().await;
+        let snapshot = journal::load_snapshot(&*guard).await?;
+        let steps = match &snapshot {
+            Some(s) => journal::load_steps_after(&*guard, s.ordinal).await?,
+            None => journal::load_steps(&*guard).await?,
+        };
+        Ok(State {
+            snapshot,
+            steps,
+            sagas: Vec::new(),
+        })
+    }
+
+    /// Inserts the dead-letter row on the shared connection, outside any transaction.
+    pub(super) async fn record_dead_letter(
+        &self,
+        dead_letter: &DeadLetter<'_>,
+    ) -> Result<(), StoreError> {
+        let guard = self.lineage.client.lock().await;
+        journal::record_dead_letter(
+            &*guard,
+            dead_letter.policy,
+            dead_letter.target_domain,
+            dead_letter.target_verb,
+            dead_letter.payload,
+            dead_letter.error,
+            dead_letter.attempts,
+        )
+        .await?;
+        Ok(())
+    }
+}
+
+/// One open transaction holding the domain's advisory lock and the connection's mutex.
+///
+/// Dropping it without committing rolls the transaction back, which also releases the lock.
+pub struct PostgresUnit<'a> {
+    guard: MutexGuard<'a, Client>,
+    open: bool,
+}
+
+impl PostgresUnit<'_> {
+    /// Loads snapshot, replay steps and sagas inside the open transaction.
+    pub(super) async fn load(&self) -> Result<State, StoreError> {
+        let snapshot = journal::load_snapshot(&*self.guard).await?;
+        let mut state = State {
+            snapshot,
+            steps: Vec::new(),
+            sagas: Vec::new(),
+        };
+        state.steps = match &state.snapshot {
+            Some(s) if !state.needs_saga_backfill() => {
+                journal::load_steps_after(&*self.guard, s.ordinal).await?
+            }
+            _ => journal::load_steps(&*self.guard).await?,
+        };
+        state.sagas = journal::load_sagas(&*self.guard).await?;
+        Ok(state)
+    }
+
+    /// Writes the journal row, snapshot, sagas and lineage mutations, then commits.
+    pub(super) async fn commit(mut self, accepted: Accepted<'_>) -> Result<(), StoreError> {
+        let client = &*self.guard;
+        let ordinal = journal::append(client, accepted.verb, accepted.args).await?;
+        journal::save_snapshot(client, ordinal, &accepted.new_seed).await?;
+        for saga in &accepted.saga_upserts {
+            journal::save_saga(
+                client,
+                &saga.process_manager,
+                &saga.correlation,
+                &saga.state,
+                &saga.memory,
+                &saga.completed_compensations,
+            )
+            .await?;
+        }
+        for (process_manager, correlation) in &accepted.saga_deletes {
+            journal::delete_saga(client, process_manager, correlation).await?;
+        }
+        if let Some(lineage) = &accepted.lineage {
+            for mutation in &lineage.mutations {
+                journal::append_lineage_mutation(client, lineage.config, mutation).await?;
+            }
+            // Sealed here, under the domain lock and inside the transaction, so the seal and the
+            // rows it covers land together or not at all.
+            if let Some(every) = lineage.config.seal_every {
+                crate::ledger::seal_due(client, &lineage.config.domain, every).await?;
+            }
+        }
+        client.batch_execute("COMMIT").await.map_err(pg)?;
+        self.open = false;
+        Ok(())
+    }
+
+    /// Rolls the transaction back and waits for the server to confirm.
+    pub(super) async fn rollback(mut self) -> Result<(), StoreError> {
+        self.guard.batch_execute("ROLLBACK").await.map_err(pg)?;
+        self.open = false;
+        Ok(())
+    }
+}
+
+impl Drop for PostgresUnit<'_> {
+    /// Sends the rollback of an unfinished transaction ahead of anything the next holder of
+    /// the connection sends. Dropping cannot await, so the request is queued by polling the
+    /// call once, the way `tokio_postgres::Transaction` rolls itself back.
+    fn drop(&mut self) {
+        if self.open {
+            let mut rollback = Box::pin(self.guard.batch_execute("ROLLBACK"));
+            let _: Poll<_> = rollback
+                .as_mut()
+                .poll(&mut Context::from_waker(Waker::noop()));
+        }
+    }
+}
+
+/// Whether a failure is Postgres reporting that a table or view does not exist.
+fn is_missing_relation(error: &anyhow::Error) -> bool {
+    error
+        .chain()
+        .filter_map(|cause| cause.downcast_ref::<tokio_postgres::Error>())
+        .any(|db| db.code() == Some(&tokio_postgres::error::SqlState::UNDEFINED_TABLE))
+}
+
+/// Wraps a driver error as a store failure, keeping the not-provisioned test working.
+fn pg(error: tokio_postgres::Error) -> StoreError {
+    anyhow::Error::from(error).into()
+}
+
+/// The relations a head may live in, in the order they are tried: each domain's head view, the
+/// plain `<storage>_head` view, then the plain table. Deduplicated, so a domain named twice never
+/// probes one relation twice.
+fn head_relations(domains: &[&str], storage_name: &str) -> Vec<String> {
+    let mut relations: Vec<String> = domains
+        .iter()
+        .map(|domain| journal::head_view(domain, storage_name))
+        .collect();
+    relations.push(format!("{storage_name}_head"));
+    relations.push(storage_name.to_string());
+    relations.dedup();
+    relations
+}
+
+#[async_trait::async_trait]
+impl super::Backend for PostgresStore {
+    async fn begin<'a>(
+        &'a self,
+        domain: &str,
+    ) -> Result<Box<dyn super::UnitBackend + 'a>, StoreError> {
+        Ok(Box::new(Self::begin(self, domain).await?))
+    }
+
+    async fn read_state(&self) -> Result<State, StoreError> {
+        Self::read_state(self).await
+    }
+
+    async fn record_dead_letter(&self, dead_letter: &DeadLetter<'_>) -> Result<(), StoreError> {
+        Self::record_dead_letter(self, dead_letter).await
+    }
+
+    async fn head_by_id(
+        &self,
+        domain: &str,
+        storage_name: &str,
+        id: &str,
+    ) -> Result<Option<serde_json::Value>, StoreError> {
+        let guard = self.lineage.lock().await;
+        Ok(journal::read_lineage_head_by_id(&*guard, domain, storage_name, id).await?)
+    }
+
+    async fn head_all(
+        &self,
+        domain: &str,
+        storage_name: &str,
+    ) -> Result<Vec<(String, serde_json::Value)>, StoreError> {
+        let guard = self.lineage.lock().await;
+        Ok(journal::read_lineage_head_all(&*guard, domain, storage_name).await?)
+    }
+
+    async fn head_states(
+        &self,
+        domains: &[&str],
+        storage_name: &str,
+    ) -> Result<Vec<serde_json::Value>, StoreError> {
+        Ok(self.lineage.head_states(domains, storage_name).await?)
+    }
+
+    async fn head_find(
+        &self,
+        domains: &[&str],
+        storage_name: &str,
+        matches: &[(&str, &str)],
+    ) -> Result<Option<serde_json::Value>, StoreError> {
+        self.lineage.head_find(domains, storage_name, matches).await
+    }
+
+    async fn put_head(
+        &self,
+        config: &journal::LineageConfig,
+        mutation: &journal::Mutation<'_>,
+    ) -> Result<(), StoreError> {
+        Ok(self.lineage.put_head(config, mutation).await?)
+    }
+
+    async fn amend_head(
+        &self,
+        config: &journal::LineageConfig,
+        domain: &str,
+        aggregate: &str,
+        storage_name: &str,
+        decide: &mut (dyn for<'r> FnMut(
+            &'r [(String, serde_json::Value)],
+        ) -> Option<(String, serde_json::Value)>
+                  + Send),
+    ) -> Result<(), StoreError> {
+        self.lineage
+            .amend_head(
+                config,
+                domain,
+                aggregate,
+                storage_name,
+                |rows: &[(String, serde_json::Value)]| ((), decide(rows)),
+            )
+            .await?;
+        Ok(())
+    }
+
+    async fn held_eras(&self, domain: &str) -> Result<Vec<journal::HeldEra>, StoreError> {
+        let guard = self.lineage.lock().await;
+        Ok(journal::held_eras(&*guard, domain).await?)
+    }
+
+    async fn current_era(&self, domain: &str) -> Result<Option<i32>, StoreError> {
+        let guard = self.lineage.lock().await;
+        Ok(journal::current_era(&guard, domain).await?)
+    }
+
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+}
+
+#[async_trait::async_trait]
+impl super::UnitBackend for PostgresUnit<'_> {
+    async fn load(&self) -> Result<State, StoreError> {
+        Self::load(self).await
+    }
+
+    async fn commit(self: Box<Self>, accepted: Accepted<'_>) -> Result<(), StoreError> {
+        (*self).commit(accepted).await
+    }
+
+    async fn rollback(self: Box<Self>) -> Result<(), StoreError> {
+        (*self).rollback().await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::head_relations;
+
+    #[test]
+    fn head_relations_cover_every_relation_a_runtime_has_written_these_rows_to() {
+        assert_eq!(
+            head_relations(&["SampleApp", "ConsoleSettings"], "state_style"),
+            vec![
+                "sample_app_state_style_head".to_string(),
+                "console_settings_state_style_head".to_string(),
+                "state_style_head".to_string(),
+                "state_style".to_string()
+            ]
+        );
+    }
+
+    #[test]
+    fn a_domain_named_twice_never_probes_one_relation_twice() {
+        assert_eq!(
+            head_relations(&["ConsoleSettings", "ConsoleSettings"], "state_style"),
+            vec![
+                "console_settings_state_style_head".to_string(),
+                "state_style_head".to_string(),
+                "state_style".to_string()
+            ]
+        );
+    }
+}

data/rust/host/src/store.rs

@@ -0,0 +1,97 @@
+//! The Postgres side of the store: the backend, the journal's SQL and the era minting, over the
+//! [`Store`] and [`Backend`] that `rust_host_core` owns.
+//!
+//! The core knows only the traits. A store over Postgres is made here, and the host crate's
+//! tools (minting, the ledger, the harnesses) reach the connection through [`LineageStore`].
+
+/// The journal's SQL: the Postgres adapter's reads and writes, with the journal's data re-exported.
+pub mod journal_sql;
+/// Era minting: the Postgres adapter's provisioning of eras, heads and lineage.
+pub mod mint;
+mod postgres;
+
+pub use postgres::{LineageStore, PostgresStore, PostgresUnit};
+pub use rust_host_core::store::*;
+use tokio_postgres::Client;
+
+/// A store over one Postgres connection, whose mutex serializes every use of it.
+#[must_use]
+pub fn postgres_store(client: Client) -> Store {
+    Store::new(PostgresStore::new(client))
+}
+
+/// The Postgres reach into a [`Store`] that the core does not offer: the connection handle, and in
+/// tests the raw rows beneath the seam.
+pub trait StoreExt {
+    /// The Postgres adapter's own connection handle, for minting, seals and operator tools; code
+    /// that runs a domain uses the head methods and never this.
+    fn lineage(&self) -> Option<&LineageStore>;
+}
+
+impl StoreExt for Store {
+    fn lineage(&self) -> Option<&LineageStore> {
+        self.backend()
+            .as_any()
+            .downcast_ref::<PostgresStore>()
+            .map(PostgresStore::lineage)
+    }
+}
+
+/// Raw access for test fixtures and assertions that reach beneath the seam.
+#[cfg(test)]
+#[allow(async_fn_in_trait)]
+pub trait StoreFixtures {
+    /// Every journaled step in order, whichever backend holds them.
+    async fn journal_steps(&self) -> Vec<serde_json::Value>;
+    /// Every dead letter as `(policy, target_domain, target_verb, attempts)`, ordered by policy.
+    async fn dead_letter_rows(&self) -> Vec<(String, String, String, i32)>;
+    /// The raw connection of a Postgres store.
+    async fn lock(&self) -> tokio::sync::MutexGuard<'_, Client>;
+}
+
+#[cfg(test)]
+impl StoreFixtures for Store {
+    async fn journal_steps(&self) -> Vec<serde_json::Value> {
+        if let Some(store) = self.backend().as_any().downcast_ref::<PostgresStore>() {
+            let guard = store.lineage().lock().await;
+            return journal_sql::load_steps(&*guard).await.unwrap();
+        }
+        let memory = self.backend().as_any().downcast_ref::<MemoryStore>();
+        memory
+            .expect("a Postgres or Memory store")
+            .committed()
+            .0
+            .into_iter()
+            .map(|(_, verb, args)| serde_json::json!({ "verb": verb, "args": args }))
+            .collect()
+    }
+
+    async fn dead_letter_rows(&self) -> Vec<(String, String, String, i32)> {
+        if let Some(store) = self.backend().as_any().downcast_ref::<PostgresStore>() {
+            let guard = store.lineage().lock().await;
+            let sql = "SELECT policy, target_domain, target_verb, attempts \
+                       FROM hecks_cross_domain_dead_letters ORDER BY policy";
+            let rows = guard.query(sql, &[]).await.unwrap();
+            return rows
+                .iter()
+                .map(|r| (r.get(0), r.get(1), r.get(2), r.get(3)))
+                .collect();
+        }
+        let memory = self.backend().as_any().downcast_ref::<MemoryStore>();
+        let mut rows: Vec<_> = memory
+            .expect("a Postgres or Memory store")
+            .dead_letters()
+            .into_iter()
+            .map(|d| (d.policy, d.target_domain, d.target_verb, d.attempts))
+            .collect();
+        rows.sort();
+        rows
+    }
+
+    async fn lock(&self) -> tokio::sync::MutexGuard<'_, Client> {
+        match self.lineage() {
+            Some(lineage) => lineage.lock().await,
+            None => panic!("a memory store has no connection to lock"),
+        }
+    }
+}