hecks 5.0.0 → 6.0.0

data/rust/host/src/server.rs

@@ -1,12 +1,15 @@
 //! HTTP server for AWS Fargate (`HECKS_SERVE_MODE=1`): the axum fallback route below shares
 //! `dispatch_body` with the Lambda custom-runtime path, so neither repeats the other's logic.
 
+use crate::arg_limits;
 use crate::dispatch;
 use crate::host::Host;
-use crate::journal::LineageConfig;
 use crate::lambda_client::LambdaInvoker;
 use crate::log;
 use crate::rate_limit::{self, RateLimits, Verdict};
+use crate::store::journal_sql::LineageConfig;
+use crate::store::Store;
+use crate::validation;
 use crate::wasm_runner;
 use crate::web;
 use axum::body::Bytes;
@@ -20,8 +23,6 @@ use serde_json::Value;
 use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::sync::Arc;
-use tokio::sync::Mutex;
-use tokio_postgres::Client;
 
 /// The per-request dispatch decision, shared between the Lambda custom-runtime path and the
 /// axum fallback route below. `body` is already-parsed JSON in one of its recognized shapes.
@@ -32,10 +33,10 @@ use tokio_postgres::Client;
 /// same either way; without `bare` the answer is the stored shape.
 pub async fn dispatch_body(body: Value, host: Host<'_>) -> Result<Value, Error> {
     if let Some(response) = web::render(&body, host).await {
-        return Ok(response);
+        return Ok(conceal_internal_error(response));
     }
     let answer = kernel_answer(&body, host).await?;
-    if !wants_bare(&body) {
+    if !wants_bare(&body) || wants_preview(&body) {
         return Ok(answer);
     }
     Ok(dispatch::present(host.wasm_path, answer)
@@ -43,14 +44,83 @@ pub async fn dispatch_body(body: Value, host: Host<'_>) -> Result<Value, Error>
         .map_err(|e| format!("{e:#}"))?)
 }
 
+/// A route's `500` keeps its detail in the log and tells the caller only a request id to quote:
+/// an error's text can name tables, queries and paths that are nobody else's business.
+fn conceal_internal_error(mut response: Value) -> Value {
+    if response.get("statusCode").and_then(Value::as_u64) != Some(500) {
+        return response;
+    }
+    let request_id = new_request_id();
+    log::error(
+        "internal_error",
+        serde_json::json!({ "request_id": request_id, "detail": response.get("body") }),
+    );
+    response["headers"]["content-type"] = Value::from("application/json");
+    response["body"] = Value::from(internal_error_body(&request_id).to_string());
+    response["isBase64Encoded"] = Value::Bool(false);
+    response
+}
+
+fn internal_error_body(request_id: &str) -> Value {
+    serde_json::json!({ "error": "internal error", "request_id": request_id })
+}
+
+fn new_request_id() -> String {
+    uuid::Uuid::new_v4().simple().to_string()
+}
+
+/// The `429` a Lambda Function-URL event over a public route's rate limit gets, judged as `admit`
+/// judges a request in serve mode, with `requestContext.http.sourceIp` as the peer. An event with
+/// no route (an internal verb or read invocation) is never limited.
+pub fn lambda_rate_limited(limits: &RateLimits, event: &Value) -> Option<Value> {
+    let (method, path) = envelope_route(event)?;
+    let peer = event
+        .pointer("/requestContext/http/sourceIp")
+        .and_then(Value::as_str)
+        .and_then(|ip| ip.parse().ok());
+    match limits.check(method, path, &event_headers(event), peer) {
+        Verdict::Limited { retry_after_secs } => {
+            Some(rate_limit::too_many_requests_event(retry_after_secs))
+        }
+        Verdict::Allowed => None,
+    }
+}
+
+// A Function-URL event's `headers` object as a `HeaderMap`; a name or value that is not a valid
+// header is left out.
+fn event_headers(event: &Value) -> HeaderMap {
+    let mut headers = HeaderMap::new();
+    for (name, value) in event
+        .get("headers")
+        .and_then(Value::as_object)
+        .into_iter()
+        .flatten()
+    {
+        let parsed = (
+            HeaderName::from_bytes(name.as_bytes()),
+            value.as_str().map(HeaderValue::from_str),
+        );
+        if let (Ok(name), Some(Ok(value))) = parsed {
+            headers.append(name, value);
+        }
+    }
+    headers
+}
+
 fn wants_bare(body: &Value) -> bool {
     body.get("bare").and_then(Value::as_bool) == Some(true)
 }
 
+// `"dry_run": true` beside a command asks whether it would be accepted, and keeps nothing. Its
+// answer is already the client envelope, so `bare` has nothing to unwrap in it.
+fn wants_preview(body: &Value) -> bool {
+    body.get("dry_run").and_then(Value::as_bool) == Some(true)
+}
+
 // The kernel's outcome for a read, query or command body, in the stored shape.
 async fn kernel_answer(body: &Value, host: Host<'_>) -> Result<Value, Error> {
     let Host {
-        client,
+        store,
         wasm_path,
         config,
         ..
@@ -59,7 +129,7 @@ async fn kernel_answer(body: &Value, host: Host<'_>) -> Result<Value, Error> {
     if body.get("read").and_then(serde_json::Value::as_bool) == Some(true) {
         // `{e:#}`: anyhow's alternate Display walks `source()` to the real message,
         // where a bare `tokio_postgres` error's Display is just "db error".
-        let result = dispatch::read(client, wasm_path)
+        let result = dispatch::read(store, wasm_path)
             .await
             .map_err(|e| format!("{e:#}"))?;
         return Ok(result);
@@ -71,7 +141,7 @@ async fn kernel_answer(body: &Value, host: Host<'_>) -> Result<Value, Error> {
             body,
             crate::ir::ir(),
             &config.domain,
-            client,
+            store,
             wasm_path,
         )
         .await?);
@@ -97,7 +167,23 @@ async fn kernel_answer(body: &Value, host: Host<'_>) -> Result<Value, Error> {
         actor_id: actor_id.as_deref(),
     };
 
-    let outcome = if let Some(to) = body.get("to").cloned() {
+    let invocation = invocation_of(body)?;
+    if wants_preview(body) {
+        let outcome = dispatch::preview_as(host, &verb, invocation, caller)
+            .await
+            .map_err(|e| format!("{e:#}"))?;
+        return Ok(validation::answer(&outcome.result));
+    }
+    let outcome = dispatch::handle_as(host, &verb, invocation, caller)
+        .await
+        .map_err(|e| format!("{e:#}"))?;
+    Ok(outcome.result)
+}
+
+// The command's arguments in the invocation shape the kernel reads, from whichever of the body's
+// three spellings it used: `to` with `with`, `with` alone, or the legacy `args`.
+fn invocation_of(body: &Value) -> Result<Value, Error> {
+    if let Some(to) = body.get("to").cloned() {
         if body.get("args").is_some() {
             return Err("cannot combine to/with with legacy args".into());
         }
@@ -105,33 +191,25 @@ async fn kernel_answer(body: &Value, host: Host<'_>) -> Result<Value, Error> {
             .get("with")
             .cloned()
             .unwrap_or_else(|| serde_json::json!({}));
-        dispatch::handle_routed_as(host, &verb, to, facts, caller)
-            .await
-            .map_err(|e| format!("{e:#}"))?
-    } else if let Some(facts) = body.get("with").cloned() {
+        return dispatch::routed_invocation(to, facts).map_err(|e| format!("{e:#}").into());
+    }
+    if let Some(facts) = body.get("with").cloned() {
         if body.get("args").is_some() {
             return Err("cannot combine \"with\" with legacy args".into());
         }
-        dispatch::handle_facts_as(host, &verb, facts, caller)
-            .await
-            .map_err(|e| format!("{e:#}"))?
-    } else {
-        let args = body
-            .get("args")
-            .cloned()
-            .unwrap_or_else(|| serde_json::json!({}));
-        dispatch::handle_as(host, &verb, args, caller)
-            .await
-            .map_err(|e| format!("{e:#}"))?
-    };
-    Ok(outcome.result)
+        return dispatch::facts_invocation(facts).map_err(|e| format!("{e:#}").into());
+    }
+    Ok(body
+        .get("args")
+        .cloned()
+        .unwrap_or_else(|| serde_json::json!({})))
 }
 
 /// Everything `dispatch_body` needs, `Arc`-shared for axum's per-request handler cloning.
-/// `client`'s mutex serializes every write onto the one Postgres connection this process holds.
+/// The store serializes every write onto the one journal this process holds.
 #[derive(Clone)]
 pub struct ServerState {
-    pub client: Arc<Mutex<Client>>,
+    pub store: Arc<Store>,
     pub wasm_path: Arc<PathBuf>,
     pub lineage_config: Arc<LineageConfig>,
     pub invoker: Arc<dyn LambdaInvoker>,
@@ -142,7 +220,7 @@ impl ServerState {
     /// The shared pieces a dispatch needs, borrowed for one call.
     pub fn host(&self) -> Host<'_> {
         Host::shared(
-            &self.client,
+            &self.store,
             &self.wasm_path,
             &self.lineage_config,
             &self.invoker,
@@ -274,13 +352,14 @@ async fn route_request(
     match dispatch_body(envelope, state.host()).await {
         Ok(value) => value_to_response(value),
         Err(e) => {
+            let request_id = new_request_id();
             log::error(
                 "dispatch_failed",
-                serde_json::json!({ "error": format!("{e}") }),
+                serde_json::json!({ "error": format!("{e}"), "request_id": request_id }),
             );
             (
                 StatusCode::INTERNAL_SERVER_ERROR,
-                axum::Json(serde_json::json!({ "error": format!("{e}") })),
+                axum::Json(internal_error_body(&request_id)),
             )
                 .into_response()
         }
@@ -326,6 +405,18 @@ fn admit(
     {
         parsed
     } else {
+        if let Some(longest) = arg_limits::oversized(&parsed) {
+            return Err(Box::new(
+                (
+                    StatusCode::PAYLOAD_TOO_LARGE,
+                    format!(
+                        "a string in the body is {longest} bytes; at most {} are accepted",
+                        arg_limits::MAX_STRING_ARG_BYTES
+                    ),
+                )
+                    .into_response(),
+            ));
+        }
         synthesize_function_url_envelope(method, uri, headers, body)
     };
 
@@ -471,6 +562,8 @@ fn value_to_response(value: Value) -> Response {
 #[cfg(test)]
 mod tests {
     use super::*;
+    #[allow(unused_imports)]
+    use crate::store::StoreFixtures;
 
     #[test]
     fn a_function_url_shaped_body_is_recognized_as_internal_dispatch() {
@@ -559,6 +652,28 @@ mod tests {
         .unwrap_or_else(|_| panic!("{from} POST {path} was refused before dispatch"))
     }
 
+    #[test]
+    fn a_public_body_with_a_string_past_the_cap_is_refused_and_a_peers_is_not() {
+        let big = format!(
+            r#"{{"verb":"Post.Publish","with":{{"body":"{}"}}}}"#,
+            "x".repeat(arg_limits::MAX_STRING_ARG_BYTES + 1)
+        );
+        let body = Bytes::from(big.clone());
+        let refused = admit(
+            &limits_from(&[]),
+            peer("203.0.113.7:5000"),
+            &Method::POST,
+            &"/api/posts".parse().unwrap(),
+            &HeaderMap::new(),
+            &body,
+        )
+        .expect_err("a public body past the cap must be refused");
+        assert_eq!(refused.status(), StatusCode::PAYLOAD_TOO_LARGE);
+
+        let envelope = admitted("127.0.0.1:5000", "/anything", &big);
+        assert_eq!(envelope["verb"], "Post.Publish");
+    }
+
     // The gap this closes: a public POST to any path the load balancer forwards carried
     // `{"verb": ..., "role": ...}` or `{"read": true}` straight to the kernel, with the
     // caller's own word for its role and no session check.
@@ -839,6 +954,75 @@ mod tests {
         );
     }
 
+    fn function_url_event(path: &str, source_ip: &str) -> Value {
+        serde_json::json!({
+            "requestContext": { "http": { "method": "POST", "sourceIp": source_ip } },
+            "rawPath": path,
+            "headers": { "content-type": "application/json" },
+            "body": "{}",
+        })
+    }
+
+    #[test]
+    fn a_lambda_burst_from_one_source_ip_is_refused_with_a_429_event() {
+        let limits = limits_from(&[("HECKS_RATE_LIMIT_SUBSCRIBE", "2")]);
+        let event = function_url_event(SUBSCRIBE, "203.0.113.7");
+        assert_eq!(lambda_rate_limited(&limits, &event), None);
+        assert_eq!(lambda_rate_limited(&limits, &event), None);
+        let refused = lambda_rate_limited(&limits, &event).expect("the third is over the limit");
+        assert_eq!(refused["statusCode"], 429);
+        assert!(
+            refused["headers"]["retry-after"].as_str().is_some(),
+            "{refused}"
+        );
+        let body: Value = serde_json::from_str(refused["body"].as_str().unwrap()).unwrap();
+        assert_eq!(body["error"], "too many requests, please try again later");
+    }
+
+    #[test]
+    fn lambda_limits_count_each_source_ip_apart_and_never_an_internal_invocation() {
+        let limits = limits_from(&[("HECKS_RATE_LIMIT_SUBSCRIBE", "1")]);
+        assert_eq!(
+            lambda_rate_limited(&limits, &function_url_event(SUBSCRIBE, "203.0.113.7")),
+            None
+        );
+        assert_eq!(
+            lambda_rate_limited(&limits, &function_url_event(SUBSCRIBE, "203.0.113.8")),
+            None
+        );
+        let read = serde_json::json!({ "read": true });
+        for _ in 0..3 {
+            assert_eq!(lambda_rate_limited(&limits, &read), None);
+        }
+    }
+
+    #[test]
+    fn a_routes_500_reaches_the_caller_as_a_request_id_not_its_detail() {
+        let failed = serde_json::json!({
+            "statusCode": 500,
+            "headers": { "content-type": "text/plain" },
+            "body": "relation \"secret_table\" does not exist",
+            "isBase64Encoded": false,
+        });
+        let concealed = conceal_internal_error(failed);
+        let body: Value = serde_json::from_str(concealed["body"].as_str().unwrap()).unwrap();
+        assert_eq!(body["error"], "internal error");
+        assert_eq!(body["request_id"].as_str().map(str::len), Some(32));
+        assert!(
+            !concealed.to_string().contains("secret_table"),
+            "{concealed}"
+        );
+        assert_eq!(concealed["headers"]["content-type"], "application/json");
+    }
+
+    #[test]
+    fn a_response_that_is_not_a_500_is_passed_through() {
+        for status in [200, 404, 422, 502] {
+            let response = serde_json::json!({ "statusCode": status, "body": "kept" });
+            assert_eq!(conceal_internal_error(response.clone()), response);
+        }
+    }
+
     #[tokio::test]
     async fn registrations_are_limited_apart_from_subscribes() {
         let addr = gated_server(limits_from(&[
@@ -977,8 +1161,8 @@ mod tests {
     // The real service on an ephemeral port, over a scratch Postgres and a cold copy of the
     // checkout fixture's wasm, so the first request is a real compile.
     async fn fixture_host(database: &str, copy_name: &str) -> (SocketAddr, Arc<PathBuf>) {
-        let client = crate::dispatch::tests::scratch_db(database).await;
-        crate::dispatch::tests::provision_lineage(
+        let client = crate::dispatch_tests::tests::scratch_db(database).await;
+        crate::dispatch_tests::tests::provision_lineage(
             &*client.lock().await,
             "CheckoutFixture",
             1,
@@ -987,12 +1171,13 @@ mod tests {
         .await;
         let wasm_path = Arc::new(crate::wasm_runner::tests::cold_copy_of_fixture(copy_name));
         let state = ServerState {
-            client: Arc::new(client),
+            store: Arc::new(client),
             wasm_path: Arc::clone(&wasm_path),
             lineage_config: Arc::new(LineageConfig {
                 domain: "CheckoutFixture".to_string(),
                 era: Some(1),
                 mirrored: None,
+                seal_every: None,
             }),
             invoker: Arc::new(crate::lambda_client::NeverInvoker),
             limits: limits_from(&[]),
@@ -1065,18 +1250,12 @@ mod tests {
     // `price` one `cents` field, so each answers as a bare scalar when asked to.
     #[tokio::test]
     async fn a_bare_body_answers_single_field_value_objects_bare_and_the_default_stays_wrapped() {
-        let client = crate::dispatch::tests::scratch_db("rust_host_serve_bare").await;
-        crate::dispatch::tests::provision_lineage(
-            &*client.lock().await,
-            "CheckoutFixture",
-            1,
-            &["Event"],
-        )
-        .await;
+        let client = crate::dispatch_tests::tests::memory_store();
         let config = LineageConfig {
             domain: "CheckoutFixture".to_string(),
-            era: Some(1),
+            era: None,
             mirrored: None,
+            seal_every: None,
         };
         let wasm = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../dist/checkout_fixture.wasm");
         let host = Host::new(&client, &wasm, &config, &crate::lambda_client::NeverInvoker);

data/rust/host/src/{journal.rs → store/journal_sql.rs}

@@ -1,10 +1,12 @@
 //! Flat, domain-agnostic journal of every command this domain has ever
 //! accepted, replayed in full on each invocation rather than filtered (ADR 0018).
 
+pub use crate::journal::*;
 use anyhow::Context;
 use sha2::{Digest, Sha256};
 use tokio_postgres::{Client, GenericClient};
 
+/// Creates the journal, snapshot, saga and dead-letter tables a domain's journal needs, if absent.
 pub async fn ensure_schema(client: &Client) -> anyhow::Result<()> {
     client
         .batch_execute(
@@ -150,16 +152,6 @@ pub async fn load_steps_after<C: GenericClient>(
         .collect())
 }
 
-/// The kernel's cached "instances" output as of `ordinal`, used to skip a
-/// full replay from empty state.
-pub struct Snapshot {
-    pub ordinal: i64,
-    pub seed: serde_json::Value,
-    /// One-time latch: false only for a snapshot written before saga
-    /// durability shipped, never a live signal about current saga state.
-    pub sagas_backfilled: bool,
-}
-
 /// The domain's cached snapshot, if one has ever been written. `None`
 /// before the first accepted command, when there's nothing to seed from yet.
 pub async fn load_snapshot<C: GenericClient>(client: &C) -> anyhow::Result<Option<Snapshot>> {
@@ -209,21 +201,17 @@ pub async fn save_snapshot<C: GenericClient>(
     Ok(())
 }
 
-/// One live saga/process-manager instance, as stored in `hecks_lambda_sagas`.
-pub struct SagaRow {
-    pub process_manager: String,
-    pub correlation: String,
-    pub state: String,
-    pub memory: serde_json::Value,
-    pub completed_compensations: serde_json::Value,
-}
-
 /// Every live saga instance for this domain, read once per invocation to
 /// seed the kernel's in-memory `sagas` map.
+///
+/// Ordered by key in byte order (`COLLATE "C"`), which is the order the memory store keeps its
+/// sagas in: the kernel echoes the sagas back in the order it was given them, so a table scan in
+/// heap order would make one command answer differently from one store to the next.
 pub async fn load_sagas<C: GenericClient>(client: &C) -> anyhow::Result<Vec<SagaRow>> {
     let rows = client
         .query(
-            "SELECT process_manager, correlation, state, memory, completed_compensations FROM hecks_lambda_sagas",
+            "SELECT process_manager, correlation, state, memory, completed_compensations FROM hecks_lambda_sagas \
+             ORDER BY process_manager COLLATE \"C\", correlation COLLATE \"C\"",
             &[],
         )
         .await?;
@@ -276,85 +264,12 @@ pub async fn delete_saga<C: GenericClient>(
     Ok(())
 }
 
-/// Which domain journal this binary writes as, and which era. Read from
-/// `HECKS_DOMAIN`/main.rs's boot-time mint decision, never computed here.
-pub struct LineageConfig {
-    pub domain: String,
-    /// `None` when lineage isn't provisioned for this domain (ADR 0034).
-    pub era: Option<i32>,
-    /// Aggregates mirrored into era-shaped head snapshots; `None` means all.
-    pub mirrored: Option<std::collections::BTreeSet<String>>,
-}
-
-impl LineageConfig {
-    /// Whether `qualified_aggregate` is mirrored into the era-shaped head
-    /// snapshots.
-    pub fn mirrors(&self, qualified_aggregate: &str) -> bool {
-        self.mirrored
-            .as_ref()
-            .is_none_or(|capable| capable.contains(qualified_aggregate))
-    }
-}
-
-// `Naming.snake` is the one shared port (rust/codegen/src/hecks_naming.rs); the host reaches it
-// through here so both runtimes derive the same storage names.
-pub use crate::hecks_naming::snake;
-
-/// `aggregate.storage_name` ported from ir/aggregate.rb — demodulizes a
-/// qualified name like `"Banking::Customer"` to `"Customer"` before snaking it.
-fn storage_name(qualified_aggregate: &str) -> String {
-    let short = qualified_aggregate
-        .rsplit("::")
-        .next()
-        .unwrap_or(qualified_aggregate);
-    snake(short)
-}
-
 /// Quotes a Postgres identifier `PG::Connection.quote_ident`-style —
 /// needed since table names can't be bound as query parameters.
 pub fn quote_ident(name: &str) -> String {
     format!("\"{}\"", name.replace('"', "\"\""))
 }
 
-fn journal_table(domain: &str) -> String {
-    format!("hecks_journal_{}", snake(domain))
-}
-
-/// Postgres's NAMEDATALEN-1 limit — an identifier over this length is
-/// silently truncated, never refused, so two overlong names sharing a
-/// prefix would otherwise collide.
-const POSTGRES_IDENTIFIER_LIMIT: usize = 63;
-
-/// Same algorithm as `Lineage#qualified_name` (lineage.rb): 63-byte limit,
-/// SHA256 suffix over the full name, a cut that never splits a character, domain-qualified so two domains' storage names can't
-/// collide (ADR 0059).
-pub(crate) fn qualified_name(domain: &str, suffix: &str) -> String {
-    let full = format!("{}_{}", snake(domain), suffix);
-    if full.len() <= POSTGRES_IDENTIFIER_LIMIT {
-        return full;
-    }
-
-    let digest = format!("{:x}", Sha256::digest(full.as_bytes()));
-    let digest = &digest[0..8];
-    let mut keep = POSTGRES_IDENTIFIER_LIMIT - digest.len() - 1;
-    // Cut at the longest prefix of whole characters that fits, as the Ruby side does.
-    while !full.is_char_boundary(keep) {
-        keep -= 1;
-    }
-    format!("{}_{}", &full[0..keep], digest)
-}
-
-fn head_snapshot_table(domain: &str, qualified_aggregate: &str, era: i32) -> String {
-    qualified_name(
-        domain,
-        &format!(
-            "{}_head_snapshot_{}",
-            storage_name(qualified_aggregate),
-            era
-        ),
-    )
-}
-
 /// The highest era ordinal on file for `domain`, not a bare existence
 /// check — `hecks_eras` is append-only, so a stale checkout still has a row.
 pub async fn current_era(client: &Client, domain: &str) -> anyhow::Result<Option<i32>> {
@@ -367,15 +282,6 @@ pub async fn current_era(client: &Client, domain: &str) -> anyhow::Result<Option
     Ok(rows.first().map(|row| row.get(0)))
 }
 
-/// One `MutationRecord` from the kernel's JSON output, parsed generically
-/// since rust/host never links the kernel crate directly (ADR 0012).
-pub struct Mutation<'a> {
-    pub aggregate: &'a str,
-    pub id: &'a str,
-    pub operation: &'a str,
-    pub state: &'a serde_json::Value,
-}
-
 /// Mirrors `PostgresEra#append`'s two-step write: journal insert, then a
 /// head-snapshot upsert that never regresses on a stale/reordered write.
 pub async fn append_lineage_mutation<C: GenericClient>(
@@ -455,14 +361,8 @@ pub async fn append_lineage_mutation<C: GenericClient>(
     Ok(())
 }
 
-// Not wired into dispatch::handle's replay seed — overlaying translated
-// head state onto raw replayed steps would cause false "AlreadyExists" refusals.
-pub(crate) fn head_view(domain: &str, storage_name: &str) -> String {
-    qualified_name(domain, &format!("{storage_name}_head"))
-}
-
-/// Every row for one lineage-capable aggregate, already translated to its
-/// current shape (reads the same head view Ruby's `Adapters::Postgres` does).
+/// Every row for one lineage-capable aggregate in id byte order (the order Memory keeps),
+/// already translated to its current shape (the head view Ruby's `Adapters::Postgres` reads).
 pub async fn read_lineage_head_all<C: GenericClient>(
     client: &C,
     domain: &str,
@@ -471,7 +371,7 @@ pub async fn read_lineage_head_all<C: GenericClient>(
     let rows = client
         .query(
             &format!(
-                "SELECT id, state FROM {}",
+                "SELECT id, state FROM {} ORDER BY id COLLATE \"C\"",
                 quote_ident(&head_view(domain, storage_name))
             ),
             &[],
@@ -502,22 +402,12 @@ pub async fn read_lineage_head_by_id<C: GenericClient>(
     Ok(row.map(|r| r.get(0)))
 }
 
-/// One row of `hecks_eras`, mirroring era_store.rb's `eras` shape.
-#[derive(Debug, Clone)]
-pub struct HeldEra {
-    pub ordinal: i32,
-    pub hash: Option<String>,
-    pub label: Option<String>,
-    pub held_text: String,
-    pub watermark: Option<i64>,
-}
-
 /// Every held era for `domain`, each verified against its own digest — an
 /// edited `held_text` refuses loudly instead of silently reporting no drift.
 pub async fn held_eras<C: GenericClient>(client: &C, domain: &str) -> anyhow::Result<Vec<HeldEra>> {
     let rows = client
         .query(
-            "SELECT ordinal, hash, label, held_text, watermark, held_digest FROM hecks_eras \
+            "SELECT ordinal, label, held_text, watermark, held_digest FROM hecks_eras \
              WHERE domain = $1 ORDER BY ordinal",
             &[&domain],
         )
@@ -542,9 +432,7 @@ pub async fn held_eras<C: GenericClient>(client: &C, domain: &str) -> anyhow::Re
 
             Ok(HeldEra {
                 ordinal,
-                hash: row.get("hash"),
                 label: row.get("label"),
-                held_text,
                 watermark: row.get("watermark"),
             })
         })
@@ -611,6 +499,7 @@ mod lineage_tests {
             domain: "Ledger".to_string(),
             era: Some(2),
             mirrored: None,
+            seal_every: None,
         };
         let accepted = append_lineage_mutation(
             &client,
@@ -633,6 +522,7 @@ mod lineage_tests {
             domain: "Ledger".to_string(),
             era: Some(1),
             mirrored: None,
+            seal_every: None,
         };
         let refused = append_lineage_mutation(
             &client,
@@ -808,6 +698,7 @@ mod lineage_tests {
             domain: "Fixtures".to_string(),
             era: Some(1),
             mirrored: None,
+            seal_every: None,
         };
         client
             .batch_execute("CREATE TABLE IF NOT EXISTS hecks_journal_fixtures (ordinal bigserial PRIMARY KEY, era int NOT NULL, aggregate text NOT NULL, aggregate_id text NOT NULL, operation text NOT NULL, state jsonb)")
@@ -886,7 +777,7 @@ mod lineage_tests {
             &client,
             "ReviewOnFreeze",
             "Compliance",
-            "Compliance::AccountFreezeReview.Open",
+            "Compliance::Review.Open",
             &serde_json::json!({ "number": { "value": "acct-1" } }),
             "ResourceNotFoundException: function not found",
             3,
@@ -911,7 +802,7 @@ mod lineage_tests {
         let attempts: i32 = row.get(5);
         assert_eq!(policy, "ReviewOnFreeze");
         assert_eq!(target_domain, "Compliance");
-        assert_eq!(target_verb, "Compliance::AccountFreezeReview.Open");
+        assert_eq!(target_verb, "Compliance::Review.Open");
         assert_eq!(
             payload,
             serde_json::json!({ "number": { "value": "acct-1" } })
@@ -929,7 +820,10 @@ mod naming_vector_tests {
     // spec/fixtures/naming_vectors.json is asserted by spec/naming_vectors_spec.rb against
     // Naming.snake and Lineage#qualified_name, so a drift in either runtime fails a test.
     fn vectors() -> Value {
-        serde_json::from_str(include_str!("../../../spec/fixtures/naming_vectors.json")).unwrap()
+        serde_json::from_str(include_str!(
+            "../../../../spec/fixtures/naming_vectors.json"
+        ))
+        .unwrap()
     }
 
     fn table(section: &str) -> Vec<Value> {