hecks 5.0.0 → 6.0.0

data/rust/host/fuzz/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "rust_host_fuzz"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+# Own workspace root, so neither `rust/host` nor `rust` adopts this crate and its fuzzing-only
+# dependencies; `cargo build` and `cargo test` in the host never compile it.
+[workspace]
+
+[lib]
+name = "rust_host_fuzz"
+path = "src/lib.rs"
+
+[package.metadata]
+cargo-fuzz = true
+
+# Every version below is the host's own (rust/host/Cargo.toml), so the lockfile resolves one copy of
+# each shared crate. Only `libfuzzer-sys`, `arbitrary`, `dashmap` and `ahash` are new, and they live
+# only here.
+[dependencies]
+rust_host = { path = ".." }
+libfuzzer-sys = "0.4"
+# `derive` is what lets a typed command script come out of libFuzzer's bytes.
+arbitrary = { version = "1", features = ["derive"] }
+# Sharded, so the footprint memo takes no global lock under the sanitizer.
+dashmap = "6"
+ahash = "0.8"
+tokio = { version = "1", features = ["macros", "net", "rt", "sync", "time"] }
+tokio-postgres = { version = "0.7", features = ["with-serde_json-1"] }
+# Same features as the host: `arbitrary_precision` keeps a number as the text the kernel wrote, and
+# `preserve_order` keeps object key order, so a value read back compares as the host sees it.
+serde_json = { version = "1", features = ["preserve_order", "arbitrary_precision"] }
+async-trait = "0.1.92"
+anyhow = "1"
+
+[[bin]]
+name = "postgres_equivalence"
+path = "fuzz_targets/postgres_equivalence.rs"
+test = false
+doc = false
+bench = false

data/rust/host/fuzz/README.md

@@ -0,0 +1,55 @@
+# Differential fuzzing of the host's journal stores
+
+Opt-in. The host stays free of fuzzing dependencies: `libfuzzer-sys`, `arbitrary`, `dashmap` and
+`ahash` live only in this crate, which is its own workspace and is never built by `cargo build` or
+`cargo test` in `rust/host`.
+
+The in-memory store (`MemoryStore`, the oracle) runs every script. The Postgres store is compared
+against it only for scripts whose structure has not been seen to agree. A script is a run of Banking
+commands that `arbitrary` decodes from libFuzzer's bytes (`src/ops.rs`).
+
+## Run it
+
+Needs the compiled kernel and, for the comparison, a scratch Postgres database.
+
+```sh
+exe/hecks build.build_wasm domain=examples/banking profile=dev --wait   # rust/dist/banking.wasm
+createdb hecks_fuzz
+
+rustup toolchain install nightly --profile minimal
+cargo install cargo-fuzz --locked
+cd rust/host
+HECKS_FUZZ_PG="host=localhost user=$USER dbname=hecks_fuzz" \
+  cargo +nightly fuzz run postgres_equivalence --sanitizer none -- \
+  -max_total_time=300 -timeout=30 -max_len=1024 > /dev/null
+```
+
+- `HECKS_FUZZ_PG` unset: only the in-memory store runs. That still checks the commit clause (an
+  accepted command advances the snapshot by one row, any other leaves the store as it was) and, with
+  the default sanitizer, memory safety.
+- `HECKS_FUZZ_WASM` names another kernel; the default is `rust/dist/banking.wasm`.
+- stdout goes to `/dev/null`: the host logs one JSON line a command. libFuzzer reports on stderr.
+- A crash lands in `fuzz/artifacts/`; replay it with
+  `cargo +nightly fuzz run postgres_equivalence fuzz/artifacts/postgres_equivalence/<file>`.
+
+Without nightly, `cargo test --release --test smoke` drives the same check from a seeded byte stream
+(`HECKS_FUZZ_SMOKE=3000` sets how many scripts).
+
+## What it compares
+
+Each step's outcome (accepted, refused with the kernel's answer, or failed), and afterwards the
+snapshot's ordinal and seed and the live sagas, read through `Store::begin` and `load`.
+
+- Event timestamps (`occurred_at`) are dropped: the stores are played moments apart.
+- The in-memory store is played twice and must answer the same both times. Without that, a
+  difference between the stores could be the kernel disagreeing with itself.
+- A difference is played again on both stores before it is reported.
+
+## What it does not cover
+
+- The journal rows themselves: no public read returns them, so a `jsonb` round trip of journaled
+  arguments is seen only through the snapshot.
+- Lineage (`era: None`): the era-shaped mirror tables need `provision_lineage`, which is test-only.
+- Crash durability: the Postgres session runs `synchronous_commit = off` on unlogged tables.
+- Scripts that differ only in the value inside one class (a length bucket, a number's magnitude)
+  share a footprint, so the memo re-verifies one in 1, 2, 4, 8... repeats instead of all of them.

data/rust/host/fuzz/fuzz_targets/postgres_equivalence.rs

@@ -0,0 +1,24 @@
+//! Coverage-guided differential target: the host's in-memory journal store against its Postgres one.
+//!
+//! libFuzzer decodes its bytes into a typed script of Banking commands (`arbitrary`), the script
+//! runs against the in-memory store, and Postgres is asked only about scripts whose structure is
+//! new. See `rust_host_fuzz::equivalence` for the rules and `rust_host_fuzz::footprint` for the memo.
+//!
+//! Needs the compiled kernel at `rust/dist/banking.wasm` (or `HECKS_FUZZ_WASM`). With
+//! `HECKS_FUZZ_PG` unset only the in-memory store runs, which still checks the commit clause.
+//!
+//! ```sh
+//! cd rust/host
+//! createdb hecks_fuzz
+//! HECKS_FUZZ_PG="host=localhost user=$USER dbname=hecks_fuzz" \
+//!   cargo +nightly fuzz run postgres_equivalence -- -max_len=1024 -timeout=30
+//! ```
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use rust_host_fuzz::equivalence;
+use rust_host_fuzz::ops::Script;
+
+fuzz_target!(init: equivalence::warm(), |script: Script| {
+    equivalence::check(&script);
+});

data/rust/host/fuzz/src/backend.rs

@@ -0,0 +1,273 @@
+//! The one seam both journal stores sit behind, and the two stores.
+//!
+//! [`MemoryBackend`] is the host's own `MemoryStore`, the in-memory adapter that already passes the
+//! same conformance cases as Postgres, so it is the oracle and not a second mock to keep in step.
+//! [`PostgresBackend`] keeps one connection for the life of the process and resets its tables
+//! between scripts instead of provisioning a database per case.
+//!
+//! The wasm kernel is compiled once per process by `rust_host::wasm_runner`, so the only per-script
+//! work left in either backend is the kernel's replay and the store's writes.
+
+use anyhow::Context;
+use rust_host::dispatch::{self, Caller};
+use rust_host::host::Host;
+use rust_host::journal::LineageConfig;
+use rust_host::store::journal_sql;
+use rust_host::lambda_client::{InvokeOutcome, LambdaInvoker};
+use rust_host::store::{MemoryStore, Store};
+use serde_json::Value;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex, PoisonError};
+use tokio_postgres::NoTls;
+
+/// The domain the compiled kernel under test belongs to.
+pub const DOMAIN: &str = "Banking";
+
+/// The tables a script writes, truncated together between scripts.
+const TABLES: [&str; 4] = [
+    "hecks_lambda_journal",
+    "hecks_lambda_snapshot",
+    "hecks_lambda_sagas",
+    "hecks_cross_domain_dead_letters",
+];
+
+/// What one dispatch answered, reduced to what two stores must agree on.
+#[derive(Debug, Clone, PartialEq)]
+pub enum Step {
+    Accepted(Value),
+    Refused(Value),
+    /// The dispatch failed outright. Only the fact is compared: a backend's error text is its own.
+    Failed,
+}
+
+impl Step {
+    #[must_use]
+    pub const fn accepted(&self) -> bool {
+        matches!(self, Self::Accepted(_))
+    }
+}
+
+/// What a second reader would see in the store after a script.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Observed {
+    /// The ordinal the snapshot is paired with, which counts the accepted commands.
+    pub snapshot_ordinal: Option<i64>,
+    pub seed: Value,
+    /// `(process_manager, correlation, state)`, sorted.
+    pub sagas: Vec<(String, String, String)>,
+}
+
+/// A journal store the host can dispatch against.
+#[allow(async_fn_in_trait)] // the harness drives one backend at a time on one thread
+pub trait Backend {
+    /// Empties the store so the next script starts from nothing.
+    async fn reset(&self) -> anyhow::Result<()>;
+    async fn dispatch(&self, verb: &str, args: Value) -> Step;
+    async fn observe(&self) -> anyhow::Result<Observed>;
+}
+
+/// The path of the compiled kernel: `HECKS_FUZZ_WASM`, else `rust/dist/banking.wasm`.
+pub fn wasm_path() -> PathBuf {
+    std::env::var_os("HECKS_FUZZ_WASM").map_or_else(
+        || PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../dist/banking.wasm"),
+        PathBuf::from,
+    )
+}
+
+/// Banking's `ReviewOnFreeze` policy delivers to the Compliance domain's function. Nothing is
+/// deployed here, so every delivery is answered the way the host's own tests answer it: delivered,
+/// nothing refused. Both stores get the same answer, so a delivery never explains a difference.
+struct AcceptDelivery;
+
+#[async_trait::async_trait]
+impl LambdaInvoker for AcceptDelivery {
+    async fn invoke(&self, _function_name: &str, _payload: &str) -> anyhow::Result<InvokeOutcome> {
+        Ok(InvokeOutcome {
+            body: serde_json::json!({ "refusals": [] }),
+            function_error: false,
+        })
+    }
+}
+
+/// Every event carries the wall-clock second it ran (`occurred_at`), and the two stores are played
+/// moments apart, so the stamp is dropped before outcomes are compared.
+fn timeless(mut value: Value) -> Value {
+    match &mut value {
+        Value::Object(map) => {
+            map.remove("occurred_at");
+            map.values_mut().for_each(|v| *v = timeless(v.take()));
+        }
+        Value::Array(items) => items.iter_mut().for_each(|v| *v = timeless(v.take())),
+        _ => {}
+    }
+    value
+}
+
+/// A store, the kernel path and the lineage config a dispatch needs.
+struct Hosted {
+    store: Store,
+    wasm: PathBuf,
+    config: LineageConfig,
+}
+
+impl Hosted {
+    fn new(store: Store) -> Self {
+        let config = LineageConfig {
+            domain: DOMAIN.to_string(),
+            era: None,
+            mirrored: None,
+            seal_every: None,
+        };
+        Self {
+            store,
+            wasm: wasm_path(),
+            config,
+        }
+    }
+
+    async fn dispatch(&self, verb: &str, args: Value) -> Step {
+        let host = Host::new(&self.store, &self.wasm, &self.config, &AcceptDelivery);
+        match dispatch::handle_as(host, verb, args, Caller::default()).await {
+            Ok(outcome) if outcome.accepted => Step::Accepted(timeless(outcome.result)),
+            Ok(outcome) => Step::Refused(timeless(outcome.result)),
+            Err(_) => Step::Failed,
+        }
+    }
+
+    /// Reads inside a unit that is dropped uncommitted, so nothing is written and the lock is freed.
+    async fn observe(&self) -> anyhow::Result<Observed> {
+        let unit = self
+            .store
+            .begin(DOMAIN)
+            .await
+            .map_err(|e| anyhow::anyhow!("{e}"))?;
+        let state = unit.load().await.map_err(|e| anyhow::anyhow!("{e}"))?;
+        let mut sagas: Vec<_> = state
+            .sagas
+            .iter()
+            .map(|s| {
+                (
+                    s.process_manager.clone(),
+                    s.correlation.clone(),
+                    s.state.clone(),
+                )
+            })
+            .collect();
+        sagas.sort();
+        Ok(Observed {
+            snapshot_ordinal: state.snapshot.as_ref().map(|s| s.ordinal),
+            seed: state
+                .snapshot
+                .map_or_else(|| Value::Object(serde_json::Map::new()), |s| s.seed),
+            sagas,
+        })
+    }
+}
+
+/// The host's in-memory store. A fresh one costs an allocation, so [`Backend::reset`] swaps it.
+pub struct MemoryBackend {
+    hosted: Mutex<Arc<Hosted>>,
+}
+
+impl MemoryBackend {
+    #[must_use]
+    pub fn new() -> Self {
+        Self {
+            hosted: Mutex::new(Self::fresh()),
+        }
+    }
+
+    fn fresh() -> Arc<Hosted> {
+        Arc::new(Hosted::new(Store::memory(MemoryStore::new())))
+    }
+
+    fn current(&self) -> Arc<Hosted> {
+        Arc::clone(&self.hosted.lock().unwrap_or_else(PoisonError::into_inner))
+    }
+}
+
+impl Default for MemoryBackend {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Backend for MemoryBackend {
+    async fn reset(&self) -> anyhow::Result<()> {
+        *self.hosted.lock().unwrap_or_else(PoisonError::into_inner) = Self::fresh();
+        Ok(())
+    }
+
+    async fn dispatch(&self, verb: &str, args: Value) -> Step {
+        self.current().dispatch(verb, args).await
+    }
+
+    async fn observe(&self) -> anyhow::Result<Observed> {
+        self.current().observe().await
+    }
+}
+
+/// The host's Postgres store over one connection that lives as long as the process.
+///
+/// The session is tuned for a scratch database whose only job is to agree with the oracle: commits
+/// do not wait for the WAL (`synchronous_commit = off`) and the tables are unlogged, which removes
+/// the fsync a durable store pays on every accepted command. It follows that this backend says
+/// nothing about crash durability, only about what the store holds once a command has committed.
+pub struct PostgresBackend {
+    hosted: Hosted,
+    /// A second connection that only truncates, because the store keeps its own to itself.
+    resetter: tokio_postgres::Client,
+}
+
+impl PostgresBackend {
+    /// Connects with a libpq-style `conninfo`, creates the host's tables if they are missing and
+    /// tunes the session.
+    ///
+    /// # Errors
+    ///
+    /// Returns the connection or DDL error.
+    pub async fn connect(conninfo: &str) -> anyhow::Result<Self> {
+        let (client, connection) = tokio_postgres::connect(conninfo, NoTls)
+            .await
+            .with_context(|| format!("connecting to {conninfo}"))?;
+        tokio::spawn(async move {
+            let _ = connection.await;
+        });
+        journal_sql::ensure_schema(&client).await?;
+        client.batch_execute("SET synchronous_commit = off").await?;
+        for table in TABLES {
+            client
+                .batch_execute(&format!("ALTER TABLE {table} SET UNLOGGED"))
+                .await?;
+        }
+        Ok(Self {
+            hosted: Hosted::new(rust_host::store::postgres_store(client)),
+            resetter: Self::open(conninfo).await?,
+        })
+    }
+
+    async fn open(conninfo: &str) -> anyhow::Result<tokio_postgres::Client> {
+        let (client, connection) = tokio_postgres::connect(conninfo, NoTls).await?;
+        tokio::spawn(async move {
+            let _ = connection.await;
+        });
+        client.batch_execute("SET synchronous_commit = off").await?;
+        Ok(client)
+    }
+}
+
+impl Backend for PostgresBackend {
+    async fn reset(&self) -> anyhow::Result<()> {
+        let sql = format!("TRUNCATE {} RESTART IDENTITY", TABLES.join(", "));
+        self.resetter.batch_execute(&sql).await?;
+        Ok(())
+    }
+
+    async fn dispatch(&self, verb: &str, args: Value) -> Step {
+        self.hosted.dispatch(verb, args).await
+    }
+
+    async fn observe(&self) -> anyhow::Result<Observed> {
+        self.hosted.observe().await
+    }
+}

data/rust/host/fuzz/src/equivalence.rs

@@ -0,0 +1,195 @@
+//! Plays a script and compares the stores.
+//!
+//! Every script runs against the in-memory store first. That run checks, by itself, what any store
+//! must do: an accepted command pairs the snapshot with the newest journal row, and a refused or
+//! failed one changes nothing. A script then reaches Postgres only if its footprint is due (see
+//! [`crate::footprint`]), and a difference between the two runs is confirmed once more before it is
+//! reported, since the host stamps the wall clock into the arguments a command `needs`.
+//!
+//! Postgres is the point of the comparison: a pass on the in-memory store alone cannot see a
+//! difference between the stores, which is why a footprint seen for the first time always goes on.
+
+use crate::backend::{Backend, MemoryBackend, Observed, PostgresBackend, Step};
+use crate::footprint::{self, Memo};
+use crate::ops::Script;
+use std::sync::atomic::Ordering;
+use std::sync::OnceLock;
+use tokio::runtime::{Builder, Runtime};
+
+/// The libpq-style connection string of the scratch database. Unset, only the in-memory store runs.
+pub const POSTGRES_ENV: &str = "HECKS_FUZZ_PG";
+
+/// How often, in executions, the run says how much it skipped.
+const REPORT_EVERY: u64 = 1 << 14;
+
+/// What one script left behind in one store.
+#[derive(Debug, Clone, PartialEq)]
+struct Trace {
+    steps: Vec<Step>,
+    observed: Observed,
+}
+
+/// Everything that outlives one input: the runtime, both stores, the memo. Built on first use and
+/// reused for the rest of the campaign.
+struct Rig {
+    runtime: Runtime,
+    memory: MemoryBackend,
+    postgres: Option<PostgresBackend>,
+    memo: Memo,
+}
+
+static RIG: OnceLock<Rig> = OnceLock::new();
+static EXECS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+
+fn rig() -> &'static Rig {
+    RIG.get_or_init(|| {
+        // One thread: libFuzzer calls the target from one, and a current-thread runtime adds no
+        // scheduler for the sanitizer to track.
+        let runtime = Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("a current-thread runtime builds");
+        let postgres = std::env::var(POSTGRES_ENV)
+            .ok()
+            .filter(|v| !v.is_empty())
+            .map(|conninfo| {
+                runtime
+                    .block_on(PostgresBackend::connect(&conninfo))
+                    .unwrap_or_else(|e| {
+                        panic!("{POSTGRES_ENV} is set but the store is unusable: {e:#}")
+                    })
+            });
+        Rig {
+            runtime,
+            memory: MemoryBackend::new(),
+            postgres,
+            memo: Memo::default(),
+        }
+    })
+}
+
+/// Runs `script` from an empty store. With `clause` set, the store is read around every step and the
+/// commit clause is asserted, which costs two reads a step; only the cheap store pays for that.
+async fn play<B: Backend>(backend: &B, script: &Script, clause: bool) -> Trace {
+    backend.reset().await.expect("the store resets");
+    let mut steps = Vec::with_capacity(script.ops.len());
+    for (verb, args) in script.steps() {
+        let before = if clause {
+            Some(backend.observe().await.expect("the store reads"))
+        } else {
+            None
+        };
+        let step = backend.dispatch(verb, args).await;
+        if let Some(before) = before {
+            let after = backend.observe().await.expect("the store reads");
+            assert_commit_clause(&step, &before, &after, steps.len());
+        }
+        steps.push(step);
+    }
+    Trace {
+        steps,
+        observed: backend.observe().await.expect("the store reads"),
+    }
+}
+
+/// An accepted command pairs the snapshot with one more journal row; any other leaves the store as
+/// it was (the commit clause the host's conformance suite pins on a fixed script).
+fn assert_commit_clause(step: &Step, before: &Observed, after: &Observed, at: usize) {
+    if step.accepted() {
+        let grew = after.snapshot_ordinal.unwrap_or(0) - before.snapshot_ordinal.unwrap_or(0);
+        assert_eq!(
+            grew, 1,
+            "step {at}: an accepted command advances the snapshot by one row"
+        );
+    } else {
+        assert_eq!(
+            after, before,
+            "step {at}: a {step:?} command changed the store"
+        );
+    }
+}
+
+fn disagreement(memory: &Trace, postgres: &Trace) -> String {
+    format!("memory and postgres disagree\n memory:   {memory:#?}\n postgres: {postgres:#?}")
+}
+
+/// Builds the rig and compiles the kernel, for libFuzzer's initialization hook, which its per-input
+/// timeout does not cover. Compiling the module under a sanitizer takes longer than that timeout,
+/// so the first input would otherwise be reported as a hang.
+///
+/// # Panics
+///
+/// When the kernel is missing or does not compile, or `HECKS_FUZZ_PG` names a store that is unusable.
+pub fn warm() {
+    let _ = rig();
+    rust_host::wasm_runner::warm(&crate::backend::wasm_path())
+        .unwrap_or_else(|e| panic!("the kernel does not compile: {e:#}"));
+}
+
+/// What `check` did with one script, for a caller that wants to know.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct Summary {
+    pub steps: usize,
+    pub accepted: usize,
+    /// Whether Postgres was asked, which is false when it is off or its footprint was already agreed.
+    pub compared: bool,
+}
+
+/// Plays `script` against the in-memory store, and against Postgres when its footprint is due.
+///
+/// # Panics
+///
+/// On an invariant the in-memory run breaks, or a difference between the stores that a second run
+/// reproduces. libFuzzer keeps the input that panicked.
+pub fn check(script: &Script) -> Summary {
+    let rig = rig();
+    let summary = rig.runtime.block_on(async {
+        let memory = play(&rig.memory, script, true).await;
+        // The same script from an empty store answers the same, or no second store can be held to it.
+        let replayed = play(&rig.memory, script, false).await;
+        assert_eq!(
+            memory, replayed,
+            "a replay on the in-memory store answered differently\n first:  {memory:#?}\n second: {replayed:#?}"
+        );
+        let accepted: Vec<bool> = memory.steps.iter().map(Step::accepted).collect();
+        let mut summary = Summary {
+            steps: accepted.len(),
+            accepted: accepted.iter().filter(|a| **a).count(),
+            compared: false,
+        };
+        let Some(postgres) = &rig.postgres else {
+            return summary;
+        };
+        let print = footprint::of(script, &accepted);
+        if !rig.memo.due(print) {
+            return summary;
+        }
+        summary.compared = true;
+        let theirs = play(postgres, script, false).await;
+        if theirs != memory {
+            let again_memory = play(&rig.memory, script, false).await;
+            let again_postgres = play(postgres, script, false).await;
+            assert_eq!(
+                again_memory,
+                again_postgres,
+                "{}",
+                disagreement(&again_memory, &again_postgres)
+            );
+        }
+        rig.memo.agreed(print);
+        summary
+    });
+    report(rig);
+    summary
+}
+
+fn report(rig: &Rig) {
+    let execs = EXECS.fetch_add(1, Ordering::Relaxed) + 1;
+    if execs % REPORT_EVERY == 0 && rig.postgres.is_some() {
+        eprintln!(
+            "[equivalence] {execs} scripts: {} verified on postgres, {} skipped as already agreed",
+            rig.memo.verified.load(Ordering::Relaxed),
+            rig.memo.skipped.load(Ordering::Relaxed),
+        );
+    }
+}