@feltdb/core 0.8.2 → 0.8.4

package/dist/create/server-source/crates/feltdb-server/src/audit.rs

@@ -1,19 +1,91 @@
+//! The per-request security audit log.
+//!
+//! # What this module is for
+//!
+//! Every authenticated request the server serves records exactly one event
+//! here. PR36 measured that record as 55.1% of request-path time; PR37 asked
+//! what the cost bought and established the answer by test:
+//!
+//! * 85.1% of the audit write was one `fdatasync`, taken per request;
+//! * that barrier bought nothing against process death — an event that has
+//!   reached `write(2)` is already recoverable;
+//! * request success never depended on it. A server whose audit path could
+//!   never be opened still returned `201`, committed transactions, and lost
+//!   the security evidence silently.
+//!
+//! So the request was waiting for a barrier no request outcome depended on.
+//!
+//! # The boundary PR38 draws
+//!
+//! Three states, deliberately not conflated:
+//!
+//! ```text
+//! CONSTRUCTED   the event exists in memory
+//! ACCEPTED      the writer has appended it to the audit stream
+//! DURABLE       the group containing it has completed its barrier
+//! ```
+//!
+//! `ACCEPTED != DURABLE`, and `request success != DURABLE`. A request waits
+//! for acceptance and no further:
+//!
+//! ```text
+//! request producers ──► bounded queue ──► appender ──► stream
+//!        ▲                                   │
+//!        └──── ACCEPTED ─────────────────────┘
+//!                                            │
+//!                                     durability group
+//!                                            │
+//!                                            ▼
+//!                                     syncer ──► fdatasync
+//! ```
+//!
+//! The appender is the single writer. That is not an implementation detail:
+//! PR37 showed the old mutex was supplying the audit log's *only* ordering
+//! signal, because the record schema carries no sequence number and file
+//! position is all there is. One thread owning the append preserves exactly
+//! that property, and the concurrency sweep re-proves it.
+//!
+//! The barrier moved to a second thread on purpose. Had the appender taken it
+//! inline, a request arriving mid-barrier would wait for it, and the tail
+//! latency PR38 exists to remove would have come back at the p99.
+//!
+//! # What PR38 does not decide
+//!
+//! Whether losing security evidence should invalidate an application
+//! operation. It does not. A request whose audit event is rejected still
+//! succeeds, exactly as before — but the loss is now counted, timestamped and
+//! visible in the audit health surface instead of vanishing into a log line.
+
 use std::{
-    fs::OpenOptions,
+    fs::{File, OpenOptions},
     io::Write,
     path::PathBuf,
-    sync::{Arc, Mutex},
-    time::{SystemTime, UNIX_EPOCH},
+    sync::{
+        atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering},
+        mpsc::{sync_channel, Receiver, RecvTimeoutError, SyncSender, TrySendError},
+        Arc, Condvar, Mutex,
+    },
+    thread::JoinHandle,
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
 };
 
+use feltdb::workload_diagnostics::{self, Phase};
 use serde::Serialize;
 
+// ---------------------------------------------------------------------------
+// The public surface
+// ---------------------------------------------------------------------------
+
 #[derive(Clone)]
 pub struct AuditLog {
-    path: Arc<PathBuf>,
-    lock: Arc<Mutex<()>>,
+    inner: Arc<AuditLogInner>,
 }
 
+/// One security audit event, borrowed from the request that produced it.
+///
+/// `timestamp_ms` is written by the audit writer, not by the caller: the field
+/// is here so the record shape is declared in one place, and whatever a caller
+/// puts in it is overwritten. A caller cannot backdate an event.
 #[derive(Serialize)]
 pub struct AuditEvent<'a> {
     pub timestamp_ms: u128,
@@ -25,35 +97,1071 @@ pub struct AuditEvent<'a> {
     pub status: u16,
 }
 
+/// The same record, owned, because it crosses a thread boundary.
+///
+/// The field order is the field order of [`AuditEvent`] and must stay that
+/// way: it is the on-disk record shape, and PR38 changes no schema.
+#[derive(Serialize)]
+struct OwnedAuditEvent {
+    timestamp_ms: u128,
+    namespace: String,
+    principal: String,
+    action: String,
+    target: String,
+    outcome: String,
+    status: u16,
+}
+
+impl AuditEvent<'_> {
+    fn to_owned_event(&self) -> OwnedAuditEvent {
+        OwnedAuditEvent {
+            timestamp_ms: 0,
+            namespace: self.namespace.to_string(),
+            principal: self.principal.to_string(),
+            action: self.action.to_string(),
+            target: self.target.to_string(),
+            outcome: self.outcome.to_string(),
+            status: self.status,
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Limits
+// ---------------------------------------------------------------------------
+
+/// The bounds the audit pipeline runs under.
+///
+/// These are internal engineering constants, not a product surface. They are
+/// overridable only in a process that has already opted into test
+/// instrumentation, because a bound a deployment can dial is a bound that has
+/// to be part of the contract, and PR38 is not proposing one.
+#[derive(Debug, Clone, Copy)]
+pub struct AuditLimits {
+    /// How many events may be waiting for the appender at once. Reaching it is
+    /// a defined, observable rejection rather than unbounded memory growth.
+    pub queue_capacity: usize,
+    /// The largest number of events one durability group may contain.
+    pub max_group_events: usize,
+    /// The longest an accepted event may wait before its group is closed.
+    /// Every accepted event participates in a group within this bound unless
+    /// the process terminates first.
+    pub max_group_age: Duration,
+    /// How many closed groups may be waiting for the barrier. When it is
+    /// reached the appender blocks, which is backpressure rather than a leak.
+    pub sync_queue_capacity: usize,
+}
+
+impl Default for AuditLimits {
+    fn default() -> Self {
+        Self {
+            queue_capacity: 1_024,
+            max_group_events: 64,
+            max_group_age: Duration::from_millis(5),
+            sync_queue_capacity: 8,
+        }
+    }
+}
+
+impl AuditLimits {
+    fn resolve() -> Self {
+        let mut limits = Self::default();
+        if !workload_diagnostics::enabled() {
+            return limits;
+        }
+        let read = |name: &str| {
+            std::env::var(name)
+                .ok()
+                .and_then(|raw| raw.trim().parse::<u64>().ok())
+                .filter(|value| *value > 0)
+        };
+        if let Some(value) = read("FELTDB_AUDIT_DIAGNOSTIC_QUEUE_CAPACITY") {
+            limits.queue_capacity = value as usize;
+        }
+        if let Some(value) = read("FELTDB_AUDIT_DIAGNOSTIC_GROUP_EVENTS") {
+            limits.max_group_events = value as usize;
+        }
+        if let Some(value) = read("FELTDB_AUDIT_DIAGNOSTIC_GROUP_AGE_MS") {
+            limits.max_group_age = Duration::from_millis(value);
+        }
+        limits
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Health and counters
+// ---------------------------------------------------------------------------
+
+/// What an operator can see about the audit subsystem.
+///
+/// PR37 proved that security evidence could be lost permanently while every
+/// application response said success, and that nothing anywhere recorded the
+/// loss. This is the answer to that: the counters are cumulative and never
+/// reset, so "it was fine when I looked" is not the same claim as "nothing was
+/// ever lost".
+#[derive(Debug, Clone, Serialize)]
+pub struct AuditHealth {
+    /// `healthy`, `degraded` or `failed`.
+    pub state: &'static str,
+    pub audit_events_accepted_total: u64,
+    pub audit_events_durable_total: u64,
+    pub audit_write_failures_total: u64,
+    pub audit_durability_failures_total: u64,
+    pub audit_queue_rejections_total: u64,
+    pub audit_queue_depth: i64,
+    pub audit_queue_capacity: u64,
+    pub audit_durability_groups_total: u64,
+    pub audit_durability_barriers_total: u64,
+    pub audit_accepted_sequence: u64,
+    pub audit_durable_sequence: u64,
+    /// Accepted but not yet durable. Bounded by the group limits, and the
+    /// exact set of events a machine failure could take.
+    pub audit_accepted_not_durable: u64,
+    pub audit_last_failure_timestamp_ms: Option<u64>,
+    pub audit_last_failure_reason: Option<String>,
+    pub audit_writer_running: bool,
+}
+
+#[derive(Default)]
+struct AuditState {
+    accepted_total: AtomicU64,
+    durable_total: AtomicU64,
+    write_failures_total: AtomicU64,
+    durability_failures_total: AtomicU64,
+    queue_rejections_total: AtomicU64,
+    queue_depth: AtomicI64,
+    groups_total: AtomicU64,
+    barriers_total: AtomicU64,
+    accepted_sequence: AtomicU64,
+    durable_sequence: AtomicU64,
+    last_failure_ms: AtomicU64,
+    last_failure_reason: Mutex<Option<String>>,
+    /// Total failures observed when the previous barrier completed. Recovery
+    /// compares against this rather than simply clearing on any success.
+    failures_at_last_barrier: AtomicU64,
+    writer_running: AtomicBool,
+    degraded: AtomicBool,
+}
+
+impl AuditState {
+    fn record_failure(&self, counter: &AtomicU64, reason: &str) {
+        counter.fetch_add(1, Ordering::Relaxed);
+        self.degraded.store(true, Ordering::Relaxed);
+        self.last_failure_ms.store(now_ms() as u64, Ordering::Relaxed);
+        if let Ok(mut slot) = self.last_failure_reason.lock() {
+            *slot = Some(reason.to_string());
+        }
+    }
+
+    /// A clean barrier is what clears `degraded`.
+    ///
+    /// Not a successful append: an append succeeding says the stream accepts
+    /// writes, and the thing that had failed might have been the barrier. And
+    /// not merely a successful barrier either, because a subsystem shedding
+    /// events under sustained overload completes barriers the whole time it is
+    /// losing evidence — reporting that as healthy is exactly the blindness
+    /// PR37 found and PR38 exists to remove.
+    ///
+    /// So the rule is: a barrier clears `degraded` only when nothing failed
+    /// between it and the barrier before it. Recovery therefore costs two
+    /// clean groups, which is the honest price of not calling an ongoing loss
+    /// healthy.
+    fn record_barrier(&self, events: u64, last_sequence: u64) {
+        self.barriers_total.fetch_add(1, Ordering::Relaxed);
+        self.durable_total.fetch_add(events, Ordering::Relaxed);
+        self.durable_sequence
+            .fetch_max(last_sequence, Ordering::Relaxed);
+        let failures = self.write_failures_total.load(Ordering::Relaxed)
+            + self.durability_failures_total.load(Ordering::Relaxed);
+        if self
+            .failures_at_last_barrier
+            .swap(failures, Ordering::Relaxed)
+            == failures
+        {
+            self.degraded.store(false, Ordering::Relaxed);
+        }
+    }
+
+    fn health(&self, capacity: usize) -> AuditHealth {
+        let running = self.writer_running.load(Ordering::Relaxed);
+        let accepted = self.accepted_total.load(Ordering::Relaxed);
+        let durable = self.durable_total.load(Ordering::Relaxed);
+        let last_failure = self.last_failure_ms.load(Ordering::Relaxed);
+        AuditHealth {
+            state: if !running {
+                "failed"
+            } else if self.degraded.load(Ordering::Relaxed) {
+                "degraded"
+            } else {
+                "healthy"
+            },
+            audit_events_accepted_total: accepted,
+            audit_events_durable_total: durable,
+            audit_write_failures_total: self.write_failures_total.load(Ordering::Relaxed),
+            audit_durability_failures_total: self
+                .durability_failures_total
+                .load(Ordering::Relaxed),
+            audit_queue_rejections_total: self.queue_rejections_total.load(Ordering::Relaxed),
+            audit_queue_depth: self.queue_depth.load(Ordering::Relaxed),
+            audit_queue_capacity: capacity as u64,
+            audit_durability_groups_total: self.groups_total.load(Ordering::Relaxed),
+            audit_durability_barriers_total: self.barriers_total.load(Ordering::Relaxed),
+            audit_accepted_sequence: self.accepted_sequence.load(Ordering::Relaxed),
+            audit_durable_sequence: self.durable_sequence.load(Ordering::Relaxed),
+            audit_accepted_not_durable: accepted.saturating_sub(durable),
+            audit_last_failure_timestamp_ms: (last_failure > 0).then_some(last_failure),
+            audit_last_failure_reason: self
+                .last_failure_reason
+                .lock()
+                .ok()
+                .and_then(|reason| reason.clone()),
+            audit_writer_running: running,
+        }
+    }
+}
+
+fn now_ms() -> u128 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|elapsed| elapsed.as_millis())
+        .unwrap_or(0)
+}
+
+// ---------------------------------------------------------------------------
+// Diagnostics
+// ---------------------------------------------------------------------------
+
+/// Where a fault-injection abort lands inside the audit pipeline.
+///
+/// The three points are the three boundaries the durability contract turns on:
+/// before anything is handed to the kernel, after the bytes are in the kernel
+/// but before the barrier, and after the barrier. PR37's spellings are still
+/// accepted so its scenarios keep their names.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum AuditAbortPoint {
+    BeforeAppend,
+    AfterAppend,
+    AfterGroupSync,
+}
+
+impl AuditAbortPoint {
+    fn parse(raw: &str) -> Option<Self> {
+        match raw.trim() {
+            "before_append" | "before_write" => Some(Self::BeforeAppend),
+            "after_append" | "before_fsync" => Some(Self::AfterAppend),
+            "after_group_sync" | "after_fsync" => Some(Self::AfterGroupSync),
+            _ => None,
+        }
+    }
+
+    pub fn name(self) -> &'static str {
+        match self {
+            Self::BeforeAppend => "before_append",
+            Self::AfterAppend => "after_append",
+            Self::AfterGroupSync => "after_group_sync",
+        }
+    }
+}
+
+/// Where an injected *failure* — as opposed to a crash — is returned.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum AuditFailPoint {
+    Append,
+    GroupSync,
+}
+
+impl AuditFailPoint {
+    fn parse(raw: &str) -> Option<Self> {
+        match raw.trim() {
+            "append" => Some(Self::Append),
+            "group_sync" => Some(Self::GroupSync),
+            _ => None,
+        }
+    }
+
+    pub fn name(self) -> &'static str {
+        match self {
+            Self::Append => "append",
+            Self::GroupSync => "group_sync",
+        }
+    }
+}
+
+/// Test-only switches on the audit path.
+///
+/// Gated twice: each switch has its own environment variable, and none is read
+/// unless `FELTDB_WORKLOAD_DIAGNOSTICS=1` has already put the process into
+/// test-instrumentation mode. A production server sets neither, so with
+/// diagnostics off [`AuditDiagnostics::resolve`] returns the inert value and
+/// the pipeline behaves as the product does.
+///
+/// This is a diagnostic surface, not an alternative production mode.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct AuditDiagnostics {
+    /// Skip the pipeline entirely, to measure what it costs by removing it.
+    pub suppress_write: bool,
+    pub abort_point: Option<AuditAbortPoint>,
+    pub abort_target: Option<&'static str>,
+    /// Fail this stage every time it runs. Two stages fail naturally and are
+    /// therefore not injected: acceptance fails when the bounded queue is
+    /// full, and the append fails when the configured audit path cannot be
+    /// opened. The barrier is the one that has to be forced.
+    pub fail_point: Option<AuditFailPoint>,
+}
+
+impl AuditDiagnostics {
+    pub fn resolve() -> Self {
+        if !workload_diagnostics::enabled() {
+            return Self::default();
+        }
+        let suppress_write =
+            std::env::var("FELTDB_AUDIT_DIAGNOSTIC_SUPPRESS").as_deref() == Ok("1");
+        let abort_point = std::env::var("FELTDB_AUDIT_DIAGNOSTIC_ABORT_POINT")
+            .ok()
+            .and_then(|raw| AuditAbortPoint::parse(&raw));
+        let abort_target = std::env::var("FELTDB_AUDIT_DIAGNOSTIC_ABORT_TARGET")
+            .ok()
+            .filter(|value| !value.is_empty())
+            // Leaked deliberately: the value lives for the process, and a
+            // 'static borrow keeps the writer free of an allocation per event.
+            .map(|value| &*Box::leak(value.into_boxed_str()));
+        let fail_point = std::env::var("FELTDB_AUDIT_DIAGNOSTIC_FAIL_POINT")
+            .ok()
+            .and_then(|raw| AuditFailPoint::parse(&raw));
+        Self {
+            suppress_write,
+            abort_point: abort_point.filter(|_| abort_target.is_some()),
+            abort_target,
+            fail_point,
+        }
+    }
+
+    fn armed_for(&self, point: AuditAbortPoint, target: &str) -> bool {
+        self.abort_point == Some(point)
+            && self
+                .abort_target
+                .is_some_and(|marker| target.contains(marker))
+    }
+
+    fn marks(&self, target: &str) -> bool {
+        self.abort_target
+            .is_some_and(|marker| target.contains(marker))
+    }
+
+    pub fn engaged(&self) -> bool {
+        self.suppress_write || self.abort_point.is_some() || self.fail_point.is_some()
+    }
+
+    pub fn describe(&self) -> String {
+        if !self.engaged() {
+            return "inert: the production audit pipeline, unmodified".to_string();
+        }
+        let mut parts = Vec::new();
+        if self.suppress_write {
+            parts.push("suppress_write".to_string());
+        }
+        if let (Some(point), Some(target)) = (self.abort_point, self.abort_target) {
+            parts.push(format!("abort {} on target containing {target}", point.name()));
+        }
+        if let Some(point) = self.fail_point {
+            parts.push(format!("fail {}", point.name()));
+        }
+        parts.join("; ")
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Acceptance
+// ---------------------------------------------------------------------------
+
+/// The rendezvous between one request and the appender.
+///
+/// The request blocks here until the writer has appended its event or reported
+/// that it could not. It is filled exactly once, and a submission that is
+/// dropped without being appended fills it on the way out, so a writer that
+/// stops can never leave a request waiting forever.
+#[derive(Default)]
+struct AcceptanceSlot {
+    outcome: Mutex<Option<Result<u64, String>>>,
+    ready: Condvar,
+}
+
+impl AcceptanceSlot {
+    fn fill(&self, outcome: Result<u64, String>) {
+        let mut guard = self
+            .outcome
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        if guard.is_none() {
+            *guard = Some(outcome);
+            self.ready.notify_all();
+        }
+    }
+
+    fn wait(&self) -> Result<u64, String> {
+        let mut guard = self
+            .outcome
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        while guard.is_none() {
+            guard = self
+                .ready
+                .wait(guard)
+                .unwrap_or_else(|poisoned| poisoned.into_inner());
+        }
+        guard
+            .clone()
+            .unwrap_or_else(|| Err("audit acceptance was never resolved".to_string()))
+    }
+}
+
+struct EventSubmission {
+    event: OwnedAuditEvent,
+    slot: Arc<AcceptanceSlot>,
+    state: Arc<AuditState>,
+}
+
+impl Drop for EventSubmission {
+    fn drop(&mut self) {
+        self.state.queue_depth.fetch_sub(1, Ordering::Relaxed);
+        // If the writer stopped before this event was appended, the request
+        // waiting on it is released here rather than blocking forever.
+        let unresolved = self
+            .slot
+            .outcome
+            .lock()
+            .map(|guard| guard.is_none())
+            .unwrap_or(false);
+        if unresolved {
+            self.state.record_failure(
+                &self.state.write_failures_total,
+                "the audit writer stopped before this event was accepted",
+            );
+            self.slot.fill(Err(
+                "the audit writer stopped before this event was accepted".to_string(),
+            ));
+        }
+    }
+}
+
+enum Submission {
+    Event(Box<EventSubmission>),
+    Shutdown,
+}
+
+/// One closed durability group, handed from the appender to the syncer.
+struct GroupHandoff {
+    file: File,
+    events: u64,
+    last_sequence: u64,
+    carries_abort_marker: bool,
+}
+
+// ---------------------------------------------------------------------------
+// AuditLog
+// ---------------------------------------------------------------------------
+
+struct AuditLogInner {
+    path: PathBuf,
+    submissions: SyncSender<Submission>,
+    state: Arc<AuditState>,
+    diagnostics: AuditDiagnostics,
+    limits: AuditLimits,
+    threads: Mutex<Vec<JoinHandle<()>>>,
+}
+
+impl Drop for AuditLogInner {
+    fn drop(&mut self) {
+        stop(&self.submissions, &self.threads);
+    }
+}
+
+/// Close the pipeline: tell the appender to finish, then wait for both threads.
+///
+/// The appender closes its open group on the way out and the syncer drains
+/// what is queued, so an orderly stop leaves every accepted event durable.
+fn stop(submissions: &SyncSender<Submission>, threads: &Mutex<Vec<JoinHandle<()>>>) {
+    let handles = {
+        let mut guard = threads
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        std::mem::take(&mut *guard)
+    };
+    if handles.is_empty() {
+        return;
+    }
+    let _ = submissions.send(Submission::Shutdown);
+    for handle in handles {
+        let _ = handle.join();
+    }
+}
+
 impl AuditLog {
     pub fn new(path: PathBuf) -> Self {
+        let diagnostics = AuditDiagnostics::resolve();
+        let limits = AuditLimits::resolve();
+        let state = Arc::new(AuditState::default());
+        state.writer_running.store(true, Ordering::Relaxed);
+
+        let (submissions, incoming) = sync_channel::<Submission>(limits.queue_capacity);
+        let (syncs, pending) = sync_channel::<GroupHandoff>(limits.sync_queue_capacity);
+
+        let appender_state = state.clone();
+        let appender_path = path.clone();
+        let appender = std::thread::Builder::new()
+            .name("feltdb-audit-appender".to_string())
+            .spawn(move || {
+                run_appender(incoming, syncs, appender_path, limits, appender_state, diagnostics);
+            })
+            .expect("the audit appender thread must start");
+
+        let syncer_state = state.clone();
+        let syncer = std::thread::Builder::new()
+            .name("feltdb-audit-syncer".to_string())
+            .spawn(move || run_syncer(pending, syncer_state, diagnostics))
+            .expect("the audit syncer thread must start");
+
         Self {
-            path: Arc::new(path),
-            lock: Arc::new(Mutex::new(())),
+            inner: Arc::new(AuditLogInner {
+                path,
+                submissions,
+                state,
+                diagnostics,
+                limits,
+                threads: Mutex::new(vec![appender, syncer]),
+            }),
         }
     }
 
-    pub fn record(&self, mut event: AuditEvent<'_>) -> Result<(), String> {
-        let _guard = self
-            .lock
-            .lock()
-            .map_err(|_| "audit log lock poisoned".to_string())?;
-        if let Some(parent) = self.path.parent() {
-            std::fs::create_dir_all(parent).map_err(|error| error.to_string())?;
-        }
-        event.timestamp_ms = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .map_err(|error| error.to_string())?
-            .as_millis();
-        let mut file = OpenOptions::new()
-            .create(true)
-            .append(true)
-            .open(&*self.path)
-            .map_err(|error| error.to_string())?;
-        serde_json::to_writer(&mut file, &event).map_err(|error| error.to_string())?;
-        file.write_all(b"\n").map_err(|error| error.to_string())?;
-        file.flush().map_err(|error| error.to_string())?;
-        file.sync_data().map_err(|error| error.to_string())?;
-        Ok(())
+    pub fn diagnostics(&self) -> AuditDiagnostics {
+        self.inner.diagnostics
+    }
+
+    pub fn limits(&self) -> AuditLimits {
+        self.inner.limits
+    }
+
+    pub fn path(&self) -> &PathBuf {
+        &self.inner.path
+    }
+
+    /// What an operator can see. Cheap enough to serve on a metrics route.
+    pub fn health(&self) -> AuditHealth {
+        self.inner.state.health(self.inner.limits.queue_capacity)
+    }
+
+    /// Stop the pipeline and make everything accepted durable.
+    ///
+    /// Idempotent, and also run from `Drop`, so a server that forgets to call
+    /// it still leaves an orderly stream behind.
+    pub fn shutdown(&self) {
+        stop(&self.inner.submissions, &self.inner.threads);
+    }
+
+    /// Record one security audit event, returning once it is ACCEPTED.
+    ///
+    /// "Accepted" means the writer's `write(2)` returned: the bytes are in the
+    /// kernel and survive this process dying. It does **not** mean durable.
+    /// The barrier for the group containing this event happens afterwards, on
+    /// the syncer thread, and no request waits for it.
+    ///
+    /// The error path is unchanged from PR37 in what it does to the request —
+    /// the caller in the request middleware still discards it — and changed in
+    /// what it does to the world: every failure moves a counter, stamps a
+    /// time, and degrades the audit health state.
+    pub fn record(&self, event: AuditEvent<'_>) -> Result<(), String> {
+        if self.inner.diagnostics.suppress_write {
+            return Ok(());
+        }
+        let state = &self.inner.state;
+        let slot = Arc::new(AcceptanceSlot::default());
+        let submission = Submission::Event(Box::new(EventSubmission {
+            event: event.to_owned_event(),
+            slot: slot.clone(),
+            state: state.clone(),
+        }));
+
+        let submitted = {
+            let _span = workload_diagnostics::span(Phase::AuditSubmit);
+            state.queue_depth.fetch_add(1, Ordering::Relaxed);
+            self.inner.submissions.try_send(submission)
+        };
+        if let Err(error) = submitted {
+            let (rejected, reason) = match error {
+                TrySendError::Full(rejected) => {
+                    state.queue_rejections_total.fetch_add(1, Ordering::Relaxed);
+                    (
+                        rejected,
+                        "the audit queue is at capacity: this event was not accepted",
+                    )
+                }
+                TrySendError::Disconnected(rejected) => {
+                    state.writer_running.store(false, Ordering::Relaxed);
+                    (
+                        rejected,
+                        "the audit writer is not running: this event was not accepted",
+                    )
+                }
+            };
+            // Resolve the slot here and count the failure here. Dropping the
+            // rejected submission then only corrects the queue depth: its own
+            // failure path is for events that reached the queue and died with
+            // the writer, which is a different loss and must not be counted
+            // twice as this one.
+            state.record_failure(&state.write_failures_total, reason);
+            if let Submission::Event(submission) = &rejected {
+                submission.slot.fill(Err(reason.to_string()));
+            }
+            drop(rejected);
+            return Err(reason.to_string());
+        }
+
+        let _span = workload_diagnostics::span(Phase::AuditAcceptWait);
+        slot.wait().map(|_| ())
+    }
+}
+
+// ---------------------------------------------------------------------------
+// The appender: the single writer
+// ---------------------------------------------------------------------------
+
+/// The group currently being filled.
+#[derive(Default)]
+struct OpenGroup {
+    events: u64,
+    last_sequence: u64,
+    carries_abort_marker: bool,
+    opened_at: Option<Instant>,
+}
+
+impl OpenGroup {
+    fn remaining(&self, age: Duration) -> Duration {
+        self.opened_at
+            .map(|opened| age.saturating_sub(opened.elapsed()))
+            .unwrap_or(age)
+    }
+}
+
+/// The audit stream: the file, and the one buffer every record is built in.
+struct AuditStream {
+    path: PathBuf,
+    file: Option<File>,
+    /// Reused across events so a record costs no allocation of its own.
+    buffer: Vec<u8>,
+    sequence: u64,
+    state: Arc<AuditState>,
+    diagnostics: AuditDiagnostics,
+}
+
+impl AuditStream {
+    fn new(path: PathBuf, state: Arc<AuditState>, diagnostics: AuditDiagnostics) -> Self {
+        Self {
+            path,
+            file: None,
+            buffer: Vec::with_capacity(512),
+            sequence: 0,
+            state,
+            diagnostics,
+        }
+    }
+
+    /// Open once and keep the handle.
+    ///
+    /// PR37 measured this happening per event, along with a `close`. The
+    /// writer owning the handle is what removes both, and it is the reason the
+    /// per-event syscall count can be stated as a single `write`.
+    fn handle(&mut self) -> Result<&mut File, String> {
+        if self.file.is_none() {
+            let _span = workload_diagnostics::span(Phase::AuditOpen);
+            if let Some(parent) = self.path.parent() {
+                std::fs::create_dir_all(parent).map_err(|error| error.to_string())?;
+            }
+            let opened = OpenOptions::new()
+                .create(true)
+                .append(true)
+                .open(&self.path)
+                .map_err(|error| error.to_string())?;
+            self.file = Some(opened);
+        }
+        Ok(self.file.as_mut().expect("the handle was just opened"))
+    }
+
+    /// Append one event and resolve its acceptance.
+    fn append(&mut self, submission: &EventSubmission, group: &mut OpenGroup) {
+        let mut event = OwnedAuditEvent {
+            timestamp_ms: 0,
+            namespace: submission.event.namespace.clone(),
+            principal: submission.event.principal.clone(),
+            action: submission.event.action.clone(),
+            target: submission.event.target.clone(),
+            outcome: submission.event.outcome.clone(),
+            status: submission.event.status,
+        };
+        // Stamped by the writer, in append order, so the only two ordering
+        // signals the record carries — file position and timestamp — agree.
+        event.timestamp_ms = now_ms();
+
+        if self
+            .diagnostics
+            .armed_for(AuditAbortPoint::BeforeAppend, &event.target)
+        {
+            // Nothing of this event has reached the kernel.
+            std::process::abort();
+        }
+
+        let outcome = self.write_record(&event);
+        match outcome {
+            Ok(sequence) => {
+                self.state.accepted_total.fetch_add(1, Ordering::Relaxed);
+                self.state
+                    .accepted_sequence
+                    .store(sequence, Ordering::Relaxed);
+                group.events += 1;
+                group.last_sequence = sequence;
+                if group.opened_at.is_none() {
+                    group.opened_at = Some(Instant::now());
+                }
+                if self.diagnostics.marks(&event.target) {
+                    group.carries_abort_marker = true;
+                }
+                submission.slot.fill(Ok(sequence));
+                if self
+                    .diagnostics
+                    .armed_for(AuditAbortPoint::AfterAppend, &event.target)
+                {
+                    // The bytes are in the kernel; no barrier has been asked
+                    // for, and the request has just been released.
+                    std::process::abort();
+                }
+            }
+            Err(reason) => {
+                self.state
+                    .record_failure(&self.state.write_failures_total, &reason);
+                submission.slot.fill(Err(reason));
+            }
+        }
+    }
+
+    fn write_record(&mut self, event: &OwnedAuditEvent) -> Result<u64, String> {
+        if self.diagnostics.fail_point == Some(AuditFailPoint::Append) {
+            return Err("injected audit append failure".to_string());
+        }
+        // Serialized into the writer's own buffer and appended with one
+        // `write(2)`. PR37 traced fifty-four writes per event, because the
+        // serializer wrote straight into an unbuffered file; that was never a
+        // durability property, and it does not survive the writer owning
+        // serialization.
+        //
+        // The buffer is moved out and back so the record can be written while
+        // `self` is borrowed for the file handle. `mem::take` leaves an empty
+        // Vec behind and the original — capacity included — is restored.
+        let mut buffer = std::mem::take(&mut self.buffer);
+        buffer.clear();
+        let serialized = serde_json::to_writer(&mut buffer, event).map_err(|error| error.to_string());
+        buffer.push(b'\n');
+        let written = serialized.and_then(|()| {
+            let _span = workload_diagnostics::span(Phase::AuditAppend);
+            self.handle()
+                .and_then(|file| file.write_all(&buffer).map_err(|error| error.to_string()))
+        });
+        self.buffer = buffer;
+        written?;
+        self.sequence += 1;
+        Ok(self.sequence)
+    }
+
+    /// Close the open group and hand it to the syncer.
+    fn close_group(&mut self, group: &mut OpenGroup, syncs: &SyncSender<GroupHandoff>) {
+        if group.events == 0 {
+            *group = OpenGroup::default();
+            return;
+        }
+        let closed = std::mem::take(group);
+        self.state.groups_total.fetch_add(1, Ordering::Relaxed);
+        // The syncer barriers through its own descriptor for the same file, so
+        // the appender can keep appending while a group is being made durable.
+        // A barrier that catches later appends only over-delivers durability;
+        // `durable_sequence` still advances only to the group it was told
+        // about.
+        let duplicated = match self.file.as_ref().map(|file| file.try_clone()) {
+            Some(Ok(file)) => file,
+            Some(Err(error)) => {
+                self.state.record_failure(
+                    &self.state.durability_failures_total,
+                    &format!("could not duplicate the audit handle for a barrier: {error}"),
+                );
+                return;
+            }
+            None => {
+                // The stream was never opened, so nothing was appended and
+                // there is nothing to barrier.
+                return;
+            }
+        };
+        if syncs
+            .send(GroupHandoff {
+                file: duplicated,
+                events: closed.events,
+                last_sequence: closed.last_sequence,
+                carries_abort_marker: closed.carries_abort_marker,
+            })
+            .is_err()
+        {
+            self.state.record_failure(
+                &self.state.durability_failures_total,
+                "the audit syncer is not running: this group was not made durable",
+            );
+        }
+    }
+
+    fn close(&mut self) {
+        if self.file.is_some() {
+            let _span = workload_diagnostics::span(Phase::AuditClose);
+            self.file = None;
+        }
+    }
+}
+
+fn run_appender(
+    incoming: Receiver<Submission>,
+    syncs: SyncSender<GroupHandoff>,
+    path: PathBuf,
+    limits: AuditLimits,
+    state: Arc<AuditState>,
+    diagnostics: AuditDiagnostics,
+) {
+    let mut stream = AuditStream::new(path, state.clone(), diagnostics);
+    let mut group = OpenGroup::default();
+    loop {
+        // An empty group has no deadline, so an idle server performs no
+        // barriers at all: the group age bounds how long an *accepted* event
+        // waits, not how often the writer wakes up.
+        let received = if group.events == 0 {
+            incoming.recv().map_err(|_| RecvTimeoutError::Disconnected)
+        } else {
+            incoming.recv_timeout(group.remaining(limits.max_group_age))
+        };
+        match received {
+            Ok(Submission::Event(submission)) => {
+                stream.append(&submission, &mut group);
+                drop(submission);
+                if group.events >= limits.max_group_events as u64 {
+                    stream.close_group(&mut group, &syncs);
+                }
+            }
+            Ok(Submission::Shutdown) | Err(RecvTimeoutError::Disconnected) => {
+                stream.close_group(&mut group, &syncs);
+                stream.close();
+                break;
+            }
+            Err(RecvTimeoutError::Timeout) => stream.close_group(&mut group, &syncs),
+        }
+    }
+    drop(syncs);
+    state.writer_running.store(false, Ordering::Relaxed);
+}
+
+// ---------------------------------------------------------------------------
+// The syncer: durability, off the request path
+// ---------------------------------------------------------------------------
+
+fn run_syncer(pending: Receiver<GroupHandoff>, state: Arc<AuditState>, diagnostics: AuditDiagnostics) {
+    for mut handoff in pending {
+        let _group = workload_diagnostics::span(Phase::AuditGroupSync);
+        let outcome = if diagnostics.fail_point == Some(AuditFailPoint::GroupSync) {
+            Err("injected audit durability-group failure".to_string())
+        } else {
+            let flushed = {
+                let _span = workload_diagnostics::span(Phase::AuditFlush);
+                handoff.file.flush().map_err(|error| error.to_string())
+            };
+            flushed.and_then(|_| {
+                let _span = workload_diagnostics::span(Phase::AuditFsync);
+                handoff.file.sync_data().map_err(|error| error.to_string())
+            })
+        };
+        match outcome {
+            Ok(()) => {
+                state.record_barrier(handoff.events, handoff.last_sequence);
+                if handoff.carries_abort_marker
+                    && diagnostics.abort_point == Some(AuditAbortPoint::AfterGroupSync)
+                {
+                    std::process::abort();
+                }
+            }
+            Err(reason) => state.record_failure(&state.durability_failures_total, &reason),
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+
+    fn event<'a>(target: &'a str) -> AuditEvent<'a> {
+        AuditEvent {
+            timestamp_ms: 0,
+            namespace: "test",
+            principal: "principal",
+            action: "state:write",
+            target,
+            outcome: "allowed",
+            status: 200,
+        }
+    }
+
+    fn scratch(label: &str) -> PathBuf {
+        let dir = std::env::temp_dir().join(format!(
+            "feltdb-audit-{label}-{}-{:?}",
+            std::process::id(),
+            std::thread::current().id()
+        ));
+        fs::remove_dir_all(&dir).ok();
+        dir
+    }
+
+    #[test]
+    fn an_accepted_event_is_one_json_line_with_a_writer_stamped_timestamp() {
+        let dir = scratch("accept");
+        let path = dir.join("nested").join("audit.log");
+        let log = AuditLog::new(path.clone());
+        log.record(event("/collections/items/a")).expect("accepted");
+        log.record(event("/collections/items/b")).expect("accepted");
+
+        // record() returned, so both events are ACCEPTED: appended and
+        // readable, before any barrier has necessarily happened.
+        let lines: Vec<String> = fs::read_to_string(&path)
+            .expect("audit file exists")
+            .lines()
+            .map(str::to_string)
+            .collect();
+        assert_eq!(lines.len(), 2, "one line per event, in acceptance order");
+        let first: serde_json::Value = serde_json::from_str(&lines[0]).expect("valid JSON");
+        assert_eq!(first["target"], "/collections/items/a");
+        assert!(
+            first["timestamp_ms"].as_u64().unwrap_or(0) > 0,
+            "the writer stamps the time; a caller cannot backdate an event",
+        );
+        assert_eq!(
+            first.as_object().map(|fields| fields.len()),
+            Some(7),
+            "PR38 changes no record schema",
+        );
+
+        let health = log.health();
+        assert_eq!(health.audit_events_accepted_total, 2);
+        log.shutdown();
+        assert!(
+            log.health().audit_events_durable_total >= 2,
+            "an orderly shutdown makes every accepted event durable",
+        );
+        fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn an_unwritable_stream_fails_acceptance_and_is_visible_rather_than_silent() {
+        let dir = scratch("fail");
+        fs::create_dir_all(dir.join("audit.log")).expect("path is a directory");
+        let log = AuditLog::new(dir.join("audit.log"));
+        assert!(
+            log.record(event("/collections/items/a")).is_err(),
+            "an unopenable audit path must fail acceptance, not report success",
+        );
+        let health = log.health();
+        assert_eq!(health.audit_events_accepted_total, 0);
+        assert_eq!(health.audit_write_failures_total, 1);
+        assert_eq!(health.state, "degraded");
+        assert!(
+            health.audit_last_failure_timestamp_ms.is_some(),
+            "the failure is timestamped, which is the whole point of PR38's health surface",
+        );
+        fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn acceptance_is_not_durability() {
+        // The states are distinct and the counters say so: an event can be
+        // accepted while its group has not yet been through a barrier.
+        let dir = scratch("states");
+        let log = AuditLog::new(dir.join("audit.log"));
+        log.record(event("/collections/items/a")).expect("accepted");
+        let accepted = log.health();
+        assert_eq!(accepted.audit_events_accepted_total, 1);
+        assert!(accepted.audit_accepted_sequence >= 1);
+
+        log.shutdown();
+        let durable = log.health();
+        assert_eq!(durable.audit_events_durable_total, 1);
+        assert_eq!(durable.audit_accepted_not_durable, 0);
+        assert!(durable.audit_durability_barriers_total >= 1);
+        fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn a_bounded_queue_rejects_rather_than_growing() {
+        // The rejection path is reachable without a race: a pipeline whose
+        // writer has stopped cannot accept anything, and says so in the same
+        // counters an overloaded one would.
+        let dir = scratch("bounded");
+        let log = AuditLog::new(dir.join("audit.log"));
+        log.record(event("/collections/items/a")).expect("accepted");
+        log.shutdown();
+
+        let rejected = log.record(event("/collections/items/b"));
+        assert!(rejected.is_err(), "a stopped pipeline accepts nothing");
+        let health = log.health();
+        assert_eq!(health.state, "failed", "and an operator can see that it stopped");
+        assert!(health.audit_write_failures_total >= 1);
+        assert!(health.audit_queue_capacity > 0, "the queue bound is declared");
+        fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn a_durability_group_completes_in_the_background_without_a_shutdown() {
+        // The property PR38 rests on: a request that has been released does
+        // not have to wait for, or trigger, the barrier that makes its event
+        // durable. The group closes on its own age bound and the syncer takes
+        // it from there.
+        let dir = scratch("background");
+        let log = AuditLog::new(dir.join("audit.log"));
+        for _ in 0..3 {
+            log.record(event("/collections/items/a")).expect("accepted");
+            std::thread::sleep(Duration::from_millis(20));
+        }
+        let deadline = Instant::now() + Duration::from_secs(5);
+        while log.health().audit_events_durable_total < 3 && Instant::now() < deadline {
+            std::thread::sleep(Duration::from_millis(10));
+        }
+        let health = log.health();
+        assert_eq!(
+            health.audit_events_durable_total, 3,
+            "every accepted event reached a barrier without anyone asking for one",
+        );
+        assert!(health.audit_durability_barriers_total >= 1);
+        assert_eq!(health.state, "healthy");
+        fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn diagnostic_switches_are_inert_unless_workload_diagnostics_is_on() {
+        workload_diagnostics::set_enabled(false);
+        let resolved = AuditDiagnostics::resolve();
+        assert!(!resolved.engaged());
+        assert!(!resolved.suppress_write);
+        assert_eq!(resolved.abort_point, None);
+        assert_eq!(resolved.fail_point, None);
+        assert_eq!(
+            resolved.describe(),
+            "inert: the production audit pipeline, unmodified"
+        );
+        let limits = AuditLimits::resolve();
+        let default = AuditLimits::default();
+        assert_eq!(limits.queue_capacity, default.queue_capacity);
+        assert_eq!(limits.max_group_events, default.max_group_events);
     }
 }