@feltdb/core 0.12.0 → 0.12.1

package/dist/create/server-source/crates/feltdb-server/src/query_shape.rs

@@ -0,0 +1,1137 @@
+//! A bounded, opt-in, value-free observer of bounded-query *shapes*.
+//!
+//! This is diagnostic evidence about how a workload is structured. It is never
+//! authoritative: nothing in the query path reads it, no query result, error or
+//! ordering depends on it, and a failure inside it degrades observation, not the
+//! application operation. See `docs/architecture/query-shape-observer.md`.
+//!
+//! # What a shape is
+//!
+//! A shape is the structure of a `POST /query` request with every value removed:
+//! the collection, the multiset of `(field, operator)` predicates, the ordered
+//! `(field, direction)` sort, and a coarse limit class. The input type of this
+//! module ([`ShapeInput`]) has no place to put a predicate value, a cursor, a
+//! principal or a tenant, so a value cannot be recorded by construction rather
+//! than by discipline.
+//!
+//! # Bounds
+//!
+//! * Per operation, outside the lock: normalization is O(P log P + O) with
+//!   `P <= MAX_PREDICATES` and `O <= MAX_ORDER_FIELDS`; anything larger is
+//!   classified [`Unsupported`], not truncated.
+//! * Per operation, inside the lock: one hash lookup and a handful of integer
+//!   additions. The one exception is admitting a *new* shape into a full table,
+//!   which scans at most `capacity <= MAX_CAPACITY` entries to find a victim.
+//! * Memory: at most [`ObserverConfig::memory_bound_bytes`], fixed at
+//!   construction. Nothing grows with request count.
+//!
+//! The mutex is taken only after the application work has finished, for the
+//! duration of those additions. It is never held across, or inside, any part of
+//! query execution, and the state lock the database owns is untouched.
+
+use serde::Serialize;
+use serde_json::json;
+use sha2::{Digest, Sha256};
+use std::collections::{BTreeSet, HashMap};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Mutex;
+use std::time::Instant;
+
+/// Version of the normalized representation. Bumped whenever [`Shape::canonical`]
+/// would produce a different string for the same request.
+pub const SHAPE_VERSION: u32 = 1;
+pub const DEFAULT_CAPACITY: usize = 128;
+pub const MAX_CAPACITY: usize = 1024;
+pub const MAX_PREDICATES: usize = 16;
+pub const MAX_ORDER_FIELDS: usize = 4;
+/// Longest collection or field name that is retained. Longer is unsupported.
+pub const MAX_IDENTIFIER_BYTES: usize = 128;
+/// Log2-microsecond latency buckets; the last bucket is open-ended.
+pub const LATENCY_BUCKETS: usize = 24;
+/// Per-slot cost of the hash table beyond the key and value themselves.
+const HASH_ENTRY_OVERHEAD: usize = 64;
+
+pub const ENABLE_VARIABLE: &str = "FELTDB_QUERY_SHAPE_OBSERVER";
+pub const CAPACITY_VARIABLE: &str = "FELTDB_QUERY_SHAPE_CAPACITY";
+pub const EXCLUDE_COLLECTIONS_VARIABLE: &str = "FELTDB_QUERY_SHAPE_EXCLUDE_COLLECTIONS";
+pub const EXCLUDE_FIELDS_VARIABLE: &str = "FELTDB_QUERY_SHAPE_EXCLUDE_FIELDS";
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Operator {
+    Eq,
+    Neq,
+    Lt,
+    Lte,
+    Gt,
+    Gte,
+}
+
+impl Operator {
+    fn parse(value: &str) -> Option<Self> {
+        Some(match value {
+            "eq" => Self::Eq,
+            "neq" => Self::Neq,
+            "lt" => Self::Lt,
+            "lte" => Self::Lte,
+            "gt" => Self::Gt,
+            "gte" => Self::Gte,
+            _ => return None,
+        })
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Direction {
+    Asc,
+    Desc,
+}
+
+impl Direction {
+    fn parse(value: &str) -> Option<Self> {
+        match value {
+            "asc" => Some(Self::Asc),
+            "desc" => Some(Self::Desc),
+            _ => None,
+        }
+    }
+}
+
+/// The request limit, reduced to the class that matters for grouping.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize)]
+pub enum LimitClass {
+    #[serde(rename = "1")]
+    One,
+    #[serde(rename = "2-10")]
+    UpTo10,
+    #[serde(rename = "11-100")]
+    UpTo100,
+    #[serde(rename = "101-1000")]
+    UpTo1000,
+}
+
+impl LimitClass {
+    fn of(limit: usize) -> Option<Self> {
+        match limit {
+            1 => Some(Self::One),
+            2..=10 => Some(Self::UpTo10),
+            11..=100 => Some(Self::UpTo100),
+            101..=1000 => Some(Self::UpTo1000),
+            _ => None,
+        }
+    }
+}
+
+/// Why a query is not retained as a shape. Closed set; no free text.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Unsupported {
+    /// The request carries a semantic clause; its structure is not modelled.
+    SemanticClause,
+    UnknownOperator,
+    UnknownDirection,
+    LimitOutOfRange,
+    TooManyPredicates,
+    TooManyOrderFields,
+    IdentifierTooLong,
+}
+
+impl Unsupported {
+    const ALL: [Unsupported; 7] = [
+        Self::SemanticClause,
+        Self::UnknownOperator,
+        Self::UnknownDirection,
+        Self::LimitOutOfRange,
+        Self::TooManyPredicates,
+        Self::TooManyOrderFields,
+        Self::IdentifierTooLong,
+    ];
+    fn index(self) -> usize {
+        Self::ALL
+            .iter()
+            .position(|value| *value == self)
+            .unwrap_or(0)
+    }
+}
+
+/// Why a query was deliberately not observed. Closed set; no names.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+pub enum Skipped {
+    /// An internal FeltDB collection. Never application workload.
+    Reserved,
+    /// A collection or field the operator excluded.
+    Excluded,
+    Unsupported(Unsupported),
+}
+
+/// The only information the observer accepts about a query.
+///
+/// There is deliberately no field for a value, a cursor, a principal or a
+/// tenant. Predicates and sort keys are borrowed names and operator/direction
+/// words exactly as the request spelled them.
+pub struct ShapeInput<'a> {
+    pub collection: &'a str,
+    /// `(field, operator)` per ANDed condition.
+    pub predicates: &'a [(&'a str, &'a str)],
+    /// `(field, direction)` in request order.
+    pub order_by: &'a [(&'a str, &'a str)],
+    pub limit: usize,
+    pub has_semantic_clause: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
+pub struct Shape {
+    pub version: u32,
+    pub collection: String,
+    /// Sorted: a conjunction is order-independent, so equivalent requests must
+    /// normalize identically. Duplicates are kept; `a=1 AND a=2` is not `a=1`.
+    pub predicates: Vec<(String, Operator)>,
+    /// Request order: sort order is semantically relevant.
+    pub order_by: Vec<(String, Direction)>,
+    pub limit: LimitClass,
+}
+
+impl Shape {
+    /// The unambiguous, deterministic string that defines shape identity.
+    /// JSON escaping makes names containing separators unable to collide.
+    pub fn canonical(&self) -> String {
+        json!([
+            self.version,
+            self.collection,
+            self.predicates
+                .iter()
+                .map(|(field, operator)| json!([field, operator]))
+                .collect::<Vec<_>>(),
+            self.order_by
+                .iter()
+                .map(|(field, direction)| json!([field, direction]))
+                .collect::<Vec<_>>(),
+            self.limit,
+        ])
+        .to_string()
+    }
+
+    /// Bytes this shape occupies, counted from its contents. An estimate of
+    /// retained memory, not an allocator measurement.
+    fn approx_bytes(&self) -> usize {
+        let names = self.collection.len()
+            + self
+                .predicates
+                .iter()
+                .map(|(field, _)| field.len() + 48)
+                .sum::<usize>()
+            + self
+                .order_by
+                .iter()
+                .map(|(field, _)| field.len() + 48)
+                .sum::<usize>();
+        std::mem::size_of::<Self>() + names
+    }
+
+    /// A short stable identifier derived from [`Shape::canonical`].
+    pub fn id(&self) -> String {
+        let digest = Sha256::digest(self.canonical().as_bytes());
+        digest[..8]
+            .iter()
+            .map(|byte| format!("{byte:02x}"))
+            .collect()
+    }
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct ObserverConfig {
+    pub capacity: usize,
+    pub exclude_collections: BTreeSet<String>,
+    /// `(collection, field)`; `None` collection applies to every collection.
+    pub exclude_fields: BTreeSet<(Option<String>, String)>,
+}
+
+impl ObserverConfig {
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            capacity: capacity.clamp(1, MAX_CAPACITY),
+            ..Self::default()
+        }
+    }
+
+    /// Resolve configuration. `Ok(None)` is the default: observation disabled.
+    ///
+    /// A malformed value is an error rather than a silent default. An operator
+    /// who mistyped an exclusion list must not get an observer that quietly
+    /// ignores it.
+    pub fn from_environment() -> Result<Option<Self>, String> {
+        Self::from_values(
+            std::env::var(ENABLE_VARIABLE).ok().as_deref(),
+            std::env::var(CAPACITY_VARIABLE).ok().as_deref(),
+            std::env::var(EXCLUDE_COLLECTIONS_VARIABLE).ok().as_deref(),
+            std::env::var(EXCLUDE_FIELDS_VARIABLE).ok().as_deref(),
+        )
+    }
+
+    pub fn from_values(
+        enable: Option<&str>,
+        capacity: Option<&str>,
+        exclude_collections: Option<&str>,
+        exclude_fields: Option<&str>,
+    ) -> Result<Option<Self>, String> {
+        if enable != Some("1") {
+            return Ok(None);
+        }
+        let capacity = match capacity.map(str::trim).filter(|value| !value.is_empty()) {
+            None => DEFAULT_CAPACITY,
+            Some(value) => match value.parse::<usize>() {
+                Ok(parsed) if (1..=MAX_CAPACITY).contains(&parsed) => parsed,
+                _ => {
+                    return Err(format!(
+                        "{CAPACITY_VARIABLE} must be an integer from 1 to {MAX_CAPACITY}"
+                    ))
+                }
+            },
+        };
+        let mut config = Self::with_capacity(capacity);
+        for entry in list(exclude_collections) {
+            config.exclude_collections.insert(entry.to_string());
+        }
+        for entry in list(exclude_fields) {
+            // `collection.field` or `*.field`, split at the first dot.
+            let Some((collection, field)) = entry.split_once('.') else {
+                return Err(format!(
+                    "{EXCLUDE_FIELDS_VARIABLE} entry {entry:?} must be written as collection.field or *.field"
+                ));
+            };
+            if collection.is_empty() || field.is_empty() {
+                return Err(format!(
+                    "{EXCLUDE_FIELDS_VARIABLE} entry {entry:?} must name a collection and a field"
+                ));
+            }
+            let collection = (collection != "*").then(|| collection.to_string());
+            config
+                .exclude_fields
+                .insert((collection, field.to_string()));
+        }
+        Ok(Some(config))
+    }
+
+    /// Upper bound on the bytes the retained table can occupy.
+    ///
+    /// An entry holds its shape and the canonical string that keys it. Both are
+    /// bounded by the identifier, predicate and sort limits.
+    pub fn memory_bound_bytes(&self) -> usize {
+        // Each name is at most MAX_IDENTIFIER_BYTES of heap plus a 24-byte String
+        // header, an operator or direction and allocator slack (48 covers both).
+        let names = (1 + MAX_PREDICATES + MAX_ORDER_FIELDS) * (MAX_IDENTIFIER_BYTES + 48);
+        let per_entry = std::mem::size_of::<Shape>()
+            + names
+            + std::mem::size_of::<Entry>()
+            + HASH_ENTRY_OVERHEAD;
+        self.capacity * per_entry
+    }
+}
+
+fn list(value: Option<&str>) -> impl Iterator<Item = &str> {
+    value
+        .unwrap_or("")
+        .split(',')
+        .map(str::trim)
+        .filter(|entry| !entry.is_empty())
+}
+
+/// Collections that belong to FeltDB, not to the application.
+///
+/// Anything with a leading underscore is reserved (`_feltdb.*`, `_retention`),
+/// and `state` is the internal revision capability. They are excluded by
+/// default and not configurable: observing them would attribute FeltDB's own
+/// activity to the application workload.
+pub fn is_reserved_collection(collection: &str) -> bool {
+    collection.starts_with('_') || collection == "state"
+}
+
+/// Pure normalization with no policy beyond structural support.
+pub fn normalize(input: &ShapeInput<'_>) -> Result<Shape, Unsupported> {
+    if input.has_semantic_clause {
+        return Err(Unsupported::SemanticClause);
+    }
+    if input.predicates.len() > MAX_PREDICATES {
+        return Err(Unsupported::TooManyPredicates);
+    }
+    if input.order_by.len() > MAX_ORDER_FIELDS {
+        return Err(Unsupported::TooManyOrderFields);
+    }
+    let too_long = |name: &str| name.len() > MAX_IDENTIFIER_BYTES;
+    if too_long(input.collection)
+        || input.predicates.iter().any(|(field, _)| too_long(field))
+        || input.order_by.iter().any(|(field, _)| too_long(field))
+    {
+        return Err(Unsupported::IdentifierTooLong);
+    }
+    let limit = LimitClass::of(input.limit).ok_or(Unsupported::LimitOutOfRange)?;
+    let mut predicates = Vec::with_capacity(input.predicates.len());
+    for (field, operator) in input.predicates {
+        let operator = Operator::parse(operator).ok_or(Unsupported::UnknownOperator)?;
+        predicates.push((field.to_string(), operator));
+    }
+    predicates.sort();
+    let mut order_by = Vec::with_capacity(input.order_by.len());
+    for (field, direction) in input.order_by {
+        let direction = Direction::parse(direction).ok_or(Unsupported::UnknownDirection)?;
+        order_by.push((field.to_string(), direction));
+    }
+    Ok(Shape {
+        version: SHAPE_VERSION,
+        collection: input.collection.to_string(),
+        predicates,
+        order_by,
+        limit,
+    })
+}
+
+/// Which access path the real execution took. Only what the path reports.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Access {
+    Indexed,
+    Scan,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
+pub struct LatencyStats {
+    pub count: u64,
+    pub sum_nanos: u64,
+    pub max_nanos: u64,
+    /// Bucket `i` counts latencies in `[2^i, 2^(i+1))` microseconds (bucket 0
+    /// also holds sub-microsecond); the last bucket is open-ended.
+    pub log2_micros: [u64; LATENCY_BUCKETS],
+}
+
+impl LatencyStats {
+    fn record(&mut self, nanos: u64) {
+        self.count += 1;
+        self.sum_nanos = self.sum_nanos.saturating_add(nanos);
+        self.max_nanos = self.max_nanos.max(nanos);
+        let micros = (nanos / 1_000).max(1);
+        let bucket = (63 - micros.leading_zeros() as usize).min(LATENCY_BUCKETS - 1);
+        self.log2_micros[bucket] += 1;
+    }
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
+pub struct ShapeStats {
+    /// Fresh executions of the query (page one): `ok + errors`.
+    pub executions: u64,
+    pub ok: u64,
+    /// Failures after the request was validated and observation began.
+    pub errors: u64,
+    /// Cursor continuations. These replay a stored result; they execute nothing.
+    pub continuations: u64,
+    pub indexed: u64,
+    pub scans: u64,
+    /// Records matching the conjunction before the limit, over successful
+    /// executions that reported it.
+    pub matched_sum: u64,
+    pub matched_max: u64,
+    /// Records returned in the page, over successful executions.
+    pub returned_sum: u64,
+    /// Handler time from validated request to response, fresh executions only.
+    pub latency: LatencyStats,
+}
+
+#[derive(Debug, Clone)]
+struct Entry {
+    first_seq: u64,
+    stats: ShapeStats,
+}
+
+impl Entry {
+    fn weight(&self) -> u64 {
+        self.stats.executions + self.stats.continuations
+    }
+}
+
+#[derive(Default)]
+struct Table {
+    /// Keyed by the shape itself, so the hit path allocates no canonical string.
+    entries: HashMap<Shape, Entry>,
+    next_seq: u64,
+    evictions: u64,
+    evicted_executions: u64,
+    reserved: u64,
+    excluded: u64,
+    unsupported: [u64; Unsupported::ALL.len()],
+}
+
+/// What the observer can report. Counts and names of structure only.
+#[derive(Debug, Clone, Serialize)]
+pub struct ObserverSnapshot {
+    pub shape_version: u32,
+    pub capacity: usize,
+    pub memory_bound_bytes: usize,
+    pub memory_estimate_bytes: usize,
+    pub retained_shapes: usize,
+    pub evictions: u64,
+    /// Executions that had been counted for shapes since evicted.
+    pub evicted_executions: u64,
+    pub reserved: u64,
+    pub excluded: u64,
+    pub unsupported: Vec<(Unsupported, u64)>,
+    /// Observer-internal failures (a poisoned lock). Never application errors.
+    pub observer_failures: u64,
+    /// Ordered by executions descending, then canonical shape ascending.
+    pub shapes: Vec<ShapeReport>,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct ShapeReport {
+    pub id: String,
+    pub shape: Shape,
+    pub first_seen_sequence: u64,
+    pub stats: ShapeStats,
+}
+
+impl ObserverSnapshot {
+    /// The snapshot with every wall-clock measurement zeroed, so two runs of the
+    /// same trace compare equal. Everything else is a pure function of the
+    /// ordered trace and the configuration.
+    pub fn without_timing(mut self) -> Self {
+        for report in &mut self.shapes {
+            report.stats.latency = LatencyStats::default();
+        }
+        self
+    }
+}
+
+pub struct QueryShapeObserver {
+    config: ObserverConfig,
+    table: Mutex<Table>,
+    failures: AtomicU64,
+}
+
+impl QueryShapeObserver {
+    pub fn new(mut config: ObserverConfig) -> Self {
+        config.capacity = config.capacity.clamp(1, MAX_CAPACITY);
+        Self {
+            config,
+            table: Mutex::new(Table::default()),
+            failures: AtomicU64::new(0),
+        }
+    }
+
+    pub fn config(&self) -> &ObserverConfig {
+        &self.config
+    }
+
+    fn is_excluded(&self, input: &ShapeInput<'_>) -> bool {
+        if self.config.exclude_collections.contains(input.collection) {
+            return true;
+        }
+        if self.config.exclude_fields.is_empty() {
+            return false;
+        }
+        let hit = |field: &str| {
+            self.config
+                .exclude_fields
+                .contains(&(None, field.to_string()))
+                || self
+                    .config
+                    .exclude_fields
+                    .contains(&(Some(input.collection.to_string()), field.to_string()))
+        };
+        input.predicates.iter().any(|(field, _)| hit(field))
+            || input.order_by.iter().any(|(field, _)| hit(field))
+    }
+
+    /// Decide whether and how to observe a validated query.
+    ///
+    /// Exclusions are applied before any shape is built, so an excluded name is
+    /// never copied, hashed or stored. The result is always an [`Observation`];
+    /// it records nothing unless the query was admitted.
+    pub fn begin(&self, input: &ShapeInput<'_>) -> Observation<'_> {
+        let skipped = if is_reserved_collection(input.collection) {
+            Some(Skipped::Reserved)
+        } else if self.is_excluded(input) {
+            Some(Skipped::Excluded)
+        } else {
+            None
+        };
+        let outcome = match skipped {
+            Some(skipped) => Err(skipped),
+            None => normalize(input).map_err(Skipped::Unsupported),
+        };
+        match outcome {
+            Ok(shape) => Observation {
+                observer: self,
+                shape: Some(shape),
+                started: Some(Instant::now()),
+                done: false,
+            },
+            Err(skipped) => {
+                self.count_skipped(skipped);
+                Observation {
+                    observer: self,
+                    shape: None,
+                    started: None,
+                    done: true,
+                }
+            }
+        }
+    }
+
+    fn with_table(&self, update: impl FnOnce(&mut Table)) {
+        match self.table.lock() {
+            Ok(mut table) => update(&mut table),
+            Err(_) => {
+                self.failures.fetch_add(1, Ordering::Relaxed);
+            }
+        }
+    }
+
+    fn count_skipped(&self, skipped: Skipped) {
+        self.with_table(|table| match skipped {
+            Skipped::Reserved => table.reserved += 1,
+            Skipped::Excluded => table.excluded += 1,
+            Skipped::Unsupported(reason) => table.unsupported[reason.index()] += 1,
+        });
+    }
+
+    fn record(&self, shape: Shape, update: impl FnOnce(&mut ShapeStats)) {
+        let capacity = self.config.capacity;
+        self.with_table(|table| {
+            if let Some(entry) = table.entries.get_mut(&shape) {
+                update(&mut entry.stats);
+                return;
+            }
+            if table.entries.len() >= capacity {
+                // Deterministic victim: lowest weight, then oldest admission.
+                // `first_seq` is unique, so there are no further ties.
+                let victim = table
+                    .entries
+                    .iter()
+                    .min_by_key(|(_, entry)| (entry.weight(), entry.first_seq))
+                    .map(|(shape, _)| shape.clone());
+                if let Some(victim) = victim {
+                    if let Some(removed) = table.entries.remove(&victim) {
+                        table.evictions += 1;
+                        table.evicted_executions += removed.stats.executions;
+                    }
+                }
+            }
+            let first_seq = table.next_seq;
+            table.next_seq += 1;
+            let mut stats = ShapeStats::default();
+            update(&mut stats);
+            table.entries.insert(shape, Entry { first_seq, stats });
+        });
+    }
+
+    pub fn snapshot(&self) -> ObserverSnapshot {
+        let failures = self.failures.load(Ordering::Relaxed);
+        let mut shapes = Vec::new();
+        let (mut evictions, mut evicted_executions, mut reserved, mut excluded) = (0, 0, 0, 0);
+        let mut unsupported = [0u64; Unsupported::ALL.len()];
+        let mut estimate = std::mem::size_of::<Table>();
+        if let Ok(table) = self.table.lock() {
+            evictions = table.evictions;
+            evicted_executions = table.evicted_executions;
+            reserved = table.reserved;
+            excluded = table.excluded;
+            unsupported = table.unsupported;
+            for (shape, entry) in &table.entries {
+                estimate +=
+                    shape.approx_bytes() + std::mem::size_of::<Entry>() + HASH_ENTRY_OVERHEAD;
+                shapes.push((shape.canonical(), shape.clone(), entry.clone()));
+            }
+        }
+        shapes.sort_by(|(left_key, _, left), (right_key, _, right)| {
+            right
+                .stats
+                .executions
+                .cmp(&left.stats.executions)
+                .then_with(|| left_key.cmp(right_key))
+        });
+        ObserverSnapshot {
+            shape_version: SHAPE_VERSION,
+            capacity: self.config.capacity,
+            memory_bound_bytes: self.config.memory_bound_bytes(),
+            memory_estimate_bytes: estimate,
+            retained_shapes: shapes.len(),
+            evictions,
+            evicted_executions,
+            reserved,
+            excluded,
+            unsupported: Unsupported::ALL
+                .iter()
+                .map(|reason| (*reason, unsupported[reason.index()]))
+                .collect(),
+            observer_failures: failures,
+            shapes: shapes
+                .into_iter()
+                .map(|(_, shape, entry)| ShapeReport {
+                    id: shape.id(),
+                    shape,
+                    first_seen_sequence: entry.first_seq,
+                    stats: entry.stats,
+                })
+                .collect(),
+        }
+    }
+}
+
+/// One in-flight observation of a validated query.
+///
+/// Dropping it without finishing records an error outcome: every path out of
+/// the handler, including `?` early returns and cancellation, is accounted for
+/// without the handler enumerating them.
+pub struct Observation<'a> {
+    observer: &'a QueryShapeObserver,
+    shape: Option<Shape>,
+    started: Option<Instant>,
+    done: bool,
+}
+
+impl Observation<'_> {
+    /// Whether this query was admitted into observation.
+    pub fn is_recorded(&self) -> bool {
+        self.shape.is_some()
+    }
+
+    /// A fresh execution completed. `matched` is the conjunction's match count
+    /// before the limit; `returned` is the page size.
+    pub fn finish_executed(mut self, access: Access, matched: usize, returned: usize) {
+        self.done = true;
+        let nanos = self.elapsed_nanos();
+        if let Some(shape) = self.shape.take() {
+            self.observer.record(shape, |stats| {
+                stats.executions += 1;
+                stats.ok += 1;
+                match access {
+                    Access::Indexed => stats.indexed += 1,
+                    Access::Scan => stats.scans += 1,
+                }
+                stats.matched_sum = stats.matched_sum.saturating_add(matched as u64);
+                stats.matched_max = stats.matched_max.max(matched as u64);
+                stats.returned_sum = stats.returned_sum.saturating_add(returned as u64);
+                stats.latency.record(nanos);
+            });
+        }
+    }
+
+    /// A cursor continuation was served from a stored result.
+    pub fn finish_continuation(mut self, returned: usize) {
+        self.done = true;
+        if let Some(shape) = self.shape.take() {
+            self.observer.record(shape, |stats| {
+                stats.continuations += 1;
+                stats.returned_sum = stats.returned_sum.saturating_add(returned as u64);
+            });
+        }
+    }
+
+    fn elapsed_nanos(&self) -> u64 {
+        self.started
+            .map(|started| started.elapsed().as_nanos().min(u64::MAX as u128) as u64)
+            .unwrap_or(0)
+    }
+}
+
+impl Drop for Observation<'_> {
+    fn drop(&mut self) {
+        if self.done {
+            return;
+        }
+        let nanos = self.elapsed_nanos();
+        if let Some(shape) = self.shape.take() {
+            self.observer.record(shape, |stats| {
+                stats.executions += 1;
+                stats.errors += 1;
+                stats.latency.record(nanos);
+            });
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn input<'a>(
+        collection: &'a str,
+        predicates: &'a [(&'a str, &'a str)],
+        order_by: &'a [(&'a str, &'a str)],
+        limit: usize,
+    ) -> ShapeInput<'a> {
+        ShapeInput {
+            collection,
+            predicates,
+            order_by,
+            limit,
+            has_semantic_clause: false,
+        }
+    }
+
+    fn observer(capacity: usize) -> QueryShapeObserver {
+        QueryShapeObserver::new(ObserverConfig::with_capacity(capacity))
+    }
+
+    #[test]
+    fn disabled_is_the_default() {
+        assert_eq!(
+            ObserverConfig::from_values(None, None, None, None),
+            Ok(None)
+        );
+        assert_eq!(
+            ObserverConfig::from_values(Some("0"), None, None, None),
+            Ok(None)
+        );
+        assert_eq!(
+            ObserverConfig::from_values(Some("true"), Some("9"), None, None),
+            Ok(None),
+            "only the literal 1 enables observation"
+        );
+        let enabled = ObserverConfig::from_values(Some("1"), None, None, None)
+            .unwrap()
+            .unwrap();
+        assert_eq!(enabled.capacity, DEFAULT_CAPACITY);
+    }
+
+    #[test]
+    fn malformed_configuration_is_an_error_not_a_default() {
+        for capacity in ["0", "-1", "abc", "1025"] {
+            assert!(
+                ObserverConfig::from_values(Some("1"), Some(capacity), None, None).is_err(),
+                "capacity {capacity}"
+            );
+        }
+        for field in ["nodot", ".field", "coll."] {
+            assert!(
+                ObserverConfig::from_values(Some("1"), None, None, Some(field)).is_err(),
+                "field {field}"
+            );
+        }
+        // A disabled observer ignores malformed settings: it reads nothing.
+        assert_eq!(
+            ObserverConfig::from_values(None, Some("abc"), None, Some("nodot")),
+            Ok(None)
+        );
+    }
+
+    #[test]
+    fn equivalent_structures_share_one_shape() {
+        let a = normalize(&input(
+            "orders",
+            &[("status", "eq"), ("tenant", "eq")],
+            &[("createdAt", "desc")],
+            50,
+        ))
+        .unwrap();
+        let b = normalize(&input(
+            "orders",
+            &[("tenant", "eq"), ("status", "eq")],
+            &[("createdAt", "desc")],
+            99,
+        ))
+        .unwrap();
+        assert_eq!(
+            a, b,
+            "condition order and limit within a class do not matter"
+        );
+        assert_eq!(a.canonical(), b.canonical());
+        assert_eq!(a.id(), b.id());
+        assert_eq!(a.id().len(), 16);
+    }
+
+    #[test]
+    fn semantically_relevant_differences_produce_distinct_shapes() {
+        let base = normalize(&input("o", &[("a", "eq")], &[("x", "asc")], 10)).unwrap();
+        let variants = [
+            normalize(&input("p", &[("a", "eq")], &[("x", "asc")], 10)),
+            normalize(&input("o", &[("b", "eq")], &[("x", "asc")], 10)),
+            normalize(&input("o", &[("a", "gt")], &[("x", "asc")], 10)),
+            normalize(&input("o", &[("a", "eq")], &[("x", "desc")], 10)),
+            normalize(&input("o", &[("a", "eq")], &[("y", "asc")], 10)),
+            normalize(&input(
+                "o",
+                &[("a", "eq")],
+                &[("x", "asc"), ("y", "asc")],
+                10,
+            )),
+            normalize(&input("o", &[("a", "eq")], &[("x", "asc")], 11)),
+            normalize(&input(
+                "o",
+                &[("a", "eq"), ("a", "eq")],
+                &[("x", "asc")],
+                10,
+            )),
+        ];
+        let mut ids = BTreeSet::from([base.id()]);
+        for variant in variants {
+            assert!(ids.insert(variant.unwrap().id()));
+        }
+        // Sort order is meaningful: (x, y) is not (y, x).
+        let xy = normalize(&input("o", &[], &[("x", "asc"), ("y", "asc")], 1)).unwrap();
+        let yx = normalize(&input("o", &[], &[("y", "asc"), ("x", "asc")], 1)).unwrap();
+        assert_ne!(xy.id(), yx.id());
+    }
+
+    #[test]
+    fn names_containing_separators_cannot_collide() {
+        let one = normalize(&input("c", &[("a\",\"b", "eq")], &[], 1)).unwrap();
+        let two = normalize(&input("c", &[("a", "eq"), ("b", "eq")], &[], 1)).unwrap();
+        assert_ne!(one.canonical(), two.canonical());
+    }
+
+    #[test]
+    fn unsupported_forms_are_classified_not_guessed() {
+        let mut semantic = input("o", &[], &[], 1);
+        semantic.has_semantic_clause = true;
+        assert_eq!(normalize(&semantic), Err(Unsupported::SemanticClause));
+        assert_eq!(
+            normalize(&input("o", &[("a", "like")], &[], 1)),
+            Err(Unsupported::UnknownOperator)
+        );
+        assert_eq!(
+            normalize(&input("o", &[], &[("a", "up")], 1)),
+            Err(Unsupported::UnknownDirection)
+        );
+        assert_eq!(
+            normalize(&input("o", &[], &[], 0)),
+            Err(Unsupported::LimitOutOfRange)
+        );
+        assert_eq!(
+            normalize(&input("o", &[], &[], 1001)),
+            Err(Unsupported::LimitOutOfRange)
+        );
+        let many = vec![("a", "eq"); MAX_PREDICATES + 1];
+        assert_eq!(
+            normalize(&input("o", &many, &[], 1)),
+            Err(Unsupported::TooManyPredicates)
+        );
+        let sorts = vec![("a", "asc"); MAX_ORDER_FIELDS + 1];
+        assert_eq!(
+            normalize(&input("o", &[], &sorts, 1)),
+            Err(Unsupported::TooManyOrderFields)
+        );
+        let long = "x".repeat(MAX_IDENTIFIER_BYTES + 1);
+        assert_eq!(
+            normalize(&input("o", &[(long.as_str(), "eq")], &[], 1)),
+            Err(Unsupported::IdentifierTooLong)
+        );
+    }
+
+    #[test]
+    fn unsupported_queries_are_counted_by_reason_and_never_retained() {
+        let observer = observer(4);
+        let mut semantic = input("o", &[("a", "eq")], &[], 1);
+        semantic.has_semantic_clause = true;
+        let observation = observer.begin(&semantic);
+        assert!(!observation.is_recorded());
+        observation.finish_executed(Access::Scan, 1, 1);
+        let snapshot = observer.snapshot();
+        assert!(snapshot.shapes.is_empty());
+        assert!(snapshot
+            .unsupported
+            .contains(&(Unsupported::SemanticClause, 1)));
+    }
+
+    #[test]
+    fn reserved_collections_are_excluded_by_default() {
+        let observer = observer(4);
+        for name in ["_feltdb.Proposal", "_feltdb_x", "_retention", "state"] {
+            assert!(is_reserved_collection(name), "{name}");
+            observer
+                .begin(&input(name, &[("a", "eq")], &[], 1))
+                .finish_executed(Access::Scan, 0, 0);
+        }
+        assert!(!is_reserved_collection("orders"));
+        let snapshot = observer.snapshot();
+        assert_eq!(snapshot.reserved, 4);
+        assert!(snapshot.shapes.is_empty());
+        let rendered = serde_json::to_string(&snapshot).unwrap();
+        assert!(
+            !rendered.contains("feltdb"),
+            "reserved names must not be retained"
+        );
+    }
+
+    #[test]
+    fn excluded_collections_and_fields_never_reach_retained_data() {
+        let mut config = ObserverConfig::with_capacity(8);
+        config.exclude_collections.insert("secrets".into());
+        config.exclude_fields.insert((None, "ssn".into()));
+        config
+            .exclude_fields
+            .insert((Some("orders".into()), "cardNumber".into()));
+        let observer = QueryShapeObserver::new(config);
+        type Case<'a> = (&'a str, &'a [(&'a str, &'a str)], &'a [(&'a str, &'a str)]);
+        let cases: [Case<'_>; 5] = [
+            ("secrets", &[("a", "eq")], &[]),
+            ("people", &[("ssn", "eq")], &[]),
+            ("people", &[("a", "eq")], &[("ssn", "asc")]),
+            ("orders", &[("cardNumber", "eq")], &[]),
+            ("orders", &[("a", "eq"), ("cardNumber", "gt")], &[]),
+        ];
+        for (collection, predicates, order) in cases {
+            observer
+                .begin(&input(collection, predicates, order, 1))
+                .finish_executed(Access::Scan, 0, 0);
+        }
+        // Not excluded: the same field name on another collection.
+        observer
+            .begin(&input("other", &[("cardNumber", "eq")], &[], 1))
+            .finish_executed(Access::Scan, 0, 0);
+        let snapshot = observer.snapshot();
+        assert_eq!(snapshot.excluded, 5);
+        assert_eq!(snapshot.shapes.len(), 1);
+        let rendered = serde_json::to_string(&snapshot).unwrap();
+        for leaked in ["secrets", "ssn", "people"] {
+            assert!(!rendered.contains(leaked), "{leaked} leaked: {rendered}");
+        }
+    }
+
+    #[test]
+    fn statistics_aggregate_by_shape_and_never_hold_values() {
+        let observer = observer(8);
+        let predicates = [("status", "eq")];
+        let order = [("createdAt", "desc")];
+        for matched in [3usize, 7] {
+            observer
+                .begin(&input("orders", &predicates, &order, 50))
+                .finish_executed(Access::Indexed, matched, matched.min(5));
+        }
+        observer
+            .begin(&input("orders", &predicates, &order, 50))
+            .finish_executed(Access::Scan, 10, 5);
+        observer
+            .begin(&input("orders", &predicates, &order, 50))
+            .finish_continuation(5);
+        // Dropped without finishing: an error after validation.
+        drop(observer.begin(&input("orders", &predicates, &order, 50)));
+        let snapshot = observer.snapshot();
+        assert_eq!(snapshot.shapes.len(), 1);
+        let stats = &snapshot.shapes[0].stats;
+        assert_eq!(stats.executions, 4);
+        assert_eq!((stats.ok, stats.errors), (3, 1));
+        assert_eq!(stats.continuations, 1);
+        assert_eq!((stats.indexed, stats.scans), (2, 1));
+        assert_eq!((stats.matched_sum, stats.matched_max), (20, 10));
+        assert_eq!(stats.returned_sum, 3 + 5 + 5 + 5);
+        assert_eq!(stats.latency.count, 4);
+        assert_eq!(stats.latency.log2_micros.iter().sum::<u64>(), 4);
+    }
+
+    #[test]
+    fn capacity_and_eviction_are_deterministic() {
+        let run = || {
+            let observer = observer(3);
+            for round in 0..4 {
+                for collection in ["a", "b", "c", "d", "e", "f"] {
+                    if round == 0 || collection != "f" {
+                        observer
+                            .begin(&input(collection, &[("x", "eq")], &[], 1))
+                            .finish_executed(Access::Scan, 1, 1);
+                    }
+                }
+            }
+            observer.snapshot().without_timing()
+        };
+        let first = run();
+        let second = run();
+        assert_eq!(first.retained_shapes, 3);
+        assert!(first.retained_shapes <= first.capacity);
+        assert!(first.evictions > 0);
+        assert_eq!(
+            serde_json::to_string(&first).unwrap(),
+            serde_json::to_string(&second).unwrap(),
+            "same trace, same retained shapes and counters"
+        );
+    }
+
+    #[test]
+    fn eviction_prefers_the_lightest_then_the_oldest_shape() {
+        let observer = observer(2);
+        let heavy = input("heavy", &[("x", "eq")], &[], 1);
+        for _ in 0..5 {
+            observer.begin(&heavy).finish_executed(Access::Scan, 0, 0);
+        }
+        observer
+            .begin(&input("old", &[("x", "eq")], &[], 1))
+            .finish_executed(Access::Scan, 0, 0);
+        observer
+            .begin(&input("new", &[("x", "eq")], &[], 1))
+            .finish_executed(Access::Scan, 0, 0);
+        let snapshot = observer.snapshot();
+        let names: Vec<_> = snapshot
+            .shapes
+            .iter()
+            .map(|report| report.shape.collection.as_str())
+            .collect();
+        assert_eq!(names, ["heavy", "new"]);
+        assert_eq!(snapshot.evictions, 1);
+        assert_eq!(snapshot.evicted_executions, 1);
+    }
+
+    #[test]
+    fn memory_stays_within_the_declared_bound() {
+        let observer = observer(MAX_CAPACITY);
+        let long = "n".repeat(MAX_IDENTIFIER_BYTES);
+        let fields: Vec<String> = (0..MAX_PREDICATES)
+            .map(|i| format!("{i:02}{long}")[..MAX_IDENTIFIER_BYTES].to_string())
+            .collect();
+        for index in 0..(MAX_CAPACITY * 2) {
+            let name = format!("c{index}");
+            let predicates: Vec<(&str, &str)> =
+                fields.iter().map(|field| (field.as_str(), "eq")).collect();
+            observer
+                .begin(&input(&name, &predicates, &[], 1))
+                .finish_executed(Access::Scan, 0, 0);
+        }
+        let snapshot = observer.snapshot();
+        assert_eq!(snapshot.retained_shapes, MAX_CAPACITY);
+        assert!(
+            snapshot.memory_estimate_bytes <= snapshot.memory_bound_bytes,
+            "{} > {}",
+            snapshot.memory_estimate_bytes,
+            snapshot.memory_bound_bytes
+        );
+    }
+
+    #[test]
+    fn a_poisoned_lock_degrades_observation_without_panicking() {
+        let observer = observer(4);
+        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            let _guard = observer.table.lock().unwrap();
+            panic!("poison the table");
+        }));
+        observer
+            .begin(&input("o", &[("a", "eq")], &[], 1))
+            .finish_executed(Access::Scan, 1, 1);
+        drop(observer.begin(&input("o", &[("a", "eq")], &[], 1)));
+        observer.begin(&input("_x", &[], &[], 1));
+        let snapshot = observer.snapshot();
+        assert!(snapshot.observer_failures >= 3);
+        assert!(snapshot.shapes.is_empty());
+    }
+
+    #[test]
+    fn concurrent_recording_loses_nothing_within_capacity() {
+        let observer = std::sync::Arc::new(observer(16));
+        let handles: Vec<_> = (0..8)
+            .map(|worker| {
+                let observer = observer.clone();
+                std::thread::spawn(move || {
+                    for _ in 0..500 {
+                        let name = format!("c{}", worker % 4);
+                        observer
+                            .begin(&input(&name, &[("a", "eq")], &[], 1))
+                            .finish_executed(Access::Indexed, 1, 1);
+                    }
+                })
+            })
+            .collect();
+        for handle in handles {
+            handle.join().unwrap();
+        }
+        let snapshot = observer.snapshot();
+        let total: u64 = snapshot.shapes.iter().map(|s| s.stats.executions).sum();
+        assert_eq!(total, 4000);
+        assert_eq!(snapshot.shapes.len(), 4);
+    }
+}