@feltdb/core 0.8.3 → 0.8.5

package/dist/create/server-source/crates/feltdb-server/src/main.rs

@@ -62,7 +62,7 @@ use feltdb::{
     },
     workload::{CreateWorkload, WorkloadStore},
     AtomicMutation, DatabaseSnapshot, FeltDb, FlowError, JsonCasResult, Operation,
-    PeerAdvertisement, PeerId, RecordPrecondition, StoredRow,
+    PeerAdvertisement, PeerId, RecordPrecondition, StateTriggerStore, StoredRow,
 };
 use feltdb_server::{
     app_state::{AppState, BoundedQueryCursor},
@@ -1421,9 +1421,10 @@ struct WorkerReconcileRequest {
 fn workload_error(error: feltdb::workload::WorkloadError) -> ApiError {
     let status = match error.code.as_str() {
         "WORKLOAD_NOT_FOUND" => StatusCode::NOT_FOUND,
-        "WORKLOAD_STALE_FENCE" | "WORKLOAD_TRANSITION_DENIED" | "WORKLOAD_NOT_READY" => {
-            StatusCode::CONFLICT
-        }
+        "WORKLOAD_STALE_FENCE"
+        | "WORKLOAD_TRANSITION_DENIED"
+        | "WORKLOAD_NOT_READY"
+        | "WORKLOAD_NOT_YET_ELIGIBLE" => StatusCode::CONFLICT,
         "WORKLOAD_UNAUTHORIZED" | "WORKLOAD_CAPABILITY_DENIED" => StatusCode::FORBIDDEN,
         "WORKLOAD_STORAGE_FAILURE" => StatusCode::SERVICE_UNAVAILABLE,
         _ => StatusCode::UNPROCESSABLE_ENTITY,
@@ -7861,6 +7862,54 @@ struct SyncPullRequest {
 struct SyncPullResponse {
     operations: Vec<Operation>,
     versions: HashMap<String, u64>,
+    catchup: CatchupStatus,
+}
+
+#[derive(Serialize, Deserialize)]
+struct CatchupStatus {
+    peer_id: String,
+    peer_position: HashMap<String, u64>,
+    authority_position: HashMap<String, u64>,
+    retained_from: HashMap<String, u64>,
+    retained_through: HashMap<String, u64>,
+    lag: u64,
+    recoverability: String,
+    catchup_mode: String,
+    recovery_required: bool,
+}
+
+fn catchup_status(
+    peer_id: String,
+    peer_position: HashMap<String, u64>,
+    authority_position: HashMap<String, u64>,
+    retained_from: HashMap<String, u64>,
+) -> CatchupStatus {
+    let lag = authority_position.iter().map(|(origin, sequence)| {
+        sequence.saturating_sub(peer_position.get(origin).copied().unwrap_or(0))
+    }).sum();
+    let recovery_required = authority_position.iter().any(|(origin, through)| {
+        let position = peer_position.get(origin).copied().unwrap_or(0);
+        let floor = retained_from.get(origin).copied().unwrap_or(through.saturating_add(1));
+        position < *through && position.saturating_add(1) < floor
+    });
+    let catchup_mode = if recovery_required {
+        "snapshot_rebootstrap_required"
+    } else if lag == 0 {
+        "current"
+    } else {
+        "incremental_replay"
+    };
+    CatchupStatus {
+        peer_id,
+        peer_position,
+        retained_through: authority_position.clone(),
+        authority_position,
+        retained_from,
+        lag,
+        recoverability: if recovery_required { "rebootstrap_required" } else { "recoverable" }.into(),
+        catchup_mode: catchup_mode.into(),
+        recovery_required,
+    }
 }
 
 #[derive(Serialize, Deserialize)]
@@ -8241,6 +8290,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     if manage_workloads()? {
         return Ok(());
     }
+    if manage_state_triggers()? {
+        return Ok(());
+    }
     if manage_backup()? {
         return Ok(());
     }
@@ -8250,6 +8302,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let config = Config::from_args().map_err(|error| format!("configuration error: {error}"))?;
     validate_production_configuration(&config)?;
     let db = FeltDb::open(&config.data)?;
+    // Indexes are derived execution structures, so they are declared by
+    // configuration and built from the durable state this open just replayed.
+    // Nothing about them is read from disk: a restart re-derives every bucket
+    // from the records that survived, which is why there is no index format,
+    // no index log, and nothing to repair.
+    let declared_indexes = declare_configured_equality_indexes(&db)?;
+    if !declared_indexes.is_empty() {
+        println!(
+            "feltdb: equality indexes maintained for {}",
+            declared_indexes.join(", ")
+        );
+    }
     let peer_token = match std::env::var(&config.peer_token_env) {
         Ok(token) => Some(Arc::<str>::from(token)),
         Err(_) if config.peers.is_empty() => None,
@@ -8276,6 +8340,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let grant_store = GrantStore::load(config.data.with_extension("grants.json"))?;
     let sync_store = SyncStore::load(config.data.with_extension("sync.json"))?;
     let workload_store = WorkloadStore::load(config.data.with_extension("workloads.json"))?;
+    let state_trigger_store = StateTriggerStore::load(config.data.with_extension("triggers.json"))?;
     let mesh_store = WorkerMeshStore::load(config.data.with_extension("workers.json"))?;
     let content_store = ContentStore::new(config.data.with_extension("content"))?;
     let artifact_store = ArtifactStore::load(
@@ -8287,6 +8352,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let causal_store = CausalStore::load(config.data.with_extension("causal-events.json"))?;
     let provider_store = ProviderStore::load(config.data.with_extension("providers.json"))?;
     let readiness_probe = config.data.with_extension("readiness");
+    // Constructed before the state so the same handle can be stopped after
+    // serving: the pipeline outlives the router by exactly one shutdown.
+    let audit_handle = AuditLog::new(config.audit.clone());
     let state = AppState {
         started_at: Instant::now(),
         ids: Arc::new(AtomicU64::new(db.sequence()?)),
@@ -8299,7 +8367,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             config.data.with_extension("cluster.json"),
             config.peers.clone(),
         )?,
-        audit: AuditLog::new(config.audit.clone()),
+        audit: audit_handle.clone(),
         peer_client,
         peer_token,
         lifecycle_lock: Arc::new(tokio::sync::Mutex::new(())),
@@ -8340,6 +8408,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         grants: Arc::new(std::sync::Mutex::new(grant_store)),
         sync: Arc::new(std::sync::Mutex::new(sync_store)),
         workloads: Arc::new(std::sync::Mutex::new(workload_store)),
+        state_triggers: Arc::new(std::sync::Mutex::new(state_trigger_store)),
         mesh: Arc::new(std::sync::Mutex::new(mesh_store)),
         readiness_probe: Arc::new(readiness_probe),
         bounded_query_cursors: Arc::new(std::sync::Mutex::new(HashMap::new())),
@@ -9016,7 +9085,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             axum::routing::delete(delete_certification_fixture),
         )
         .merge(protected)
+        .merge(query_execution_diagnostics_router())
         .layer(middleware::from_fn(protocol_version))
+        .layer(middleware::from_fn(attribute_request))
         .layer(middleware::from_fn_with_state(state.clone(), count_request))
         .layer(RequestBodyLimitLayer::new(1024 * 1024))
         .layer(TraceLayer::new_for_http())
@@ -9049,9 +9120,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         "FeltDB server ready at http://{bound_address} (namespace: {})",
         config.namespace
     );
-    axum::serve(listener, app)
+    let serving = axum::serve(listener, app)
         .with_graceful_shutdown(shutdown_signal())
-        .await?;
+        .await;
+    // The audit writer owns the stream, so an orderly stop is what makes the
+    // last durability group durable. Run it whether or not serving ended
+    // cleanly: a server that failed still recorded events, and losing the tail
+    // of the security log because of an unrelated bind error would be its own
+    // defect. Dropping the last handle would do this too; doing it here means
+    // it happens at a point in the process the operator can reason about.
+    audit_handle.shutdown();
+    serving?;
     Ok(())
 }
 
@@ -9081,6 +9160,12 @@ async fn authenticate(
     mut request: Request<axum::body::Body>,
     next: Next,
 ) -> Response {
+    // Everything up to the handoff below is authorization: key lookup, session
+    // resolution and the capability check. It is timed separately from the rest
+    // of the request because "the request costs something outside state" is not
+    // an answer to PR36's question — which part of the request does is.
+    let authorization =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::Authorization);
     let principal = if state.auth_enabled {
         let token = request
             .headers()
@@ -9271,6 +9356,7 @@ async fn authenticate(
         .key_id
         .clone();
     let target = request.uri().path().to_string();
+    drop(authorization);
     let response = next.run(request).await;
     let status = response.status().as_u16();
     audit(
@@ -9292,6 +9378,18 @@ fn audit(
     outcome: &str,
     status: u16,
 ) {
+    // Every authenticated request records one of these. Since PR38 the request
+    // waits only for ACCEPTANCE — the audit writer's append — and never for a
+    // durability barrier; the barrier happens afterwards, in a bounded group,
+    // on the writer's own threads.
+    //
+    // The error is still discarded, and that is still deliberate: PR37 proved
+    // request success does not depend on audit durability, and PR38 does not
+    // change that policy. What changed is that the failure is no longer
+    // invisible — `AuditLog::record` has already counted it, timestamped it and
+    // degraded the audit health state by the time this returns.
+    let _span =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::AuditWrite);
     if let Err(error) = state.audit.record(AuditEvent {
         timestamp_ms: 0,
         namespace: &state.namespace,
@@ -9318,6 +9416,21 @@ async fn protocol_version(request: Request<axum::body::Body>, next: Next) -> Res
     next.run(request).await
 }
 
+/// Time a whole request inside the server, from the middleware stack down.
+///
+/// This is the server's own share of a request. Subtracting it from the client's
+/// measured latency leaves transport, connection handling, and the JSON decode
+/// and encode that axum performs in the extractor and the response — none of
+/// which can be bracketed from inside a handler. PR36 reports that difference as
+/// a bounded residual rather than pretending to have decomposed it.
+///
+/// Disabled, this costs one relaxed atomic load per request.
+async fn attribute_request(request: Request<axum::body::Body>, next: Next) -> Response {
+    let _span =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HttpHandler);
+    next.run(request).await
+}
+
 async fn health(State(state): State<AppState>) -> Json<HealthResponse<'static>> {
     let membership = match state.cluster.proposal().map(|value| value.phase) {
         Some(ProposalPhase::Preparing | ProposalPhase::Prepared) => "recovering",
@@ -9459,10 +9572,15 @@ fn manage_keys() -> Result<bool, Box<dyn std::error::Error>> {
                     .map(str::to_string)
                     .collect()
             };
-            let key = store.create(
+            // `--worker <id>` issues a credential that authenticates as that
+            // one worker rather than as a service. It is a narrowing: the
+            // worker lifecycle routes accept it only for its own worker id, and
+            // it cannot satisfy a check that wants a human or a service.
+            let key = store.create_for_worker(
                 name,
                 values("--scope", "state:read,state:write,events:read"),
                 values("--namespace", "default"),
+                option("--worker"),
             )?;
             println!(
                 "API key created (id: {}). This secret will not be shown again:\n{}",
@@ -9477,7 +9595,7 @@ fn manage_keys() -> Result<bool, Box<dyn std::error::Error>> {
             }
             println!("API key revoked: {id}");
         }
-        _ => return Err("usage: feltdb-server keys <create|list|revoke> [--keys path] [--name name] [--scope scopes] [--namespace names] [--id key-id]".into()),
+        _ => return Err("usage: feltdb-server keys <create|list|revoke> [--keys path] [--name name] [--scope scopes] [--namespace names] [--worker worker-id] [--id key-id]".into()),
     }
     Ok(true)
 }
@@ -9575,6 +9693,61 @@ fn manage_workloads() -> Result<bool, Box<dyn std::error::Error>> {
     Ok(true)
 }
 
+fn manage_state_triggers() -> Result<bool, Box<dyn std::error::Error>> {
+    let arguments: Vec<String> = std::env::args().collect();
+    if arguments.get(1).map(String::as_str) != Some("trigger") {
+        return Ok(false);
+    }
+    let command = arguments.get(2).map(String::as_str).unwrap_or("list");
+    let option = |name: &str| {
+        arguments
+            .iter()
+            .position(|value| value == name)
+            .and_then(|index| arguments.get(index + 1))
+            .cloned()
+    };
+    let data = PathBuf::from(option("--data").unwrap_or_else(|| "./data/feltdb.log".into()));
+    let trigger_path = PathBuf::from(option("--store").unwrap_or_else(|| {
+        data.with_extension("triggers.json")
+            .to_string_lossy()
+            .into_owned()
+    }));
+    let mut triggers = StateTriggerStore::load(trigger_path)?;
+    match command {
+        "define" => {
+            let input = option("--input").ok_or("trigger define requires --input trigger.json")?;
+            let trigger: feltdb::StateTrigger = serde_json::from_slice(&std::fs::read(input)?)?;
+            triggers.define(trigger)?;
+            println!("{}", serde_json::to_string_pretty(&triggers.list())?);
+        }
+        "list" => println!("{}", serde_json::to_string_pretty(&triggers.list())?),
+        "status" => println!(
+            "{}",
+            serde_json::to_string_pretty(&json!({
+                "triggers": triggers.triggers.len(),
+                "cursors": triggers.cursor_versions(),
+            }))?
+        ),
+        "evaluate" => {
+            let db = FeltDb::open(&data)?;
+            let workload_path = PathBuf::from(
+                option("--workload-store")
+                    .unwrap_or_else(|| data.with_extension("workloads.json").to_string_lossy().into_owned()),
+            );
+            let mut workloads = WorkloadStore::load(workload_path)?;
+            let produced = triggers.evaluate(&db, &mut workloads, unix_seconds_i64())?;
+            println!("{}", serde_json::to_string_pretty(&produced)?);
+        }
+        _ => {
+            return Err(
+                "usage: feltdb-server trigger <define|list|status|evaluate> [--data path] [--store path] [--workload-store path] [--input trigger.json]"
+                    .into(),
+            )
+        }
+    }
+    Ok(true)
+}
+
 fn manage_backup() -> Result<bool, Box<dyn std::error::Error>> {
     let arguments: Vec<String> = std::env::args().collect();
     if arguments.get(1).map(String::as_str) != Some("backup") {
@@ -9749,10 +9922,21 @@ async fn create_online_backup(
     Ok(Json(manifest))
 }
 
-async fn network_metrics(
-    State(state): State<AppState>,
-) -> Json<feltdb_server::metrics::MetricsSnapshot> {
-    Json(state.metrics.snapshot())
+/// Network counters, plus the audit subsystem's health.
+///
+/// The audit block is here rather than on a route of its own because PR37's
+/// finding was that losing security evidence was invisible, and a signal an
+/// operator has to know to go and look for is not much better than no signal.
+/// It rides with the metrics they already read.
+async fn network_metrics(State(state): State<AppState>) -> Json<Value> {
+    let mut body = serde_json::to_value(state.metrics.snapshot()).unwrap_or_else(|_| json!({}));
+    if let Some(fields) = body.as_object_mut() {
+        fields.insert(
+            "audit".to_string(),
+            serde_json::to_value(state.audit.health()).unwrap_or(Value::Null),
+        );
+    }
+    Json(body)
 }
 
 async fn sync_pull(
@@ -9760,11 +9944,25 @@ async fn sync_pull(
     Json(request): Json<SyncPullRequest>,
 ) -> Result<Json<SyncPullResponse>, ApiError> {
     authorize_sync_member(&state, &request.requester, request.membership_epoch)?;
-    let operations = state.db.operations_since(&request.versions)?;
+    let peer_position = request.versions;
+    let authority_position = state.db.operation_versions()?;
+    let retained_from = state.db.retained_operation_floors()?;
+    let catchup = catchup_status(request.requester, peer_position.clone(), authority_position.clone(), retained_from);
+    if catchup.recovery_required {
+        return Err(ApiError::structured(
+            StatusCode::CONFLICT,
+            serde_json::to_value(&catchup).unwrap_or_else(|_| json!({
+                "catchup_mode": "snapshot_rebootstrap_required",
+                "recovery_required": true
+            })),
+        ));
+    }
+    let operations = state.db.operations_since(&peer_position)?;
     state.metrics.sync_sent(operations.len() as u64);
     Ok(Json(SyncPullResponse {
         operations,
-        versions: state.db.operation_versions()?,
+        versions: authority_position,
+        catchup,
     }))
 }
 
@@ -9971,6 +10169,197 @@ async fn run_worker_pass(state: &AppState) -> Result<(), ApiError> {
         }
         execute_agent_run(state, &row.value, &worker).await?;
     }
+
+    evaluate_state_triggers(state)?;
+    execute_eligible_workloads(state, &worker)?;
+    Ok(())
+}
+
+/// How long an autonomous claim holds its lease.
+///
+/// Long enough that an ordinary capability program finishes inside it, short
+/// enough that a worker dying mid-execution returns the work quickly. The lease
+/// is the only thing that makes worker death recoverable, so this is a real
+/// bound rather than a formality.
+const WORKLOAD_LEASE_MS: u64 = 30_000;
+
+fn state_trigger_store(
+    state: &AppState,
+) -> Result<std::sync::MutexGuard<'_, feltdb::StateTriggerStore>, ApiError> {
+    state.state_triggers.lock().map_err(|_| {
+        ApiError(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "state trigger store unavailable".into(),
+        )
+    })
+}
+
+fn state_trigger_error(error: feltdb::StateTriggerError) -> ApiError {
+    let status = match error.code.as_str() {
+        "TRIGGER_INVALID" | "TRIGGER_UNAUTHORIZED" => StatusCode::BAD_REQUEST,
+        "WORKLOAD_UNAUTHORIZED" => StatusCode::FORBIDDEN,
+        "TRIGGER_STATE_REPLAY_FAILED" | "TRIGGER_STORAGE_FAILURE" | "WORKLOAD_STORAGE_FAILURE" => {
+            StatusCode::SERVICE_UNAVAILABLE
+        }
+        _ => StatusCode::CONFLICT,
+    };
+    ApiError(
+        status,
+        serde_json::to_string(&error).unwrap_or(error.message),
+    )
+}
+
+fn evaluate_state_triggers(state: &AppState) -> Result<(), ApiError> {
+    let mut triggers = state_trigger_store(state)?;
+    if triggers.triggers.is_empty() {
+        return Ok(());
+    }
+    let mut workloads = workload_store(state)?;
+    triggers
+        .evaluate(&state.db, &mut workloads, unix_seconds_i64())
+        .map_err(state_trigger_error)?;
+    Ok(())
+}
+
+/// The durable workload store, or a service error if its lock is poisoned.
+fn workload_store(
+    state: &AppState,
+) -> Result<std::sync::MutexGuard<'_, feltdb::workload::WorkloadStore>, ApiError> {
+    state.workloads.lock().map_err(|_| {
+        ApiError(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "workload store unavailable".into(),
+        )
+    })
+}
+
+/// The capabilities this node can actually run.
+///
+/// This is what the worker presents to [`WorkloadStore::claim`], which requires
+/// the workload's `capability_snapshot` to be a subset of it. That check is
+/// therefore load-bearing rather than ceremonial: a workload whose declared
+/// capabilities this node cannot execute is never claimed by it, and stays
+/// `Ready` for a node that can.
+fn worker_capabilities(state: &AppState) -> Result<BTreeSet<String>, ApiError> {
+    let mut capabilities: BTreeSet<String> = ["search", "identity"]
+        .into_iter()
+        .map(str::to_string)
+        .collect();
+    for row in state.db.list_collection("_flow_capabilities")? {
+        if let Some((_, name)) = row.key.split_once(':') {
+            capabilities.insert(name.to_string());
+        }
+    }
+    Ok(capabilities)
+}
+
+/// Claim and execute the durable workloads this node is eligible to run.
+///
+/// This is the composition that makes `WorkloadStore` an execution primitive
+/// rather than a work data model. It adds no queue and no worker system: the
+/// scan is the pass that already runs, the claim is the existing lease and
+/// fencing, the execution is the existing capability interpreter, and the
+/// transitions are the existing state machine.
+///
+/// Temporal eligibility needs no special handling here — `Workload::is_eligible_at`
+/// filters candidates and `WorkloadStore::claim` refuses anything early, so
+/// `not_before` and retry backoff gate this path by construction.
+///
+/// The store lock is released before execution. A capability program reads
+/// application state through the same database, so holding the workload lock
+/// across it would make every other workload operation wait on unrelated work.
+fn execute_eligible_workloads(state: &AppState, worker: &str) -> Result<(), ApiError> {
+    let capabilities = worker_capabilities(state)?;
+    let now = unix_seconds_i64();
+    let candidates: Vec<(String, String, Value)> = {
+        let store = workload_store(state)?;
+        store
+            .workloads
+            .values()
+            // Selection, not enforcement. `WorkloadStore::claim` independently
+            // refuses work that is early or beyond this node's capabilities, and
+            // that is where the guarantee lives; filtering here only avoids
+            // attempting a claim, every pass, that is known to be refused.
+            .filter(|workload| {
+                workload.state == feltdb::workload::WorkloadState::Ready
+                    && workload.is_eligible_at(now)
+                    && workload.capability_snapshot.is_subset(&capabilities)
+            })
+            .map(|workload| {
+                (
+                    workload.workload_id.clone(),
+                    workload.definition_id.clone(),
+                    workload.input.clone(),
+                )
+            })
+            .collect()
+    };
+
+    for (id, definition, input) in candidates {
+        // Unlike the filter above, this one is enforcement: nothing downstream
+        // knows whether `definition_id` names something runnable. Claiming a
+        // workload only to dead-letter it would consume work another node could
+        // have executed, so an unrunnable definition is left untouched.
+        if !is_executable_capability(state, &definition)? {
+            continue;
+        }
+        let claim = match workload_store(state)?.claim(
+            &id,
+            worker,
+            &capabilities,
+            unix_seconds_i64(),
+            WORKLOAD_LEASE_MS,
+        ) {
+            Ok(claim) => claim,
+            // Another worker took it, or eligibility moved between the scan and
+            // the claim. Both are ordinary outcomes of concurrent workers, not
+            // failures of this pass.
+            Err(_) => continue,
+        };
+        workload_store(state)?
+            .start(
+                &id,
+                &claim.claim_id,
+                worker,
+                claim.fencing_token,
+                unix_seconds_i64(),
+            )
+            .map_err(workload_error)?;
+
+        match execute_named_capability(state, &definition, &input) {
+            Ok(output) => {
+                workload_store(state)?
+                    .complete(
+                        &id,
+                        &claim.claim_id,
+                        worker,
+                        claim.fencing_token,
+                        &format!("result_{id}_{}", claim.fencing_token),
+                        output,
+                        vec![],
+                        1,
+                        unix_seconds_i64(),
+                    )
+                    .map_err(workload_error)?;
+            }
+            Err(error) => {
+                // A capability program that fails does so deterministically, so
+                // this failure kind is deliberately outside the default
+                // `retry_on` set: retrying it would burn attempts to reach the
+                // same dead letter. Only the message is carried, because the
+                // store refuses failure text that looks like a secret.
+                let _ = workload_store(state)?.fail(
+                    &id,
+                    &claim.claim_id,
+                    worker,
+                    claim.fencing_token,
+                    "execution_error",
+                    &format!("capability {definition} failed with status {}", error.0),
+                    unix_seconds_i64(),
+                );
+            }
+        }
+    }
     Ok(())
 }
 
@@ -11795,7 +12184,15 @@ fn query_scalar_cmp(left: Option<&Value>, right: Option<&Value>) -> std::cmp::Or
     }
 }
 
-fn condition_matches(record: &Value, condition: &BoundedQueryCondition) -> Result<bool, ApiError> {
+/// Apply one condition to the field value the record supplies for it.
+///
+/// Taking the field value rather than the record lets the scan path evaluate a
+/// condition against a borrowed record plus its authority-supplied `recordId`,
+/// without first cloning the record to insert that field.
+fn condition_matches_field(
+    actual: Option<&Value>,
+    condition: &BoundedQueryCondition,
+) -> Result<bool, ApiError> {
     if condition.operators.len() != 1 || condition.field.trim().is_empty() {
         return Err(bounded_query_error(
             "INVALID_QUERY",
@@ -11807,7 +12204,6 @@ fn condition_matches(record: &Value, condition: &BoundedQueryCondition) -> Resul
         .iter()
         .next()
         .expect("validated operator");
-    let actual = record.get(&condition.field);
     if operator == "eq" {
         return Ok(actual == Some(expected));
     }
@@ -11832,11 +12228,166 @@ fn condition_matches(record: &Value, condition: &BoundedQueryCondition) -> Resul
     })
 }
 
+/// The record identity the query surface exposes as `recordId`.
+fn bounded_query_record_id(key: &str) -> &str {
+    key.split_once(':').map(|(_, id)| id).unwrap_or(key)
+}
+
+/// Does a stored record satisfy every condition of a bounded query?
+///
+/// This is the same conjunction the materializing path applied, evaluated
+/// against a borrowed record. `recordId` is authority metadata rather than
+/// caller-controlled document data, so it shadows a document field of that name
+/// exactly where the materializing path inserted it: on object records only.
+fn bounded_query_matches(row: &feltdb::StoredRow, conditions: &[BoundedQueryCondition]) -> bool {
+    let record_id = row
+        .value
+        .is_object()
+        .then(|| Value::String(bounded_query_record_id(&row.key).to_string()));
+    conditions.iter().all(|condition| {
+        let actual = match record_id.as_ref() {
+            Some(id) if condition.field == "recordId" => Some(id),
+            _ => row.value.get(&condition.field),
+        };
+        condition_matches_field(actual, condition).unwrap_or(false)
+    })
+}
+
+/// Declare the equality indexes this authority maintains.
+///
+/// `FELTDB_EQUALITY_INDEXES=orders.status,orders.tenantId` — a comma-separated
+/// list of `collection.field`, split at the first dot so a field name may itself
+/// contain one. Declaration is explicit and static on purpose: PR35 adds an
+/// equality execution primitive, not a planner, so nothing here inspects a
+/// workload, collects statistics, or decides on its own what deserves an index.
+///
+/// Each declaration is populated from authoritative state as it is made, so the
+/// authority never serves a query against a half-built index.
+fn declare_configured_equality_indexes(db: &FeltDb) -> Result<Vec<String>, String> {
+    let Ok(declaration) = std::env::var("FELTDB_EQUALITY_INDEXES") else {
+        return Ok(Vec::new());
+    };
+    let mut declared = Vec::new();
+    for entry in declaration.split(',').map(str::trim).filter(|entry| !entry.is_empty()) {
+        let Some((collection, field)) = entry.split_once('.') else {
+            return Err(format!(
+                "FELTDB_EQUALITY_INDEXES entry {entry:?} must be written as collection.field"
+            ));
+        };
+        db.create_equality_index(collection, field)
+            .map_err(|error| format!("cannot index {entry}: {error}"))?;
+        declared.push(format!("{collection}.{field}"));
+    }
+    Ok(declared)
+}
+
+/// Which execution a bounded query is allowed to take.
+///
+/// `Auto` is the only mode the product has: use the index when one applies,
+/// scan otherwise. `Scan` and `Index` exist so a test can run the *same* query
+/// down both paths and compare the externally visible result, which is the only
+/// way to demonstrate that an optimization changed nothing but cost. They are
+/// not a public API: the mode arrives on an undocumented request header that is
+/// read only when the process was started with `FELTDB_QUERY_DIAGNOSTICS=1`, so
+/// a production authority has exactly one execution mode and no way to be asked
+/// for another. The request body — the query contract itself — is untouched.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum BoundedQueryExecution {
+    Auto,
+    Scan,
+    Index,
+}
+
+/// Test instrumentation is compiled in but inert unless the process opted in,
+/// evaluated once so a request cannot pay for the lookup.
+fn query_diagnostics_enabled() -> bool {
+    static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    *ENABLED.get_or_init(|| std::env::var("FELTDB_QUERY_DIAGNOSTICS").as_deref() == Ok("1"))
+}
+
+const QUERY_EXECUTION_HEADER: &str = "x-feltdb-query-execution";
+
+fn bounded_query_execution(headers: &HeaderMap) -> Result<BoundedQueryExecution, ApiError> {
+    if !query_diagnostics_enabled() {
+        return Ok(BoundedQueryExecution::Auto);
+    }
+    match headers
+        .get(QUERY_EXECUTION_HEADER)
+        .and_then(|value| value.to_str().ok())
+    {
+        None | Some("auto") => Ok(BoundedQueryExecution::Auto),
+        Some("scan") => Ok(BoundedQueryExecution::Scan),
+        Some("index") => Ok(BoundedQueryExecution::Index),
+        Some(other) => Err(bounded_query_error(
+            "INVALID_QUERY",
+            format!("unsupported query execution mode: {other}"),
+        )),
+    }
+}
+
+/// The equality conditions of a query, as candidate-selection inputs.
+///
+/// Only `eq` contributes: every other operator is a comparison this index does
+/// not model, and PR35 adds no range index. `recordId` is excluded because on
+/// this surface it is authority metadata that shadows a document field of the
+/// same name on object-valued records only — an index over the document field
+/// would answer a different question from the one `bounded_query_matches` asks,
+/// so a `recordId` condition is never index eligible and its query falls back to
+/// the scan (or is served by candidates from a different, indexed condition).
+///
+/// Conditions the index cannot use are simply absent here. They are not dropped
+/// from the query: `bounded_query_matches` still evaluates every one of them
+/// against authoritative records.
+fn bounded_query_equalities(conditions: &[BoundedQueryCondition]) -> Vec<(&str, &Value)> {
+    conditions
+        .iter()
+        .filter(|condition| condition.field != "recordId" && condition.operators.len() == 1)
+        .filter_map(|condition| {
+            let (operator, expected) = condition.operators.iter().next()?;
+            (operator == "eq").then_some((condition.field.as_str(), expected))
+        })
+        .collect()
+}
+
+/// Evaluate a bounded query's conjunction against authoritative records.
+///
+/// Two executions, one result. The indexed execution narrows *which* records the
+/// predicate sees; `bounded_query_matches` — unchanged, and the sole authority on
+/// what matches — decides the answer on either path. When no index applies the
+/// scan runs exactly as PR34 left it.
+fn bounded_query_records(
+    state: &AppState,
+    request: &BoundedQueryRequest,
+    execution: BoundedQueryExecution,
+) -> Result<Vec<feltdb::StoredRow>, ApiError> {
+    if execution != BoundedQueryExecution::Scan {
+        let equalities = bounded_query_equalities(&request.conditions);
+        let indexed = state.db.query_collection_by_equality(
+            &request.collection,
+            &equalities,
+            |row| bounded_query_matches(row, &request.conditions),
+        )?;
+        if let Some(records) = indexed {
+            return Ok(records);
+        }
+        if execution == BoundedQueryExecution::Index {
+            // Only reachable through the test-only header. A forced-index run
+            // that silently scanned would make an equivalence test prove nothing.
+            return Err(bounded_query_error(
+                "INDEX_UNAVAILABLE",
+                "no equality index applies to this query",
+            ));
+        }
+    }
+    Ok(state.db.query_collection(&request.collection, None, |row| {
+        bounded_query_matches(row, &request.conditions)
+    })?)
+}
+
 fn issue_bounded_cursor(
     state: &AppState,
     mut cursor: BoundedQueryCursor,
-) -> Result<String, ApiError> {
-    let token = uuid::Uuid::new_v4().simple().to_string();
+) -> Result<String, ApiError> {    let token = uuid::Uuid::new_v4().simple().to_string();
     let now = unix_seconds_i64().max(0) as u64;
     cursor.created_at = now;
     let mut cursors = state.bounded_query_cursors.lock().map_err(|_| {
@@ -11859,8 +12410,12 @@ fn issue_bounded_cursor(
 async fn execute_bounded_query(
     State(state): State<AppState>,
     Extension(principal): Extension<Principal>,
+    headers: HeaderMap,
     Json(request): Json<BoundedQueryRequest>,
 ) -> Result<Json<BoundedQueryPage>, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
+    let execution = bounded_query_execution(&headers)?;
     validate_segment(&request.collection)?;
     if request.limit == 0 || request.limit > MAX_BOUNDED_QUERY_LIMIT {
         return Err(bounded_query_error(
@@ -11928,30 +12483,40 @@ async fn execute_bounded_query(
         }
         (cursor.records, cursor.position)
     } else {
-        let mut records = Vec::new();
-        for row in state.db.list_collection(&request.collection)? {
+        // Bounded queries evaluate their conjunction against borrowed records
+        // and clone only the matches, so a query never pays to materialize the
+        // records it is about to discard. An equality index, where one applies,
+        // narrows which records the conjunction is evaluated against; it does
+        // not decide the answer and it does not order it. Ordering is part of
+        // this surface's contract, so the matching set — not the collection, and
+        // not an index bucket — is what gets sorted, and the limit is applied to
+        // that ordered set. Nothing below this line knows which execution ran.
+        let matches = bounded_query_records(&state, &request, execution)?;
+        // Result materialization happens outside the state lock, so it is a cost
+        // this request pays alone rather than one it imposes on every other
+        // operation. PR36 measures it separately for exactly that reason.
+        let materialization = feltdb::workload_diagnostics::span(
+            feltdb::workload_diagnostics::Phase::ResultMaterialization,
+        );
+        let mut records = Vec::with_capacity(matches.len());
+        for row in matches {
             let mut value = row.value;
             if let Some(object) = value.as_object_mut() {
-                let id = row
-                    .key
-                    .split_once(':')
-                    .map(|(_, id)| id)
-                    .unwrap_or(&row.key);
                 // `recordId` is authority metadata for this query surface, not
                 // caller-controlled document data. It is the final total-order
                 // tie-breaker even when a document contains a field by that name.
-                object.insert("recordId".into(), Value::String(id.to_string()));
-            }
-            if request
-                .conditions
-                .iter()
-                .all(|condition| condition_matches(&value, condition).unwrap_or(false))
-            {
-                records.push(value);
+                object.insert(
+                    "recordId".into(),
+                    Value::String(bounded_query_record_id(&row.key).to_string()),
+                );
             }
+            records.push(value);
         }
+        drop(materialization);
         // The authority always adds its immutable record identity as the final
         // tie-breaker, so equal user sort values still form a total order.
+        let ordering =
+            feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::Ordering);
         records.sort_by(|left, right| {
             for order in &request.order_by {
                 let comparison = query_scalar_cmp(left.get(&order.field), right.get(&order.field));
@@ -11965,6 +12530,7 @@ async fn execute_bounded_query(
             }
             query_scalar_cmp(left.get("recordId"), right.get("recordId"))
         });
+        drop(ordering);
         (Arc::new(records), 0)
     };
     let end = position.saturating_add(request.limit).min(records.len());
@@ -11991,6 +12557,131 @@ async fn execute_bounded_query(
     }))
 }
 
+/// Collection-read counters, mounted only when the process opts in.
+///
+/// This is test instrumentation for the PR34 no-materialization proof, not part
+/// of the query contract, so a server only carries the route when it is started
+/// with `FELTDB_QUERY_DIAGNOSTICS=1`. It returns counts, never record data.
+fn query_execution_diagnostics_router() -> Router<AppState> {
+    let mut router = Router::new();
+    if std::env::var("FELTDB_QUERY_DIAGNOSTICS").as_deref() == Ok("1") {
+        router = router.route(
+            "/internal/query-execution-diagnostics",
+            get(query_execution_diagnostics),
+        );
+    }
+    // Phase attribution is a separate switch from the execution counters,
+    // because it is a separate cost: the counters are a handful of atomic adds
+    // per query, while attribution reads the clock at every instrumented
+    // boundary. Keeping them apart lets a run take execution counts without
+    // paying for timing, which is how the headline PR36 figures are measured.
+    if std::env::var("FELTDB_WORKLOAD_DIAGNOSTICS").as_deref() == Ok("1") {
+        router = router
+            .route(
+                "/internal/workload-diagnostics",
+                get(workload_phase_diagnostics),
+            )
+            .route(
+                "/internal/workload-diagnostics/reset",
+                axum::routing::post(reset_workload_phase_diagnostics),
+            );
+    }
+    router
+}
+
+/// Phase attribution: where elapsed time went, as counts and nanoseconds.
+///
+/// Test instrumentation, mounted only under `FELTDB_WORKLOAD_DIAGNOSTICS=1`.
+/// It reports durations and call counts and nothing else — no record, no field,
+/// no value, no id — and no product surface reads it.
+///
+/// `parent` is part of the payload because the phases nest: a caller that summed
+/// every row would double-count the work reported inside `state_lock_hold` and
+/// `indexed_query`. The nesting is declared here so an analysis can compute
+/// exclusive time instead of guessing at it.
+async fn workload_phase_diagnostics() -> Json<Value> {
+    let phases: Vec<Value> = feltdb::workload_diagnostics::counters()
+        .into_iter()
+        .map(|counter| {
+            json!({
+                "phase": counter.phase.name(),
+                "parent": counter.phase.parent().map(|parent| parent.name()),
+                "calls": counter.calls,
+                "nanos": counter.nanos,
+                "meanNanos": counter.mean_nanos(),
+            })
+        })
+        .collect();
+    Json(json!({
+        "enabled": feltdb::workload_diagnostics::enabled(),
+        "phases": phases,
+    }))
+}
+
+/// Zero the phase counters, so a measured window starts from a clean base.
+async fn reset_workload_phase_diagnostics() -> Json<Value> {
+    feltdb::workload_diagnostics::reset();
+    Json(json!({ "reset": true }))
+}
+
+/// Ask the diagnostics route to also verify the index against records.
+#[derive(Deserialize)]
+struct QueryDiagnosticsScope {
+    #[serde(default)]
+    verify: Option<String>,
+}
+
+async fn query_execution_diagnostics(
+    State(state): State<AppState>,
+    Query(scope): Query<QueryDiagnosticsScope>,
+) -> Json<Value> {
+    let counters = feltdb::query_execution_diagnostics::counters();
+    // Verifying rebuilds the whole index from records to compare against the
+    // live one, so it is linear in collection size and is asked for explicitly.
+    // A counter read that silently paid for a rebuild would make every
+    // measurement that brackets a query with a counter read measure the rebuild.
+    let consistency = match scope.verify.as_deref() {
+        Some("1") => state
+            .db
+            .verify_equality_index()
+            .map(|outcome| Value::Bool(outcome.is_ok()))
+            .unwrap_or(Value::Null),
+        _ => Value::Null,
+    };
+    Json(json!({
+        "fullCollectionMaterializations": counters.full_collection_materializations,
+        "fullCollectionRecordsCloned": counters.full_collection_records_cloned,
+        "collectionScans": counters.collection_scans,
+        "scanRecordsVisited": counters.scan_records_visited,
+        "scanRecordsMaterialized": counters.scan_records_materialized,
+        "queriesTotal": counters.queries_total,
+        "queriesScan": counters.queries_scan,
+        "queriesIndexed": counters.queries_indexed,
+        "indexHits": counters.index_hits,
+        "indexMisses": counters.index_misses,
+        "indexCandidatesExamined": counters.index_candidates_examined,
+        "recordsPredicateEvaluated": counters.records_predicate_evaluated,
+        "equalityIndexes": state
+            .db
+            .equality_indexes()
+            .map(|indexes| {
+                indexes
+                    .into_iter()
+                    .map(|(collection, field)| format!("{collection}.{field}"))
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default(),
+        "equalityIndexStats": state.db.equality_index_stats().map(|stats| json!({
+            "indexedFields": stats.indexed_fields,
+            "valueBuckets": stats.value_buckets,
+            "entries": stats.entries,
+        })).unwrap_or(Value::Null),
+        // Null unless `?verify=1` asked for the check; a null here means "not
+        // checked", never "checked and inconsistent".
+        "equalityIndexConsistent": consistency,
+    }))
+}
+
 async fn list_records(
     State(state): State<AppState>,
     Path(collection): Path<String>,
@@ -12015,6 +12706,8 @@ async fn get_record(
     State(state): State<AppState>,
     Path((collection, id)): Path<(String, String)>,
 ) -> Result<Json<RecordResponse>, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -12123,6 +12816,8 @@ async fn commit_transaction(
     State(state): State<AppState>,
     Json(request): Json<AtomicTxRequest>,
 ) -> Result<(StatusCode, Json<AtomicTxResponse>), ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if request.transaction_id.trim().is_empty() {
         return Err(ApiError(
             StatusCode::BAD_REQUEST,
@@ -12272,6 +12967,8 @@ async fn create_record(
     Path(collection): Path<String>,
     Json(mut record): Json<CreateRecord>,
 ) -> Result<(StatusCode, Json<RecordResponse>), ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -12316,6 +13013,8 @@ async fn update_record(
     Path((collection, id)): Path<(String, String)>,
     Json(changes): Json<Map<String, Value>>,
 ) -> Result<Json<RecordResponse>, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -12353,6 +13052,8 @@ async fn compare_and_set_record(
     Path((collection, id)): Path<(String, String)>,
     Json(request): Json<CasRecordRequest>,
 ) -> Result<Response, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     let key = record_key(&collection, &id)?;
     let result = state.db.compare_and_set_json(
         &key,
@@ -12442,6 +13143,8 @@ async fn delete_record(
     State(state): State<AppState>,
     Path((collection, id)): Path<(String, String)>,
 ) -> Result<StatusCode, ApiError> {
+    let _handler =
+        feltdb::workload_diagnostics::span(feltdb::workload_diagnostics::Phase::HandlerBody);
     if collection == PROPOSAL_COLLECTION
         || collection == PROPOSAL_EVENT_COLLECTION
         || collection == CONVERGENCE_COLLECTION
@@ -12562,6 +13265,69 @@ async fn shutdown_signal() {
     tokio::select! { _ = ctrl_c => {}, _ = terminate => {} }
 }
 
+#[cfg(test)]
+mod worker_identity_tests {
+    use super::grant_subject;
+    use feltdb::authorization::Subject as GrantSubject;
+    use feltdb_server::auth::Principal;
+
+    fn principal(subject_type: &str, identity: Option<&str>) -> Principal {
+        Principal {
+            key_id: "key-1".into(),
+            scopes: vec!["*".into()],
+            subject_type: subject_type.into(),
+            identity_id: identity.map(str::to_string),
+            session_id: None,
+        }
+    }
+
+    /// The workload worker lifecycle compares the authenticated subject against
+    /// the `worker_id` in the request, so which subject each principal maps to
+    /// *is* the security boundary. Nothing but the authenticated principal
+    /// participates in this mapping.
+    #[test]
+    fn only_a_worker_principal_maps_to_a_worker_subject() {
+        assert_eq!(
+            grant_subject(&principal("worker", Some("worker-x"))),
+            GrantSubject::Worker("worker-x".into()),
+        );
+        // Every other principal maps somewhere a worker check can never accept,
+        // no matter what identity it carries or what scopes it holds.
+        for (subject_type, expected) in [
+            ("human", GrantSubject::Human("worker-x".into())),
+            ("user", GrantSubject::Human("worker-x".into())),
+            ("service", GrantSubject::Service("worker-x".into())),
+            ("api_key", GrantSubject::Service("worker-x".into())),
+            ("agent", GrantSubject::Agent("worker-x".into())),
+        ] {
+            let subject = grant_subject(&principal(subject_type, Some("worker-x")));
+            assert_eq!(subject, expected, "{subject_type} mapped unexpectedly");
+            assert_ne!(
+                subject,
+                GrantSubject::Worker("worker-x".into()),
+                "{subject_type} must never satisfy a worker check"
+            );
+        }
+    }
+
+    /// A worker principal is one worker, not any worker.
+    #[test]
+    fn a_worker_subject_is_bound_to_its_own_identity() {
+        let subject = grant_subject(&principal("worker", Some("worker-x")));
+        assert_ne!(subject, GrantSubject::Worker("worker-y".into()));
+    }
+
+    /// Without an identity the principal falls back to its key id, so a worker
+    /// credential missing its worker id cannot silently become a wildcard.
+    #[test]
+    fn a_worker_principal_without_an_identity_falls_back_to_its_key_id() {
+        assert_eq!(
+            grant_subject(&principal("worker", None)),
+            GrantSubject::Worker("key-1".into()),
+        );
+    }
+}
+
 #[cfg(test)]
 mod authority_gate_tests {
     use super::{api_error_body, authorize_transaction_collections, state_authorization};