@hatk/hatk 0.0.1-alpha.9 → 0.0.1-alpha.92

package/dist/indexer.js

@@ -1,24 +1,42 @@
+import { reshapeRow } from "./database/db.js";
+import { unfilteredQuerySQL } from "./spaces/guard.js";
+import { rkeyOf } from "./spaces/uri.js";
 import { cborDecode } from "./cbor.js";
 import { parseCarFrame } from "./car.js";
-import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses } from "./db.js";
+import { isPrivateCollection } from "./private-collections.js";
+import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses, getDatabasePort, updateRepoHandle, } from "./database/db.js";
 import { backfillRepo } from "./backfill.js";
-import { rebuildAllIndexes } from "./fts.js";
+import { rebuildAllIndexes } from "./database/fts.js";
 import { log, emit, timer } from "./logger.js";
 import { runLabelRules } from "./labels.js";
-import { getLexiconArray } from "./schema.js";
+import { fireOnCommitHooks } from "./hooks.js";
+import { getLexiconArray } from "./database/schema.js";
 import { validateRecord } from '@bigmoves/lexicon';
 let buffer = [];
 let flushTimer = null;
 let lastSeq = null;
+let lastPersistedSeq = null;
+let cursorCheckpointTimer = null;
+/**
+ * Which `_cursor` row this process persists its stream position to.
+ *
+ * The relay's `subscribeRepos` seq and Jetstream's seq are different
+ * coordinate systems — resuming one from the other's value would either skip
+ * a swathe of the stream or replay from a nonsense offset. Each source owns
+ * its own key so switching between them (or back) is safe.
+ */
+let cursorKey = 'relay';
 const BATCH_SIZE = 100;
 const FLUSH_INTERVAL_MS = 500;
+const CURSOR_CHECKPOINT_INTERVAL_MS = 5_000;
 let writesSinceRebuild = 0;
 let ftsRebuildInterval = 500;
 // Event buffer for DIDs mid-backfill
 const pendingBuffers = new Map();
 // Track in-flight backfills to avoid duplicates
 const backfillInFlight = new Set();
-const MAX_CONCURRENT_BACKFILLS = 5;
+const backfillPromises = new Map();
+const pendingReschedule = new Set();
 // In-memory cache of repo status to avoid flooding the DB read queue
 const repoStatusCache = new Map();
 // Set by startIndexer
@@ -27,28 +45,62 @@ let indexerSignalCollections;
 let indexerPinnedRepos = null;
 let indexerFetchTimeout;
 let indexerMaxRetries;
+let indexerPlcUrl;
+let maxConcurrentBackfills = 3;
+/**
+ * Flush the write buffer — apply all buffered puts and deletes in arrival
+ * order, update the relay cursor, run label rules on inserted records, and
+ * trigger FTS rebuilds when the write threshold is reached. Emits a wide event
+ * with batch stats.
+ */
 async function flushBuffer() {
     if (buffer.length === 0)
         return;
     const elapsed = timer();
     const batch = buffer.splice(0);
     let insertedCount = 0;
+    let deletedCount = 0;
     const errors = [];
     let cursorError;
     const inserted = [];
+    const applied = [];
+    // Strictly sequential: two writes to the same URI in one batch must land in
+    // the order they arrived, so nothing here may run concurrently.
     for (const item of batch) {
         try {
-            await insertRecord(item.collection, item.uri, item.cid, item.authorDid, item.record);
-            insertedCount++;
-            inserted.push(item);
+            if (item.action === 'delete') {
+                await deleteRecord(item.collection, item.uri);
+                deletedCount++;
+                applied.push({
+                    action: 'delete',
+                    collection: item.collection,
+                    uri: item.uri,
+                    authorDid: item.authorDid,
+                    record: null,
+                });
+            }
+            else {
+                await insertRecord(item.collection, item.uri, item.cid, item.authorDid, item.record);
+                insertedCount++;
+                inserted.push(item);
+                applied.push({
+                    action: 'create',
+                    collection: item.collection,
+                    uri: item.uri,
+                    authorDid: item.authorDid,
+                    record: item.record,
+                });
+            }
         }
         catch (err) {
             errors.push(err.message);
         }
     }
     if (lastSeq !== null) {
+        const seq = lastSeq;
         try {
-            await setCursor('relay', String(lastSeq));
+            await setCursor(cursorKey, String(seq));
+            lastPersistedSeq = seq;
         }
         catch (err) {
             cursorError = err.message;
@@ -64,6 +116,9 @@ async function flushBuffer() {
             value: item.record,
         }).catch(() => { });
     }
+    // Fire on-commit hooks for everything the batch applied, in the order it was
+    // applied (async, non-blocking)
+    fireOnCommitHooks(applied);
     // Aggregate collection counts and unique DIDs for wide event
     const collections = {};
     const dids = new Set();
@@ -74,6 +129,7 @@ async function flushBuffer() {
     emit('indexer', 'flush', {
         batch_size: batch.length,
         inserted_count: insertedCount,
+        deleted_count: deletedCount,
         error_count: errors.length,
         cursor_seq: lastSeq,
         duration_ms: elapsed(),
@@ -86,17 +142,39 @@ async function flushBuffer() {
     writesSinceRebuild += batch.length;
     if (writesSinceRebuild >= ftsRebuildInterval) {
         writesSinceRebuild = 0;
-        rebuildAllIndexes([...indexerCollections]).catch(() => { });
+        // Skip periodic full rebuild for SQLite — it uses incremental FTS updates
+        const port = getDatabasePort();
+        if (port.dialect !== 'sqlite') {
+            rebuildAllIndexes([...indexerCollections]).catch(() => { });
+        }
     }
 }
+/**
+ * Run a flush once every flush already queued has finished.
+ *
+ * Nothing awaits the flush a full batch triggers, and the interval timer can
+ * fire while that flush is still waiting on the database. Two flushes in flight
+ * hold disjoint batches but interleave their awaits, so a write in the later
+ * batch can reach the database before one in the earlier batch — which would
+ * give back exactly the cross-batch reordering the single buffer removes.
+ * Chaining them costs nothing on an idle indexer and keeps the queue global.
+ */
+let flushChain = Promise.resolve();
+function enqueueFlush() {
+    const next = flushChain.then(() => flushBuffer(), () => flushBuffer());
+    flushChain = next;
+    return next;
+}
+/** Schedule a flush after FLUSH_INTERVAL_MS if one isn't already pending. */
 function scheduleFlush() {
     if (flushTimer)
         return;
-    flushTimer = setTimeout(async () => {
+    flushTimer = setTimeout(() => {
         flushTimer = null;
-        await flushBuffer();
+        void enqueueFlush().catch(() => { });
     }, FLUSH_INTERVAL_MS);
 }
+/** Add a write to the buffer. Flushes immediately if BATCH_SIZE is reached. */
 function bufferWrite(item) {
     buffer.push(item);
     if (buffer.length >= BATCH_SIZE) {
@@ -104,17 +182,207 @@ function bufferWrite(item) {
             clearTimeout(flushTimer);
             flushTimer = null;
         }
-        flushBuffer();
+        void enqueueFlush().catch(() => { });
     }
     else {
         scheduleFlush();
     }
 }
+/** Record the latest firehose sequence seen. Source of truth for cursor persistence and reconnect resume. */
+export function noteSeq(seq) {
+    lastSeq = seq;
+}
+/** The highest seq this process has seen, or null before the first event. */
+export function getLastSeq() {
+    return lastSeq;
+}
+/**
+ * Persist the latest firehose sequence when it has advanced past the stored
+ * cursor. Runs on a timer independent of the write path: an app whose
+ * collections rarely appear on the firehose never flushes a batch, so the
+ * flush-path cursor write alone leaves the stored cursor frozen — and a frozen
+ * cursor makes the relay replay its whole retention window at line rate
+ * (~135 GB/hour observed against bsky.network) on every boot and reconnect.
+ */
+export async function checkpointCursor() {
+    if (lastSeq === null || lastSeq === lastPersistedSeq)
+        return;
+    const seq = lastSeq;
+    try {
+        await setCursor(cursorKey, String(seq));
+        lastPersistedSeq = seq;
+    }
+    catch (err) {
+        emit('indexer', 'cursor_checkpoint_error', { cursor_seq: seq, error: err.message });
+    }
+}
+/** Point cursor persistence at a stream's own `_cursor` row. See {@link cursorKey}. */
+export function setCursorKey(key) {
+    cursorKey = key;
+}
+/**
+ * The cursor a reconnect should resume from: the latest seq this process has
+ * seen, falling back to the boot-time cursor before any message has arrived.
+ * Reusing the boot-time cursor on every reconnect would replay everything
+ * received since the process started.
+ */
+export function resumeCursor(liveSeq, bootCursor) {
+    return liveSeq !== null ? String(liveSeq) : bootCursor;
+}
+/** Reset module cursor state between tests. */
+export function _resetCursorStateForTests() {
+    lastSeq = null;
+    lastPersistedSeq = null;
+    cursorKey = 'relay';
+}
+/**
+ * Forget which repos this process is tracking.
+ *
+ * The status cache is deliberately process-global — it is what keeps the
+ * commit hot path off the database — and it outlives any table a test
+ * truncates. A suite that clears `_repos` between cases and does not clear
+ * this sees the second case behave as though the first one's repos were
+ * still known, which is how the seam between tracking and backfill went
+ * untested.
+ */
+export function _resetRepoTrackingForTests() {
+    repoStatusCache.clear();
+    backfillInFlight.clear();
+    backfillPromises.clear();
+    pendingReschedule.clear();
+    pendingBuffers.clear();
+}
+/**
+ * Drain the write buffer instead of waiting out FLUSH_INTERVAL_MS. Lets
+ * end-to-end tests assert on rows immediately after feeding a frame — deletes
+ * included, since they are buffered alongside puts and so are covered by the
+ * same await.
+ */
+export async function _flushForTests() {
+    if (flushTimer) {
+        clearTimeout(flushTimer);
+        flushTimer = null;
+    }
+    await enqueueFlush();
+}
+/**
+ * Auto-backfill a DID's repo when first seen on the firehose.
+ *
+ * Fetches the full repo via CAR export, inserts all records, then replays any
+ * firehose events that arrived during the backfill. Concurrency is capped at
+ * `maxConcurrentBackfills`. Failed backfills retry with exponential delay up
+ * to `maxRetries`.
+ */
+/** Wait for a DID's backfill to complete if one is in flight. */
+export function awaitBackfill(did) {
+    const entry = backfillPromises.get(did);
+    return entry ? entry.promise : Promise.resolve();
+}
+/**
+ * Track a repo no signal collection will ever point at.
+ *
+ * Backfill and the stream find repos by what they write into the signal
+ * collections. A writer in a followed space is wanted for another reason:
+ * their public profile is what puts a name and a face on everything they
+ * wrote there, and an app that only indexes the community's own records
+ * would render every member as a DID. So the space engine names them here,
+ * and an unknown one is backfilled like a repo the stream just surfaced.
+ * Idempotent and cheap: a repo already known is a map lookup.
+ */
+let indexerReferences = [];
+/** Follow a dot path into a record; `$rkey` names the record key itself. */
+function referencedDid(record, rkey, field) {
+    if (field === '$rkey')
+        return rkey.startsWith('did:') ? rkey : null;
+    let cur = record;
+    for (const part of field.split('.')) {
+        if (!cur || typeof cur !== 'object')
+            return null;
+        cur = cur[part];
+    }
+    return typeof cur === 'string' && cur.startsWith('did:') ? cur : null;
+}
+/**
+ * Track every repo a record names, per the configured references.
+ *
+ * Called wherever a record enters the index — the stream, a backfill, a
+ * space sync — so a roster read any of those ways brings its members' repos
+ * with it.
+ */
+export function noteReferencedRepos(collection, rkey, record) {
+    for (const ref of indexerReferences) {
+        if (ref.collection !== collection)
+            continue;
+        const did = referencedDid(record, rkey, ref.field);
+        if (did)
+            trackRepo(did);
+    }
+}
+/**
+ * Apply the references to what is already indexed.
+ *
+ * A reference fires as a record comes in. Rows indexed before the reference
+ * was configured — or before this instance had it — would otherwise never
+ * name their repos until the record was written again. Run once at boot;
+ * a table of a few thousand rows is a moment, and the roster is what these
+ * point at, not the firehose.
+ */
+export async function sweepReferences() {
+    let named = 0;
+    for (const ref of indexerReferences) {
+        let rows;
+        try {
+            rows = await unfilteredQuerySQL(`SELECT * FROM "${ref.collection}"`);
+        }
+        catch {
+            continue; // no table for this collection on this instance
+        }
+        for (const raw of rows) {
+            const row = reshapeRow(raw);
+            if (!row)
+                continue;
+            const did = referencedDid(row.value, rkeyOf(row.uri), ref.field);
+            if (!did)
+                continue;
+            named++;
+            trackRepo(did);
+        }
+    }
+    if (named > 0)
+        emit('indexer', 'references_swept', { named });
+    return named;
+}
+export function trackRepo(did) {
+    if (indexerPinnedRepos && !indexerPinnedRepos.has(did))
+        return;
+    const status = repoStatusCache.get(did);
+    if (status && status !== 'unknown')
+        return;
+    repoStatusCache.set(did, 'pending');
+    void triggerAutoBackfill(did);
+}
 export async function triggerAutoBackfill(did, attempt = 0) {
     if (backfillInFlight.has(did))
         return;
+    if (backfillInFlight.size >= maxConcurrentBackfills) {
+        if (!pendingReschedule.has(did)) {
+            pendingReschedule.add(did);
+            setTimeout(() => {
+                pendingReschedule.delete(did);
+                triggerAutoBackfill(did, attempt);
+            }, 10_000);
+        }
+        return;
+    }
     backfillInFlight.add(did);
     pendingBuffers.set(did, []);
+    if (!backfillPromises.has(did)) {
+        let resolveBackfill;
+        const promise = new Promise((r) => {
+            resolveBackfill = r;
+        });
+        backfillPromises.set(did, { promise, resolve: resolveBackfill });
+    }
     if (attempt === 0)
         await setRepoStatus(did, 'pending');
     const elapsed = timer();
@@ -135,7 +403,14 @@ export async function triggerAutoBackfill(did, attempt = 0) {
     backfillInFlight.delete(did);
     for (const item of buffered) {
         try {
-            await insertRecord(item.collection, item.uri, item.cid, item.authorDid, item.record);
+            if (item.action === 'delete') {
+                // A delete that arrived mid-backfill has to be replayed after the CAR
+                // export lands, or the export's snapshot of the record outlives it.
+                await deleteRecord(item.collection, item.uri);
+            }
+            else {
+                await insertRecord(item.collection, item.uri, item.cid, item.authorDid, item.record);
+            }
         }
         catch {
             replayErrors++;
@@ -154,6 +429,12 @@ export async function triggerAutoBackfill(did, attempt = 0) {
         error,
         retry_count: currentRetryCount,
     });
+    // Resolve awaiting callers (e.g. on-login hooks)
+    const entry = backfillPromises.get(did);
+    if (entry) {
+        entry.resolve();
+        backfillPromises.delete(did);
+    }
     if (status === 'error' && currentRetryCount < indexerMaxRetries) {
         const delaySecs = Math.min(currentRetryCount * 60, 3600);
         const delayMs = Math.max(delaySecs, 60) * 1000;
@@ -162,37 +443,27 @@ export async function triggerAutoBackfill(did, attempt = 0) {
         }, delayMs);
     }
 }
-// Periodic memory diagnostics
-function startMemoryDiagnostics() {
-    setInterval(() => {
-        const mem = process.memoryUsage();
-        let pendingBufferItems = 0;
-        for (const [, items] of pendingBuffers) {
-            pendingBufferItems += items.length;
-        }
-        emit('diagnostics', 'memory', {
-            heap_used_mb: Math.round(mem.heapUsed / 1024 / 1024),
-            heap_total_mb: Math.round(mem.heapTotal / 1024 / 1024),
-            rss_mb: Math.round(mem.rss / 1024 / 1024),
-            external_mb: Math.round(mem.external / 1024 / 1024),
-            array_buffers_mb: Math.round(mem.arrayBuffers / 1024 / 1024),
-            write_buffer_len: buffer.length,
-            pending_buffer_dids: pendingBuffers.size,
-            pending_buffer_items: pendingBufferItems,
-            backfill_in_flight: backfillInFlight.size,
-            repo_status_cache_size: repoStatusCache.size,
-        });
-    }, 30_000);
-}
-export async function startIndexer(opts) {
-    const { relayUrl, collections, cursor, fetchTimeout } = opts;
+/**
+ * Connect to the AT Protocol relay firehose and begin indexing.
+ *
+ * Opens a WebSocket to `subscribeRepos`, processes commit messages synchronously
+ * on the event loop to minimize backpressure, and batches writes through
+ * {@link flushBuffer}. New DIDs trigger auto-backfill via {@link triggerAutoBackfill}.
+ * Reconnects automatically on disconnect after a 3s delay.
+ *
+ * @returns The WebSocket connection (for shutdown coordination)
+ */
+export async function configureIndexer(opts) {
     if (opts.ftsRebuildInterval != null)
         ftsRebuildInterval = opts.ftsRebuildInterval;
-    indexerCollections = collections;
-    indexerSignalCollections = opts.signalCollections || collections;
+    indexerCollections = opts.collections;
+    indexerSignalCollections = opts.signalCollections || opts.collections;
+    indexerReferences = opts.references ?? [];
     indexerPinnedRepos = opts.pinnedRepos || null;
-    indexerFetchTimeout = fetchTimeout;
+    indexerFetchTimeout = opts.fetchTimeout;
     indexerMaxRetries = opts.maxRetries;
+    indexerPlcUrl = opts.plcUrl;
+    maxConcurrentBackfills = opts.parallelism ?? 3;
     // Pre-populate repo status cache from DB so non-signal updates
     // (e.g. profile changes) are processed for already-tracked DIDs
     if (repoStatusCache.size === 0) {
@@ -202,7 +473,18 @@ export async function startIndexer(opts) {
         }
         log(`[indexer] Warmed repo status cache with ${statuses.length} entries`);
     }
-    startMemoryDiagnostics();
+    // Checkpoint the cursor on a timer regardless of write activity (see
+    // checkpointCursor). Guarded so reconnects don't stack intervals; unref'd so
+    // it never keeps the process alive.
+    if (!cursorCheckpointTimer) {
+        cursorCheckpointTimer = setInterval(() => void checkpointCursor(), CURSOR_CHECKPOINT_INTERVAL_MS);
+        cursorCheckpointTimer.unref?.();
+    }
+}
+export async function startIndexer(opts) {
+    const { relayUrl, collections, cursor } = opts;
+    setCursorKey('relay');
+    await configureIndexer(opts);
     let wsUrl = `${relayUrl}/xrpc/com.atproto.sync.subscribeRepos`;
     if (cursor) {
         wsUrl += `?cursor=${cursor}`;
@@ -220,41 +502,167 @@ export async function startIndexer(opts) {
             const bytes = new Uint8Array(event.data);
             processMessage(bytes, collections);
         }
-        catch {
-            // Skip unparseable firehose messages silently
+        catch (err) {
+            emit('indexer', 'decode_error', { error: err instanceof Error ? err.message : String(err) });
         }
     });
     ws.addEventListener('open', () => log('[indexer] Connected to relay'));
     ws.addEventListener('close', () => {
         log('[indexer] Disconnected, reconnecting in 3s...');
-        setTimeout(() => startIndexer(opts), 3000);
+        setTimeout(() => startIndexer({ ...opts, cursor: resumeCursor(lastSeq, opts.cursor) }), 3000);
     });
     return ws;
 }
-function processMessage(bytes, collections) {
-    const header = cborDecode(bytes, 0);
-    const body = cborDecode(bytes, header.offset);
-    if (header.value.op !== 1 || header.value.t !== '#commit')
+/** The `_cursor` row an auxiliary firehose persists its position to. */
+export function auxCursorKey(relayUrl) {
+    return `relay:${relayUrl}`;
+}
+/**
+ * Tail a second `subscribeRepos` alongside the primary stream.
+ *
+ * A relay is one coordinate system; a PDS tailed directly is another. Nothing
+ * below the wire cares which socket a frame arrived on — `processMessage`
+ * decodes it and `applyCommit` indexes it — so the only state an extra source
+ * needs of its own is a seq and a cursor row. Both live in this closure,
+ * keyed by URL, so an aux stream never advances (or resumes from) the primary
+ * cursor. Use this when a repo's PDS is not behind the relay being tailed —
+ * a self-hosted network, a dev stack with more than one PDS — rather than
+ * standing up a relay just to merge two streams.
+ *
+ * Must be called after {@link configureIndexer} (or {@link startIndexer}),
+ * which owns the shared indexer configuration.
+ */
+export function startAuxIndexer(opts) {
+    const { relayUrl, collections } = opts;
+    const cursorKey = auxCursorKey(relayUrl);
+    let seq = null;
+    let persistedSeq = null;
+    const checkpoint = async () => {
+        if (seq === null || seq === persistedSeq)
+            return;
+        const s = seq;
+        try {
+            await setCursor(cursorKey, String(s));
+            persistedSeq = s;
+        }
+        catch (err) {
+            emit('indexer', 'cursor_checkpoint_error', { source: relayUrl, cursor_seq: s, error: err.message });
+        }
+    };
+    const timer = setInterval(() => void checkpoint(), CURSOR_CHECKPOINT_INTERVAL_MS);
+    timer.unref?.();
+    const connect = (cursor) => {
+        let wsUrl = `${relayUrl}/xrpc/com.atproto.sync.subscribeRepos`;
+        if (cursor) {
+            wsUrl += `?cursor=${cursor}`;
+            log(`[indexer:aux] Resuming ${relayUrl} from cursor ${cursor}`);
+        }
+        log(`[indexer:aux] Connecting to ${relayUrl}...`);
+        const ws = new WebSocket(wsUrl);
+        ws.binaryType = 'arraybuffer';
+        ws.addEventListener('message', (event) => {
+            try {
+                if (!(event.data instanceof ArrayBuffer))
+                    return;
+                processMessage(new Uint8Array(event.data), collections, (s) => {
+                    seq = s;
+                });
+            }
+            catch (err) {
+                emit('indexer', 'decode_error', { source: relayUrl, error: err instanceof Error ? err.message : String(err) });
+            }
+        });
+        ws.addEventListener('open', () => log(`[indexer:aux] Connected to ${relayUrl}`));
+        ws.addEventListener('close', () => {
+            log(`[indexer:aux] Disconnected from ${relayUrl}, reconnecting in 3s...`);
+            setTimeout(() => connect(resumeCursor(seq, opts.cursor)), 3000);
+        });
+        return ws;
+    };
+    return connect(opts.cursor);
+}
+/**
+ * Handle a `#identity` firehose event for a DID. The `handle` field on the
+ * event is optional per the lexicon, and some emitters omit it (signalling
+ * "re-resolve"). When absent, we re-resolve from the PLC directory so handle
+ * renames propagate even when the relay payload is sparse.
+ *
+ * Only updates DIDs we already track (present in repoStatusCache) to avoid
+ * writing rows for the entire network.
+ */
+export async function handleIdentityEvent(did, payloadHandle) {
+    if (!repoStatusCache.has(did))
         return;
-    if (!body.value.blocks || !body.value.ops)
+    let handle = payloadHandle;
+    const payloadHadHandle = handle !== undefined;
+    if (!handle) {
+        try {
+            // Bound the PLC fetch so a slow plc.directory can't pile up unbounded
+            // promises during an identity-event burst (fire-and-forget caller).
+            const res = await fetch(`${indexerPlcUrl}/${did}`, {
+                signal: AbortSignal.timeout(indexerFetchTimeout * 1000),
+            });
+            if (res.ok) {
+                const doc = (await res.json());
+                // First at:// entry is the canonical handle (per @atproto/identity convention)
+                const aka = doc.alsoKnownAs?.find((u) => u.startsWith('at://'));
+                handle = aka ? aka.slice('at://'.length) : undefined;
+            }
+            else {
+                emit('indexer', 'identity_resolve_error', { did, status: res.status });
+            }
+        }
+        catch (err) {
+            emit('indexer', 'identity_resolve_error', {
+                did,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    if (!handle) {
+        emit('indexer', 'identity_no_handle', { did, payload_had_handle: payloadHadHandle });
         return;
-    // Track sequence number for cursor
-    if (body.value.seq)
-        lastSeq = body.value.seq;
-    const did = body.value.repo;
-    if (!did)
+    }
+    try {
+        await updateRepoHandle(did, handle);
+        emit('indexer', 'identity_handle_update', { did, handle, payload_had_handle: payloadHadHandle });
+    }
+    catch (err) {
+        emit('indexer', 'identity_update_error', {
+            did,
+            handle,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+}
+/**
+ * Whether a collection's records may be written from network data.
+ *
+ * Private collections are AppView-authoritative: nothing on the network may
+ * write or delete their rows, so stream ops naming them are spoofed by
+ * definition and must be dropped regardless of which source delivered them.
+ */
+export function isIndexableCollection(collection, collections) {
+    return collections.has(collection) && !isPrivateCollection(collection);
+}
+/**
+ * Apply a repo's commit ops to the index.
+ *
+ * Wire-agnostic: the relay path reaches here after CBOR/CAR decoding, the
+ * Jetstream path after parsing JSON. Callers must have already filtered `ops`
+ * through {@link isIndexableCollection}.
+ *
+ * Handles auto-backfill signalling, buffering for DIDs mid-backfill, lexicon
+ * validation, and delete/put routing.
+ */
+export function applyCommit(did, ops) {
+    if (ops.length === 0)
         return;
     // When repos are pinned, only process events from those DIDs
     if (indexerPinnedRepos && !indexerPinnedRepos.has(did))
         return;
-    // Check if any ops in this commit are for collections we care about
-    const relevantOps = body.value.ops.filter((op) => collections.has(op.path.split('/')[0]));
-    if (relevantOps.length === 0)
-        return;
-    // Copy blocks out of the original buffer before it can be GC'd
-    const { blocks } = parseCarFrame(new Uint8Array(body.value.blocks));
     // Only auto-backfill when we see activity in a signal collection
-    const hasSignalOp = relevantOps.some((op) => indexerSignalCollections.has(op.path.split('/')[0]));
+    const hasSignalOp = ops.some((op) => indexerSignalCollections.has(op.collection));
     // Use in-memory cache only — never hit DB from the hot path.
     // Unknown DIDs stay unknown until backfill or auto-backfill discovers them.
     // The cache is populated by triggerAutoBackfill and setRepoStatus calls.
@@ -263,8 +671,8 @@ function processMessage(bytes, collections) {
     if (cachedStatus === undefined) {
         repoStatusCache.set(did, 'unknown');
     }
-    if (hasSignalOp && (!indexerPinnedRepos || indexerPinnedRepos.has(did))) {
-        if (repoStatus === null && backfillInFlight.size < MAX_CONCURRENT_BACKFILLS) {
+    if (hasSignalOp) {
+        if (repoStatus === null && backfillInFlight.size < maxConcurrentBackfills) {
             repoStatusCache.set(did, 'pending');
             triggerAutoBackfill(did);
         }
@@ -274,43 +682,97 @@ function processMessage(bytes, collections) {
         }
     }
     // For non-signal ops (e.g. profile updates), only process if this DID is already tracked
-    if (!hasSignalOp) {
-        if (repoStatus === null)
-            return;
+    if (!hasSignalOp && repoStatus === null)
+        return;
+    /** Queue a write behind anything already queued for this DID. */
+    const enqueue = (item) => {
+        // If DID is mid-backfill, buffer instead of writing directly
+        if (pendingBuffers.has(did))
+            pendingBuffers.get(did).push(item);
+        else
+            bufferWrite(item);
+    };
+    for (const op of ops) {
+        const uri = `at://${did}/${op.collection}/${op.rkey}`;
+        if (op.action === 'delete') {
+            // Buffered, not applied here: see {@link WriteBuffer}. The on-commit hook
+            // fires from the flush too, so a handler never sees a delete announced
+            // before the row is actually gone.
+            enqueue({ action: 'delete', collection: op.collection, uri, authorDid: did });
+            continue;
+        }
+        const record = op.record;
+        if (!op.cid || !record)
+            continue;
+        if (record.$type !== op.collection)
+            continue;
+        const validationError = validateRecord(getLexiconArray(), op.collection, record);
+        if (validationError) {
+            emit('indexer', 'validation_skip', {
+                uri,
+                collection: op.collection,
+                path: validationError.path,
+                error: validationError.message,
+            });
+            continue;
+        }
+        noteReferencedRepos(op.collection, op.rkey, record);
+        enqueue({ action: 'put', collection: op.collection, uri, cid: op.cid, authorDid: did, record });
     }
+}
+/**
+ * Process a single firehose message. Decodes the CBOR header/body, filters
+ * for relevant collections, validates records against lexicons, and routes
+ * writes to the buffer (or pending buffer if the DID is mid-backfill).
+ */
+export function processMessage(bytes, collections, onSeq = noteSeq) {
+    const header = cborDecode(bytes, 0);
+    const body = cborDecode(bytes, header.offset);
+    // Handle identity events (handle changes). Fire-and-forget — keeps
+    // processMessage synchronous so the WS event loop drains without backpressure.
+    if (header.value.t === '#identity') {
+        const did = typeof body.value.did === 'string' ? body.value.did : undefined;
+        const handle = typeof body.value.handle === 'string' ? body.value.handle : undefined;
+        if (did)
+            handleIdentityEvent(did, handle);
+        return;
+    }
+    if (header.value.op !== 1 || header.value.t !== '#commit')
+        return;
+    if (!body.value.blocks || !body.value.ops)
+        return;
+    // Track sequence number for cursor
+    if (body.value.seq)
+        onSeq(body.value.seq);
+    const did = body.value.repo;
+    if (!did)
+        return;
+    // Check if any ops in this commit are for collections we care about, before
+    // paying for the CAR parse below.
+    const relevantOps = body.value.ops.filter((op) => isIndexableCollection(op.path.split('/')[0], collections));
+    if (relevantOps.length === 0)
+        return;
+    // Copy blocks out of the original buffer before it can be GC'd
+    const { blocks } = parseCarFrame(new Uint8Array(body.value.blocks));
+    const ops = [];
     for (const op of relevantOps) {
         const collection = op.path.split('/')[0];
-        const uri = `at://${did}/${op.path}`;
+        const rkey = op.path.split('/').slice(1).join('/');
         if (op.action === 'delete') {
-            deleteRecord(collection, uri);
+            ops.push({ action: 'delete', collection, rkey });
             continue;
         }
-        for (const [cid, data] of blocks) {
-            try {
-                const { value: record } = cborDecode(data);
-                if (record?.$type === collection) {
-                    const validationError = validateRecord(getLexiconArray(), collection, record);
-                    if (validationError) {
-                        emit('indexer', 'validation_skip', {
-                            uri,
-                            collection,
-                            path: validationError.path,
-                            error: validationError.message,
-                        });
-                        break;
-                    }
-                    const item = { collection, uri, cid, authorDid: did, record };
-                    // If DID is mid-backfill, buffer instead of writing directly
-                    if (pendingBuffers.has(did)) {
-                        pendingBuffers.get(did).push(item);
-                    }
-                    else {
-                        bufferWrite(item);
-                    }
-                    break;
-                }
-            }
-            catch { }
+        const opCid = typeof op.cid === 'string' ? op.cid : op.cid?.$link;
+        if (!opCid)
+            continue;
+        const data = blocks.get(opCid);
+        if (!data)
+            continue;
+        try {
+            const { value: record } = cborDecode(data);
+            ops.push({ action: op.action === 'update' ? 'update' : 'create', collection, rkey, cid: opCid, record });
         }
+        catch { }
     }
+    applyCommit(did, ops);
 }