@hatk/hatk 0.0.1-alpha.11 → 0.0.1-alpha.111

package/dist/indexer.d.ts

@@ -1,15 +1,206 @@
+import type { RepoReference } from './config.ts';
+/** A single normalized repo operation, independent of the wire it arrived on. */
+export interface CommitOp {
+    action: 'create' | 'update' | 'delete';
+    collection: string;
+    rkey: string;
+    /** Absent on deletes. */
+    cid?: string;
+    /** Absent on deletes. */
+    record?: Record<string, any>;
+}
+/** Record the latest firehose sequence seen. Source of truth for cursor persistence and reconnect resume. */
+export declare function noteSeq(seq: number): void;
+/** The highest seq this process has seen, or null before the first event. */
+export declare function getLastSeq(): number | null;
+/**
+ * Persist the latest firehose sequence when it has advanced past the stored
+ * cursor. Runs on a timer independent of the write path: an app whose
+ * collections rarely appear on the firehose never flushes a batch, so the
+ * flush-path cursor write alone leaves the stored cursor frozen — and a frozen
+ * cursor makes the relay replay its whole retention window at line rate
+ * (~135 GB/hour observed against bsky.network) on every boot and reconnect.
+ */
+export declare function checkpointCursor(): Promise<void>;
+/** Point cursor persistence at a stream's own `_cursor` row. See {@link cursorKey}. */
+export declare function setCursorKey(key: string): void;
+/**
+ * The cursor a reconnect should resume from: the latest seq this process has
+ * seen, falling back to the boot-time cursor before any message has arrived.
+ * Reusing the boot-time cursor on every reconnect would replay everything
+ * received since the process started.
+ */
+export declare function resumeCursor(liveSeq: number | null, bootCursor: string | null | undefined): string | null | undefined;
+/** Reset module cursor state between tests. */
+export declare function _resetCursorStateForTests(): void;
+/**
+ * Forget which repos this process is tracking.
+ *
+ * The status cache is deliberately process-global — it is what keeps the
+ * commit hot path off the database — and it outlives any table a test
+ * truncates. A suite that clears `_repos` between cases and does not clear
+ * this sees the second case behave as though the first one's repos were
+ * still known, which is how the seam between tracking and backfill went
+ * untested.
+ */
+export declare function _resetRepoTrackingForTests(): void;
+/**
+ * Drain the write buffer instead of waiting out FLUSH_INTERVAL_MS. Lets
+ * end-to-end tests assert on rows immediately after feeding a frame — deletes
+ * included, since they are buffered alongside puts and so are covered by the
+ * same await.
+ */
+export declare function _flushForTests(): Promise<void>;
+/**
+ * Auto-backfill a DID's repo when first seen on the firehose.
+ *
+ * Fetches the full repo via CAR export, inserts all records, then replays any
+ * firehose events that arrived during the backfill. Concurrency is capped at
+ * `maxConcurrentBackfills`. Failed backfills retry with exponential delay up
+ * to `maxRetries`.
+ */
+/** Wait for a DID's backfill to complete if one is in flight. */
+export declare function awaitBackfill(did: string): Promise<void>;
+/**
+ * Track every repo a record names, per the configured references.
+ *
+ * Called wherever a record enters the index — the stream, a backfill, a
+ * space sync — so a roster read any of those ways brings its members' repos
+ * with it.
+ */
+export declare function noteReferencedRepos(collection: string, rkey: string, record: Record<string, unknown>): void;
+/**
+ * Apply the references to what is already indexed.
+ *
+ * A reference fires as a record comes in. Rows indexed before the reference
+ * was configured — or before this instance had it — would otherwise never
+ * name their repos until the record was written again. Run once at boot;
+ * a table of a few thousand rows is a moment, and the roster is what these
+ * point at, not the firehose.
+ */
+export declare function sweepReferences(): Promise<number>;
+export declare function trackRepo(did: string): void;
 export declare function triggerAutoBackfill(did: string, attempt?: number): Promise<void>;
-interface IndexerOpts {
-    relayUrl: string;
+/**
+ * Indexing behaviour shared by every stream source. Both the relay firehose
+ * and the Jetstream live tail feed the same buffers, backfill signalling, and
+ * cursor machinery — only the wire differs.
+ */
+export interface IndexerCoreOpts {
+    /** Records that name repos to track; see {@link RepoReference}. */
+    references?: RepoReference[];
+    plcUrl: string;
     collections: Set<string>;
     signalCollections?: Set<string>;
     pinnedRepos?: Set<string>;
-    cursor?: string | null;
     fetchTimeout: number;
     maxRetries: number;
     parallelism?: number;
     ftsRebuildInterval?: number;
 }
+/** Configuration for the relay firehose indexer. */
+interface IndexerOpts extends IndexerCoreOpts {
+    relayUrl: string;
+    cursor?: string | null;
+}
+/**
+ * Connect to the AT Protocol relay firehose and begin indexing.
+ *
+ * Opens a WebSocket to `subscribeRepos`, processes commit messages synchronously
+ * on the event loop to minimize backpressure, and batches writes through
+ * {@link flushBuffer}. New DIDs trigger auto-backfill via {@link triggerAutoBackfill}.
+ * Reconnects automatically on disconnect after a 3s delay.
+ *
+ * @returns The WebSocket connection (for shutdown coordination)
+ */
+export declare function configureIndexer(opts: IndexerCoreOpts): Promise<void>;
 export declare function startIndexer(opts: IndexerOpts): Promise<WebSocket>;
+/** Configuration for an auxiliary firehose (see {@link startAuxIndexer}). */
+export interface AuxIndexerOpts {
+    relayUrl: string;
+    collections: Set<string>;
+    cursor?: string | null;
+}
+/** The `_cursor` row an auxiliary firehose persists its position to. */
+export declare function auxCursorKey(relayUrl: string): string;
+/**
+ * The `_cursor` row a Jetstream instance persists its position to.
+ *
+ * Named for the instance, because a Jetstream sequence means nothing on
+ * another one. Pointed at a second instance — a region swap, an outage
+ * worked around — a shared row would hand over a number from somewhere else,
+ * and the stream would resume at whatever that happens to address there:
+ * no error, no refusal, just a silent jump past whatever fell in between.
+ */
+export declare function jetstreamCursorKey(jetstreamUrl: string): string;
+/**
+ * Tail a second `subscribeRepos` alongside the primary stream.
+ *
+ * A relay is one coordinate system; a PDS tailed directly is another. Nothing
+ * below the wire cares which socket a frame arrived on — `processMessage`
+ * decodes it and `applyCommit` indexes it — so the only state an extra source
+ * needs of its own is a seq and a cursor row. Both live in this closure,
+ * keyed by URL, so an aux stream never advances (or resumes from) the primary
+ * cursor. Use this when a repo's PDS is not behind the relay being tailed —
+ * a self-hosted network, a dev stack with more than one PDS — rather than
+ * standing up a relay just to merge two streams.
+ *
+ * Must be called after {@link configureIndexer} (or {@link startIndexer}),
+ * which owns the shared indexer configuration.
+ */
+export declare function startAuxIndexer(opts: AuxIndexerOpts): WebSocket;
+/**
+ * Handle a `#identity` firehose event for a DID. The `handle` field on the
+ * event is optional per the lexicon, and some emitters omit it (signalling
+ * "re-resolve"). When absent, we re-resolve from the PLC directory so handle
+ * renames propagate even when the relay payload is sparse.
+ *
+ * Only updates DIDs we already track (present in repoStatusCache) to avoid
+ * writing rows for the entire network.
+ */
+export declare function handleIdentityEvent(did: string, payloadHandle: string | undefined): Promise<void>;
+/**
+ * Handle an `#account` event: the network saying an account is active again,
+ * or is not — deactivated, suspended, taken down by its host, or deleted.
+ *
+ * - deleted: every record it holds here is removed, and it is marked
+ *   `deleted`.
+ * - deactivated, suspended, taken down by its host: marked `deactivated`. Its
+ *   records stay, and no read serves them (see HIDDEN_REPO_STATUSES), so an
+ *   account that comes back is whole again at once.
+ * - active again, after either: marked `pending` and backfilled, since it may
+ *   have changed while it was gone.
+ *
+ * An admin's own takedown here wins over all of it: only reverse-takedown
+ * lifts that. Statuses that say nothing about the account itself (a relay's
+ * `desynchronized` or `throttled`) change nothing. Like identity events, only
+ * DIDs already tracked are touched.
+ */
+export declare function handleAccountEvent(did: string, active: boolean, status: string | undefined): Promise<void>;
+/**
+ * Whether a collection's records may be written from network data.
+ *
+ * Private collections are AppView-authoritative: nothing on the network may
+ * write or delete their rows, so stream ops naming them are spoofed by
+ * definition and must be dropped regardless of which source delivered them.
+ */
+export declare function isIndexableCollection(collection: string, collections: Set<string>): boolean;
+/**
+ * Apply a repo's commit ops to the index.
+ *
+ * Wire-agnostic: the relay path reaches here after CBOR/CAR decoding, the
+ * Jetstream path after parsing JSON. Callers must have already filtered `ops`
+ * through {@link isIndexableCollection}.
+ *
+ * Handles auto-backfill signalling, buffering for DIDs mid-backfill, lexicon
+ * validation, and delete/put routing.
+ */
+export declare function applyCommit(did: string, ops: CommitOp[]): void;
+/**
+ * Process a single firehose message. Decodes the CBOR header/body, filters
+ * for relevant collections, validates records against lexicons, and routes
+ * writes to the buffer (or pending buffer if the DID is mid-backfill).
+ */
+export declare function processMessage(bytes: Uint8Array, collections: Set<string>, onSeq?: (seq: number) => void): void;
 export {};
 //# sourceMappingURL=indexer.d.ts.map

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAmIA,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAgEjF;AAED,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAmDxE"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAkDhD,iFAAiF;AACjF,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACtC,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC7B;AAoMD,6GAA6G;AAC7G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAEzC;AAED,6EAA6E;AAC7E,wBAAgB,UAAU,IAAI,MAAM,GAAG,IAAI,CAE1C;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAStD;AAED,uFAAuF;AACvF,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAErH;AAED,+CAA+C;AAC/C,wBAAgB,yBAAyB,IAAI,IAAI,CAIhD;AAED;;;;;;;;;GASG;AACH,wBAAgB,0BAA0B,IAAI,IAAI,CAMjD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAMpD;AAED;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AA0BD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAM3G;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC,CAoBvD;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAM3C;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAuFjF;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,mEAAmE;IACnE,UAAU,CAAC,EAAE,aAAa,EAAE,CAAA;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,oDAAoD;AACpD,UAAU,WAAY,SAAQ,eAAe;IAC3C,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CA4B3E;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkCxE;AAED,6EAA6E;AAC7E,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,wEAAwE;AACxE,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAE/D;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,cAAc,GAAG,SAAS,CAgD/D;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CA4CvG;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAgChH;AAKD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAE3F;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,IAAI,CAmE9D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,UAAU,EACjB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EACxB,KAAK,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAc,GACrC,IAAI,CA6DN"}