@hatk/hatk 0.0.1-alpha.70 → 0.0.1-alpha.71

package/dist/config.d.ts

@@ -45,8 +45,21 @@ export interface CdnConfig {
     key: string;
     salt: string;
 }
+export interface JetstreamConfig {
+    /** Instance base URL, e.g. `wss://jetstream.us-east.bsky.network`. */
+    url: string;
+}
 export interface HatkConfig {
     relay: string;
+    /**
+     * Consume the stream from a Jetstream v2 instance instead of `relay`.
+     *
+     * Jetstream filters server-side and delivers records as decoded JSON, so an
+     * AppView tracking a few collections stops paying to decode the whole
+     * network. Not every deployment has one — a local PDS or self-hosted relay
+     * won't — so `relay` stays the default.
+     */
+    jetstream: JetstreamConfig | null;
     plc: string;
     port: number;
     cdn: CdnConfig | null;
@@ -62,13 +75,14 @@ export interface HatkConfig {
     admins: string[];
 }
 /** Input type for defineConfig — fields that have defaults are optional. */
-export type HatkConfigInput = Partial<Omit<HatkConfig, 'oauth' | 'backfill' | 'push' | 'cdn'>> & {
+export type HatkConfigInput = Partial<Omit<HatkConfig, 'oauth' | 'backfill' | 'push' | 'cdn' | 'jetstream'>> & {
     cdn?: CdnConfig | null;
     oauth?: (Partial<OAuthConfig> & {
         clients: OAuthClientConfig[];
     }) | null;
     backfill?: Partial<BackfillConfig>;
     push?: PushConfig | null;
+    jetstream?: JetstreamConfig | null;
 };
 /** Identity function that provides type inference for hatk config files. */
 export declare function defineConfig(config: HatkConfigInput): HatkConfigInput;

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAA;IACrC,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;IACnC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC1C,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,iBAAiB,EAAE,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,EAAE,OAAO,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,cAAc,CAAA;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,SAAS,GAAG,IAAI,CAAA;IACrB,cAAc,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,kBAAkB,EAAE,MAAM,EAAE,CAAA;IAC5B,QAAQ,EAAE,cAAc,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,KAAK,EAAE,WAAW,GAAG,IAAI,CAAA;IACzB,IAAI,EAAE,UAAU,GAAG,IAAI,CAAA;IACvB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,4EAA4E;AAC5E,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,CAAC,CAAC,GAAG;IAC/F,GAAG,CAAC,EAAE,SAAS,GAAG,IAAI,CAAA;IACtB,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG;QAAE,OAAO,EAAE,iBAAiB,EAAE,CAAA;KAAE,CAAC,GAAG,IAAI,CAAA;IACxE,QAAQ,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAA;IAClC,IAAI,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;CACzB,CAAA;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,eAAe,CAErE;AAED,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CA8DxE"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAA;IACrC,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAA;IACnC,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC1C,OAAO,CAAC,EAAE,WAAW,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,iBAAiB,EAAE,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,EAAE,OAAO,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,OAAO,CAAA;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,cAAc,CAAA;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,sEAAsE;IACtE,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAA;IACb;;;;;;;OAOG;IACH,SAAS,EAAE,eAAe,GAAG,IAAI,CAAA;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,SAAS,GAAG,IAAI,CAAA;IACrB,cAAc,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,kBAAkB,EAAE,MAAM,EAAE,CAAA;IAC5B,QAAQ,EAAE,cAAc,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,KAAK,EAAE,WAAW,GAAG,IAAI,CAAA;IACzB,IAAI,EAAE,UAAU,GAAG,IAAI,CAAA;IACvB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,4EAA4E;AAC5E,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,WAAW,CAAC,CAAC,GAAG;IAC7G,GAAG,CAAC,EAAE,SAAS,GAAG,IAAI,CAAA;IACtB,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG;QAAE,OAAO,EAAE,iBAAiB,EAAE,CAAA;KAAE,CAAC,GAAG,IAAI,CAAA;IACxE,QAAQ,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAA;IAClC,IAAI,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;IACxB,SAAS,CAAC,EAAE,eAAe,GAAG,IAAI,CAAA;CACnC,CAAA;AAED,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,eAAe,CAErE;AAED,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CA+DxE"}

package/dist/config.js

@@ -40,6 +40,7 @@ export async function loadConfig(configPath) {
     const database = env.DATABASE || parsed.database;
     const config = {
         relay: env.RELAY || parsed.relay || 'ws://localhost:2583',
+        jetstream: env.JETSTREAM_URL ? { url: env.JETSTREAM_URL } : parsed.jetstream || null,
         plc: env.DID_PLC_URL || parsed.plc || 'https://plc.directory',
         port: parseInt(env.PORT || '') || parsed.port || 3000,
         databaseEngine: (env.DATABASE_ENGINE || parsed.databaseEngine || 'sqlite'),

package/dist/indexer.d.ts

@@ -1,5 +1,17 @@
+/** A single normalized repo operation, independent of the wire it arrived on. */
+export interface CommitOp {
+    action: 'create' | 'update' | 'delete';
+    collection: string;
+    rkey: string;
+    /** Absent on deletes. */
+    cid?: string;
+    /** Absent on deletes. */
+    record?: Record<string, any>;
+}
 /** Record the latest firehose sequence seen. Source of truth for cursor persistence and reconnect resume. */
 export declare function noteSeq(seq: number): void;
+/** The highest seq this process has seen, or null before the first event. */
+export declare function getLastSeq(): number | null;
 /**
  * Persist the latest firehose sequence when it has advanced past the stored
  * cursor. Runs on a timer independent of the write path: an app whose
@@ -9,6 +21,8 @@ export declare function noteSeq(seq: number): void;
  * (~135 GB/hour observed against bsky.network) on every boot and reconnect.
  */
 export declare function checkpointCursor(): Promise<void>;
+/** Point cursor persistence at a stream's own `_cursor` row. See {@link cursorKey}. */
+export declare function setCursorKey(key: 'relay' | 'jetstream'): void;
 /**
  * The cursor a reconnect should resume from: the latest seq this process has
  * seen, falling back to the boot-time cursor before any message has arrived.
@@ -18,6 +32,12 @@ export declare function checkpointCursor(): Promise<void>;
 export declare function resumeCursor(liveSeq: number | null, bootCursor: string | null | undefined): string | null | undefined;
 /** Reset module cursor state between tests. */
 export declare function _resetCursorStateForTests(): void;
+/**
+ * Drain the write buffer synchronously instead of waiting out
+ * FLUSH_INTERVAL_MS. Lets end-to-end tests assert on rows immediately after
+ * feeding a frame.
+ */
+export declare function _flushForTests(): Promise<void>;
 /**
  * Auto-backfill a DID's repo when first seen on the firehose.
  *
@@ -29,19 +49,26 @@ export declare function _resetCursorStateForTests(): void;
 /** Wait for a DID's backfill to complete if one is in flight. */
 export declare function awaitBackfill(did: string): Promise<void>;
 export declare function triggerAutoBackfill(did: string, attempt?: number): Promise<void>;
-/** Configuration for the firehose indexer. */
-interface IndexerOpts {
-    relayUrl: string;
+/**
+ * Indexing behaviour shared by every stream source. Both the relay firehose
+ * and the Jetstream live tail feed the same buffers, backfill signalling, and
+ * cursor machinery — only the wire differs.
+ */
+export interface IndexerCoreOpts {
     plcUrl: string;
     collections: Set<string>;
     signalCollections?: Set<string>;
     pinnedRepos?: Set<string>;
-    cursor?: string | null;
     fetchTimeout: number;
     maxRetries: number;
     parallelism?: number;
     ftsRebuildInterval?: number;
 }
+/** Configuration for the relay firehose indexer. */
+interface IndexerOpts extends IndexerCoreOpts {
+    relayUrl: string;
+    cursor?: string | null;
+}
 /**
  * Connect to the AT Protocol relay firehose and begin indexing.
  *
@@ -52,6 +79,42 @@ interface IndexerOpts {
  *
  * @returns The WebSocket connection (for shutdown coordination)
  */
+export declare function configureIndexer(opts: IndexerCoreOpts): Promise<void>;
 export declare function startIndexer(opts: IndexerOpts): Promise<WebSocket>;
+/**
+ * Handle a `#identity` firehose event for a DID. The `handle` field on the
+ * event is optional per the lexicon, and some emitters omit it (signalling
+ * "re-resolve"). When absent, we re-resolve from the PLC directory so handle
+ * renames propagate even when the relay payload is sparse.
+ *
+ * Only updates DIDs we already track (present in repoStatusCache) to avoid
+ * writing rows for the entire network.
+ */
+export declare function handleIdentityEvent(did: string, payloadHandle: string | undefined): Promise<void>;
+/**
+ * Whether a collection's records may be written from network data.
+ *
+ * Private collections are AppView-authoritative: nothing on the network may
+ * write or delete their rows, so stream ops naming them are spoofed by
+ * definition and must be dropped regardless of which source delivered them.
+ */
+export declare function isIndexableCollection(collection: string, collections: Set<string>): boolean;
+/**
+ * Apply a repo's commit ops to the index.
+ *
+ * Wire-agnostic: the relay path reaches here after CBOR/CAR decoding, the
+ * Jetstream path after parsing JSON. Callers must have already filtered `ops`
+ * through {@link isIndexableCollection}.
+ *
+ * Handles auto-backfill signalling, buffering for DIDs mid-backfill, lexicon
+ * validation, and delete/put routing.
+ */
+export declare function applyCommit(did: string, ops: CommitOp[]): void;
+/**
+ * Process a single firehose message. Decodes the CBOR header/body, filters
+ * for relevant collections, validates records against lexicons, and routes
+ * writes to the buffer (or pending buffer if the DID is mid-backfill).
+ */
+export declare function processMessage(bytes: Uint8Array, collections: Set<string>): void;
 export {};
 //# sourceMappingURL=indexer.d.ts.map

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AA4KA,6GAA6G;AAC7G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAEzC;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAStD;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,MAAM,GAAG,IAAI,EACtB,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GACpC,MAAM,GAAG,IAAI,GAAG,SAAS,CAE3B;AAED,+CAA+C;AAC/C,wBAAgB,yBAAyB,IAAI,IAAI,CAGhD;AAED;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CA8EjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CA4DxE"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AA8BA,iFAAiF;AACjF,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACtC,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC7B;AA0JD,6GAA6G;AAC7G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAEzC;AAED,6EAA6E;AAC7E,wBAAgB,UAAU,IAAI,MAAM,GAAG,IAAI,CAE1C;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAStD;AAED,uFAAuF;AACvF,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,GAAG,WAAW,GAAG,IAAI,CAE7D;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAErH;AAED,+CAA+C;AAC/C,wBAAgB,yBAAyB,IAAI,IAAI,CAIhD;AAED;;;;GAIG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAMpD;AAED;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CA8EjF;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,oDAAoD;AACpD,UAAU,WAAY,SAAQ,eAAe;IAC3C,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CA6B3E;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAkCxE;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CA4CvG;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAE3F;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,IAAI,CAwE9D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAoDhF"}

package/dist/indexer.js

@@ -14,6 +14,15 @@ let flushTimer = null;
 let lastSeq = null;
 let lastPersistedSeq = null;
 let cursorCheckpointTimer = null;
+/**
+ * Which `_cursor` row this process persists its stream position to.
+ *
+ * The relay's `subscribeRepos` seq and Jetstream's seq are different
+ * coordinate systems — resuming one from the other's value would either skip
+ * a swathe of the stream or replay from a nonsense offset. Each source owns
+ * its own key so switching between them (or back) is safe.
+ */
+let cursorKey = 'relay';
 const BATCH_SIZE = 100;
 const FLUSH_INTERVAL_MS = 500;
 const CURSOR_CHECKPOINT_INTERVAL_MS = 5_000;
@@ -62,7 +71,7 @@ async function flushBuffer() {
     if (lastSeq !== null) {
         const seq = lastSeq;
         try {
-            await setCursor('relay', String(seq));
+            await setCursor(cursorKey, String(seq));
             lastPersistedSeq = seq;
         }
         catch (err) {
@@ -143,6 +152,10 @@ function bufferWrite(item) {
 export function noteSeq(seq) {
     lastSeq = seq;
 }
+/** The highest seq this process has seen, or null before the first event. */
+export function getLastSeq() {
+    return lastSeq;
+}
 /**
  * Persist the latest firehose sequence when it has advanced past the stored
  * cursor. Runs on a timer independent of the write path: an app whose
@@ -156,13 +169,17 @@ export async function checkpointCursor() {
         return;
     const seq = lastSeq;
     try {
-        await setCursor('relay', String(seq));
+        await setCursor(cursorKey, String(seq));
         lastPersistedSeq = seq;
     }
     catch (err) {
         emit('indexer', 'cursor_checkpoint_error', { cursor_seq: seq, error: err.message });
     }
 }
+/** Point cursor persistence at a stream's own `_cursor` row. See {@link cursorKey}. */
+export function setCursorKey(key) {
+    cursorKey = key;
+}
 /**
  * The cursor a reconnect should resume from: the latest seq this process has
  * seen, falling back to the boot-time cursor before any message has arrived.
@@ -176,6 +193,19 @@ export function resumeCursor(liveSeq, bootCursor) {
 export function _resetCursorStateForTests() {
     lastSeq = null;
     lastPersistedSeq = null;
+    cursorKey = 'relay';
+}
+/**
+ * Drain the write buffer synchronously instead of waiting out
+ * FLUSH_INTERVAL_MS. Lets end-to-end tests assert on rows immediately after
+ * feeding a frame.
+ */
+export async function _flushForTests() {
+    if (flushTimer) {
+        clearTimeout(flushTimer);
+        flushTimer = null;
+    }
+    await flushBuffer();
 }
 /**
  * Auto-backfill a DID's repo when first seen on the firehose.
@@ -297,14 +327,13 @@ function startMemoryDiagnostics() {
  *
  * @returns The WebSocket connection (for shutdown coordination)
  */
-export async function startIndexer(opts) {
-    const { relayUrl, collections, cursor, fetchTimeout } = opts;
+export async function configureIndexer(opts) {
     if (opts.ftsRebuildInterval != null)
         ftsRebuildInterval = opts.ftsRebuildInterval;
-    indexerCollections = collections;
-    indexerSignalCollections = opts.signalCollections || collections;
+    indexerCollections = opts.collections;
+    indexerSignalCollections = opts.signalCollections || opts.collections;
     indexerPinnedRepos = opts.pinnedRepos || null;
-    indexerFetchTimeout = fetchTimeout;
+    indexerFetchTimeout = opts.fetchTimeout;
     indexerMaxRetries = opts.maxRetries;
     indexerPlcUrl = opts.plcUrl;
     maxConcurrentBackfills = opts.parallelism ?? 3;
@@ -325,6 +354,11 @@ export async function startIndexer(opts) {
         cursorCheckpointTimer = setInterval(() => void checkpointCursor(), CURSOR_CHECKPOINT_INTERVAL_MS);
         cursorCheckpointTimer.unref?.();
     }
+}
+export async function startIndexer(opts) {
+    const { relayUrl, collections, cursor } = opts;
+    setCursorKey('relay');
+    await configureIndexer(opts);
     let wsUrl = `${relayUrl}/xrpc/com.atproto.sync.subscribeRepos`;
     if (cursor) {
         wsUrl += `?cursor=${cursor}`;
@@ -362,7 +396,7 @@ export async function startIndexer(opts) {
  * Only updates DIDs we already track (present in repoStatusCache) to avoid
  * writing rows for the entire network.
  */
-async function handleIdentityEvent(did, payloadHandle) {
+export async function handleIdentityEvent(did, payloadHandle) {
     if (!repoStatusCache.has(did))
         return;
     let handle = payloadHandle;
@@ -408,49 +442,33 @@ async function handleIdentityEvent(did, payloadHandle) {
     }
 }
 /**
- * Process a single firehose message. Decodes the CBOR header/body, filters
- * for relevant collections, validates records against lexicons, and routes
- * writes to the buffer (or pending buffer if the DID is mid-backfill).
+ * Whether a collection's records may be written from network data.
+ *
+ * Private collections are AppView-authoritative: nothing on the network may
+ * write or delete their rows, so stream ops naming them are spoofed by
+ * definition and must be dropped regardless of which source delivered them.
  */
-function processMessage(bytes, collections) {
-    const header = cborDecode(bytes, 0);
-    const body = cborDecode(bytes, header.offset);
-    // Handle identity events (handle changes). Fire-and-forget — keeps
-    // processMessage synchronous so the WS event loop drains without backpressure.
-    if (header.value.t === '#identity') {
-        const did = typeof body.value.did === 'string' ? body.value.did : undefined;
-        const handle = typeof body.value.handle === 'string' ? body.value.handle : undefined;
-        if (did)
-            handleIdentityEvent(did, handle);
-        return;
-    }
-    if (header.value.op !== 1 || header.value.t !== '#commit')
-        return;
-    if (!body.value.blocks || !body.value.ops)
-        return;
-    // Track sequence number for cursor
-    if (body.value.seq)
-        noteSeq(body.value.seq);
-    const did = body.value.repo;
-    if (!did)
+export function isIndexableCollection(collection, collections) {
+    return collections.has(collection) && !isPrivateCollection(collection);
+}
+/**
+ * Apply a repo's commit ops to the index.
+ *
+ * Wire-agnostic: the relay path reaches here after CBOR/CAR decoding, the
+ * Jetstream path after parsing JSON. Callers must have already filtered `ops`
+ * through {@link isIndexableCollection}.
+ *
+ * Handles auto-backfill signalling, buffering for DIDs mid-backfill, lexicon
+ * validation, and delete/put routing.
+ */
+export function applyCommit(did, ops) {
+    if (ops.length === 0)
         return;
     // When repos are pinned, only process events from those DIDs
     if (indexerPinnedRepos && !indexerPinnedRepos.has(did))
         return;
-    // Check if any ops in this commit are for collections we care about
-    // Private collections are AppView-authoritative: nothing on the network may
-    // write or delete their rows, so firehose ops naming them are spoofed by
-    // definition and must be dropped.
-    const relevantOps = body.value.ops.filter((op) => {
-        const collection = op.path.split('/')[0];
-        return collections.has(collection) && !isPrivateCollection(collection);
-    });
-    if (relevantOps.length === 0)
-        return;
-    // Copy blocks out of the original buffer before it can be GC'd
-    const { blocks } = parseCarFrame(new Uint8Array(body.value.blocks));
     // Only auto-backfill when we see activity in a signal collection
-    const hasSignalOp = relevantOps.some((op) => indexerSignalCollections.has(op.path.split('/')[0]));
+    const hasSignalOp = ops.some((op) => indexerSignalCollections.has(op.collection));
     // Use in-memory cache only — never hit DB from the hot path.
     // Unknown DIDs stay unknown until backfill or auto-backfill discovers them.
     // The cache is populated by triggerAutoBackfill and setRepoStatus calls.
@@ -459,7 +477,7 @@ function processMessage(bytes, collections) {
     if (cachedStatus === undefined) {
         repoStatusCache.set(did, 'unknown');
     }
-    if (hasSignalOp && (!indexerPinnedRepos || indexerPinnedRepos.has(did))) {
+    if (hasSignalOp) {
         if (repoStatus === null && backfillInFlight.size < maxConcurrentBackfills) {
             repoStatusCache.set(did, 'pending');
             triggerAutoBackfill(did);
@@ -470,19 +488,16 @@ function processMessage(bytes, collections) {
         }
     }
     // For non-signal ops (e.g. profile updates), only process if this DID is already tracked
-    if (!hasSignalOp) {
-        if (repoStatus === null)
-            return;
-    }
-    for (const op of relevantOps) {
-        const collection = op.path.split('/')[0];
-        const uri = `at://${did}/${op.path}`;
+    if (!hasSignalOp && repoStatus === null)
+        return;
+    for (const op of ops) {
+        const uri = `at://${did}/${op.collection}/${op.rkey}`;
         if (op.action === 'delete') {
-            deleteRecord(collection, uri);
+            deleteRecord(op.collection, uri);
             fireOnCommitHooks([
                 {
                     action: 'delete',
-                    collection,
+                    collection: op.collection,
                     uri,
                     authorDid: did,
                     record: null,
@@ -490,6 +505,73 @@ function processMessage(bytes, collections) {
             ]);
             continue;
         }
+        const record = op.record;
+        if (!op.cid || !record)
+            continue;
+        if (record.$type !== op.collection)
+            continue;
+        const validationError = validateRecord(getLexiconArray(), op.collection, record);
+        if (validationError) {
+            emit('indexer', 'validation_skip', {
+                uri,
+                collection: op.collection,
+                path: validationError.path,
+                error: validationError.message,
+            });
+            continue;
+        }
+        const item = { collection: op.collection, uri, cid: op.cid, authorDid: did, record };
+        // If DID is mid-backfill, buffer instead of writing directly
+        if (pendingBuffers.has(did)) {
+            pendingBuffers.get(did).push(item);
+        }
+        else {
+            bufferWrite(item);
+        }
+    }
+}
+/**
+ * Process a single firehose message. Decodes the CBOR header/body, filters
+ * for relevant collections, validates records against lexicons, and routes
+ * writes to the buffer (or pending buffer if the DID is mid-backfill).
+ */
+export function processMessage(bytes, collections) {
+    const header = cborDecode(bytes, 0);
+    const body = cborDecode(bytes, header.offset);
+    // Handle identity events (handle changes). Fire-and-forget — keeps
+    // processMessage synchronous so the WS event loop drains without backpressure.
+    if (header.value.t === '#identity') {
+        const did = typeof body.value.did === 'string' ? body.value.did : undefined;
+        const handle = typeof body.value.handle === 'string' ? body.value.handle : undefined;
+        if (did)
+            handleIdentityEvent(did, handle);
+        return;
+    }
+    if (header.value.op !== 1 || header.value.t !== '#commit')
+        return;
+    if (!body.value.blocks || !body.value.ops)
+        return;
+    // Track sequence number for cursor
+    if (body.value.seq)
+        noteSeq(body.value.seq);
+    const did = body.value.repo;
+    if (!did)
+        return;
+    // Check if any ops in this commit are for collections we care about, before
+    // paying for the CAR parse below.
+    const relevantOps = body.value.ops.filter((op) => isIndexableCollection(op.path.split('/')[0], collections));
+    if (relevantOps.length === 0)
+        return;
+    // Copy blocks out of the original buffer before it can be GC'd
+    const { blocks } = parseCarFrame(new Uint8Array(body.value.blocks));
+    const ops = [];
+    for (const op of relevantOps) {
+        const collection = op.path.split('/')[0];
+        const rkey = op.path.split('/').slice(1).join('/');
+        if (op.action === 'delete') {
+            ops.push({ action: 'delete', collection, rkey });
+            continue;
+        }
         const opCid = typeof op.cid === 'string' ? op.cid : op.cid?.$link;
         if (!opCid)
             continue;
@@ -498,27 +580,9 @@ function processMessage(bytes, collections) {
             continue;
         try {
             const { value: record } = cborDecode(data);
-            if (record?.$type === collection) {
-                const validationError = validateRecord(getLexiconArray(), collection, record);
-                if (validationError) {
-                    emit('indexer', 'validation_skip', {
-                        uri,
-                        collection,
-                        path: validationError.path,
-                        error: validationError.message,
-                    });
-                    continue;
-                }
-                const item = { collection, uri, cid: opCid, authorDid: did, record };
-                // If DID is mid-backfill, buffer instead of writing directly
-                if (pendingBuffers.has(did)) {
-                    pendingBuffers.get(did).push(item);
-                }
-                else {
-                    bufferWrite(item);
-                }
-            }
+            ops.push({ action: op.action === 'update' ? 'update' : 'create', collection, rkey, cid: opCid, record });
         }
         catch { }
     }
+    applyCommit(did, ops);
 }

package/dist/jetstream.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Jetstream v2 live tail — an alternative stream source to the relay firehose.
+ *
+ * Where `com.atproto.sync.subscribeRepos` ships every commit on the network as
+ * DAG-CBOR frames wrapping a CAR block store, Jetstream filters server-side and
+ * delivers records as already-decoded JSON. For an AppView tracking a handful
+ * of collections that is the difference between decoding the whole network and
+ * decoding only what it indexes.
+ *
+ * What this trades away: the CAR/MST proofs. hatk never verified them — it
+ * already trusts the relay — so nothing is lost that was being used. The
+ * `getRepo` backfill path in backfill.ts still speaks CAR and is unaffected.
+ *
+ * @see https://bsky.network/docs/jetstream
+ * @module
+ */
+import { type CommitOp, type IndexerCoreOpts } from './indexer.ts';
+/** Server-side filter caps, rejected before the WebSocket upgrade. */
+export declare const MAX_COLLECTIONS = 100;
+export declare const MAX_DIDS = 10000;
+export interface JetstreamOpts extends IndexerCoreOpts {
+    /** Instance base URL, e.g. `wss://jetstream.us-east.bsky.network`. */
+    jetstreamUrl: string;
+    cursor?: string | null;
+}
+/** A `#commit` payload as delivered by Jetstream v2. */
+interface JetstreamCommit {
+    did?: string;
+    seq?: number;
+    operation?: string;
+    collection?: string;
+    rkey?: string;
+    cid?: string;
+    record?: Record<string, any>;
+}
+/**
+ * Build the subscribe URL.
+ *
+ * Collections are sent explicitly rather than as an `ns.*` wildcard: a wildcard
+ * would also match sibling NSIDs the app has no lexicon for (and, for private
+ * collections, ones it must never accept from the network).
+ *
+ * Exported for tests.
+ */
+export declare function buildSubscribeUrl(jetstreamUrl: string, collections: Set<string>, pinnedRepos: Set<string> | null, cursor?: string | null): string;
+/**
+ * Reject filters the server would reject at the handshake, where the message
+ * can name the actual limit instead of surfacing as a failed connection.
+ * Exported for tests.
+ */
+export declare function assertFilterLimits(collections: Set<string>, pinnedRepos: Set<string> | null): void;
+/**
+ * Translate one Jetstream commit payload into hatk's wire-agnostic op shape.
+ *
+ * Deletes carry no `record` or `cid` — only the collection and rkey that
+ * identify what went away. Returns `null` when the payload is unusable or
+ * names a collection this AppView must not accept from the network.
+ *
+ * Exported for tests.
+ */
+export declare function commitToOp(commit: JetstreamCommit, collections: Set<string>): CommitOp | null;
+/**
+ * Handle one decoded frame. Kept synchronous so the socket's event queue drains
+ * without backpressure — identity resolution is fire-and-forget, matching the
+ * relay path.
+ *
+ * Exported for tests.
+ */
+export declare function processEvent(payload: any, collections: Set<string>): void;
+/**
+ * Connect to a Jetstream v2 instance and begin indexing.
+ *
+ * Reconnects on disconnect after {@link RECONNECT_DELAY_MS}, resuming from the
+ * highest seq seen rather than the boot-time cursor. Jetstream's cursor is
+ * inclusive and delivery is at-least-once, so the event at the resume point
+ * arrives again — harmless, since writes upsert on the record's `at://` URI.
+ *
+ * @returns The WebSocket connection (for shutdown coordination)
+ */
+export declare function startJetstreamIndexer(opts: JetstreamOpts): Promise<WebSocket>;
+export {};
+//# sourceMappingURL=jetstream.d.ts.map

package/dist/jetstream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jetstream.d.ts","sourceRoot":"","sources":["../src/jetstream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EASL,KAAK,QAAQ,EACb,KAAK,eAAe,EACrB,MAAM,cAAc,CAAA;AAGrB,sEAAsE;AACtE,eAAO,MAAM,eAAe,MAAM,CAAA;AAClC,eAAO,MAAM,QAAQ,QAAS,CAAA;AAO9B,MAAM,WAAW,aAAc,SAAQ,eAAe;IACpD,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,wDAAwD;AACxD,UAAU,eAAe;IACvB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC7B;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EACxB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAC/B,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GACrB,MAAM,CASR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,IAAI,CAalG;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,QAAQ,GAAG,IAAI,CAY7F;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAuBzE;AAED;;;;;;;;;GASG;AACH,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC,CAuCnF"}

package/dist/jetstream.js

@@ -0,0 +1,161 @@
+/**
+ * Jetstream v2 live tail — an alternative stream source to the relay firehose.
+ *
+ * Where `com.atproto.sync.subscribeRepos` ships every commit on the network as
+ * DAG-CBOR frames wrapping a CAR block store, Jetstream filters server-side and
+ * delivers records as already-decoded JSON. For an AppView tracking a handful
+ * of collections that is the difference between decoding the whole network and
+ * decoding only what it indexes.
+ *
+ * What this trades away: the CAR/MST proofs. hatk never verified them — it
+ * already trusts the relay — so nothing is lost that was being used. The
+ * `getRepo` backfill path in backfill.ts still speaks CAR and is unaffected.
+ *
+ * @see https://bsky.network/docs/jetstream
+ * @module
+ */
+import { applyCommit, configureIndexer, getLastSeq, handleIdentityEvent, isIndexableCollection, noteSeq, resumeCursor, setCursorKey, } from "./indexer.js";
+import { log, emit } from "./logger.js";
+/** Server-side filter caps, rejected before the WebSocket upgrade. */
+export const MAX_COLLECTIONS = 100;
+export const MAX_DIDS = 10_000;
+/** Event kinds hatk consumes. `commit` carries records; `identity` drives handle renames. */
+const KINDS = ['commit', 'identity'];
+const RECONNECT_DELAY_MS = 3000;
+/**
+ * Build the subscribe URL.
+ *
+ * Collections are sent explicitly rather than as an `ns.*` wildcard: a wildcard
+ * would also match sibling NSIDs the app has no lexicon for (and, for private
+ * collections, ones it must never accept from the network).
+ *
+ * Exported for tests.
+ */
+export function buildSubscribeUrl(jetstreamUrl, collections, pinnedRepos, cursor) {
+    const params = new URLSearchParams();
+    for (const collection of collections)
+        params.append('collections', collection);
+    for (const kind of KINDS)
+        params.append('kinds', kind);
+    if (pinnedRepos) {
+        for (const did of pinnedRepos)
+            params.append('dids', did);
+    }
+    if (cursor)
+        params.append('cursor', cursor);
+    return `${jetstreamUrl}/xrpc/network.bsky.jetstream.subscribeEvents?${params}`;
+}
+/**
+ * Reject filters the server would reject at the handshake, where the message
+ * can name the actual limit instead of surfacing as a failed connection.
+ * Exported for tests.
+ */
+export function assertFilterLimits(collections, pinnedRepos) {
+    if (collections.size > MAX_COLLECTIONS) {
+        throw new Error(`Jetstream accepts at most ${MAX_COLLECTIONS} collections, got ${collections.size}. ` +
+            `Narrow the indexed collections or use the relay firehose instead.`);
+    }
+    if (pinnedRepos && pinnedRepos.size > MAX_DIDS) {
+        throw new Error(`Jetstream accepts at most ${MAX_DIDS} dids, got ${pinnedRepos.size}. ` +
+            `Unpin some repos or use the relay firehose instead.`);
+    }
+}
+/**
+ * Translate one Jetstream commit payload into hatk's wire-agnostic op shape.
+ *
+ * Deletes carry no `record` or `cid` — only the collection and rkey that
+ * identify what went away. Returns `null` when the payload is unusable or
+ * names a collection this AppView must not accept from the network.
+ *
+ * Exported for tests.
+ */
+export function commitToOp(commit, collections) {
+    const { collection, rkey, operation } = commit;
+    if (!collection || !rkey || !operation)
+        return null;
+    if (!isIndexableCollection(collection, collections))
+        return null;
+    if (operation === 'delete') {
+        return { action: 'delete', collection, rkey };
+    }
+    if (operation !== 'create' && operation !== 'update')
+        return null;
+    if (!commit.cid || !commit.record)
+        return null;
+    return { action: operation, collection, rkey, cid: commit.cid, record: commit.record };
+}
+/**
+ * Handle one decoded frame. Kept synchronous so the socket's event queue drains
+ * without backpressure — identity resolution is fire-and-forget, matching the
+ * relay path.
+ *
+ * Exported for tests.
+ */
+export function processEvent(payload, collections) {
+    const kind = typeof payload?.$type === 'string' ? payload.$type.split('#')[1] : undefined;
+    if (kind === 'identity') {
+        const did = typeof payload.did === 'string' ? payload.did : undefined;
+        const handle = typeof payload.handle === 'string' ? payload.handle : undefined;
+        if (did)
+            handleIdentityEvent(did, handle);
+        return;
+    }
+    if (kind !== 'commit')
+        return;
+    // Track sequence for the cursor before filtering: a stream of events for
+    // collections we don't index still advances the position we must resume from.
+    if (typeof payload.seq === 'number')
+        noteSeq(payload.seq);
+    const did = typeof payload.did === 'string' ? payload.did : undefined;
+    if (!did)
+        return;
+    const op = commitToOp(payload, collections);
+    if (!op)
+        return;
+    applyCommit(did, [op]);
+}
+/**
+ * Connect to a Jetstream v2 instance and begin indexing.
+ *
+ * Reconnects on disconnect after {@link RECONNECT_DELAY_MS}, resuming from the
+ * highest seq seen rather than the boot-time cursor. Jetstream's cursor is
+ * inclusive and delivery is at-least-once, so the event at the resume point
+ * arrives again — harmless, since writes upsert on the record's `at://` URI.
+ *
+ * @returns The WebSocket connection (for shutdown coordination)
+ */
+export async function startJetstreamIndexer(opts) {
+    const { jetstreamUrl, collections, cursor } = opts;
+    const pinnedRepos = opts.pinnedRepos || null;
+    assertFilterLimits(collections, pinnedRepos);
+    setCursorKey('jetstream');
+    await configureIndexer(opts);
+    const wsUrl = buildSubscribeUrl(jetstreamUrl, collections, pinnedRepos, cursor);
+    if (cursor)
+        log(`[jetstream] Resuming from cursor ${cursor}`);
+    log(`[jetstream] Connecting to ${jetstreamUrl} (${collections.size} collections)...`);
+    // The lexicon default is identical framing, so an empty subprotocol echo is
+    // fine; we offer it so the server can pick the JSON arm explicitly.
+    const ws = new WebSocket(wsUrl, ['xrpc.v1.json']);
+    ws.addEventListener('message', (event) => {
+        try {
+            if (typeof event.data !== 'string')
+                return;
+            const frame = JSON.parse(event.data);
+            // Events arrive wrapped in an envelope with the event under `payload`.
+            if (frame?.payload)
+                processEvent(frame.payload, collections);
+        }
+        catch (err) {
+            emit('jetstream', 'decode_error', { error: err instanceof Error ? err.message : String(err) });
+        }
+    });
+    ws.addEventListener('open', () => log('[jetstream] Connected'));
+    ws.addEventListener('close', () => {
+        log(`[jetstream] Disconnected, reconnecting in ${RECONNECT_DELAY_MS / 1000}s...`);
+        // Read the seq at close time, not connect time — reusing the boot cursor on
+        // every reconnect would replay everything received since the process started.
+        setTimeout(() => startJetstreamIndexer({ ...opts, cursor: resumeCursor(getLastSeq(), opts.cursor) }), RECONNECT_DELAY_MS);
+    });
+    return ws;
+}

package/dist/main.js

@@ -23,6 +23,7 @@ import { initXrpc, listXrpc, configureRelay, configureCdn, configureOAuth, callX
 import { initOpengraph } from "./opengraph.js";
 import { initLabels, getLabelDefinitions } from "./labels.js";
 import { startIndexer } from "./indexer.js";
+import { startJetstreamIndexer } from "./jetstream.js";
 import { rebuildAllIndexes } from "./database/fts.js";
 import { createHandler, registerCoreHandlers } from "./server.js";
 import { setPrivateCollections } from "./private-collections.js";
@@ -206,19 +207,27 @@ log(`  Feeds: ${listFeeds()
 const ignoreSavedCursor = Boolean(process.env.HATK_IGNORE_SAVED_CURSOR);
 if (ignoreSavedCursor)
     log('[main] HATK_IGNORE_SAVED_CURSOR set — starting indexer from live');
-const cursor = ignoreSavedCursor ? null : await getCursor('relay');
-startIndexer({
-    relayUrl: config.relay,
+// Each source keeps its own cursor row: relay seqs and Jetstream seqs are
+// different coordinate systems, so switching sources must not resume one from
+// the other's value.
+const indexerCore = {
     plcUrl: config.plc,
     collections: collectionSet,
     signalCollections: config.backfill.signalCollections ? new Set(config.backfill.signalCollections) : undefined,
     pinnedRepos: config.backfill.repos ? new Set(config.backfill.repos) : undefined,
-    cursor,
     fetchTimeout: config.backfill.fetchTimeout,
     maxRetries: config.backfill.maxRetries,
     parallelism: config.backfill.parallelism,
     ftsRebuildInterval: config.ftsRebuildInterval,
-});
+};
+if (config.jetstream) {
+    const cursor = ignoreSavedCursor ? null : await getCursor('jetstream');
+    startJetstreamIndexer({ ...indexerCore, jetstreamUrl: config.jetstream.url, cursor });
+}
+else {
+    const cursor = ignoreSavedCursor ? null : await getCursor('relay');
+    startIndexer({ ...indexerCore, relayUrl: config.relay, cursor });
+}
 // 7. Run backfill in background
 runBackfillAndRestart();
 // Graceful shutdown

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatk/hatk",
-  "version": "0.0.1-alpha.70",
+  "version": "0.0.1-alpha.71",
   "license": "MIT",
   "bin": {
     "hatk": "dist/cli.js"