@hatk/hatk 0.0.1-alpha.67 → 0.0.1-alpha.69

package/dist/backfill.d.ts

@@ -8,6 +8,15 @@
  * Exported for tests.
  */
 export declare function purgeableCollections(collections: Iterable<string>): string[];
+/**
+ * Whether a repo with this stored status should be queued for backfill.
+ *
+ * Takendown repos are excluded: re-backfilling one ends in
+ * `setRepoStatus(did, 'active')`, which would silently reinstate an account an
+ * admin took down — takedowns must survive every redeploy. Reinstatement goes
+ * through `/admin/reverse-takedown` only. Exported for tests.
+ */
+export declare function backfillEligible(status: string | null): boolean;
 import type { BackfillConfig } from './config.ts';
 /** Options passed to {@link runBackfill}. */
 interface BackfillOpts {

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAiBA;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAE5E;AAGD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAIjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,wDAAwD;IACxD,MAAM,EAAE,cAAc,CAAA;CACvB;AA+FD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkK/G;AA8BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAkIrE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAiBA;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAE5E;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAE/D;AAGD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAIjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,wDAAwD;IACxD,MAAM,EAAE,cAAc,CAAA;CACvB;AA+FD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAyK/G;AA8BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAkIrE"}

package/dist/backfill.js

@@ -15,6 +15,17 @@ import { isPrivateCollection } from "./private-collections.js";
 export function purgeableCollections(collections) {
     return [...collections].filter((c) => !isPrivateCollection(c));
 }
+/**
+ * Whether a repo with this stored status should be queued for backfill.
+ *
+ * Takendown repos are excluded: re-backfilling one ends in
+ * `setRepoStatus(did, 'active')`, which would silently reinstate an account an
+ * admin took down — takedowns must survive every redeploy. Reinstatement goes
+ * through `/admin/reverse-takedown` only. Exported for tests.
+ */
+export function backfillEligible(status) {
+    return status !== 'active' && status !== 'takendown';
+}
 import { emit, timer } from "./logger.js";
 import { validateRecord } from '@bigmoves/lexicon';
 import { getLexiconArray } from "./database/schema.js";
@@ -131,6 +142,13 @@ async function* listReposByCollection(pdsUrl, collection) {
  * ```
  */
 export async function backfillRepo(did, collections, fetchTimeout) {
+    // A takendown repo must never be re-imported: the success path ends in
+    // setRepoStatus('active'), which would reinstate the account. Guarding here
+    // covers every caller (boot scan, auto-backfill, admin re-index).
+    if ((await getRepoStatus(did)) === 'takendown') {
+        emit('backfill', 'takedown_skip', { did });
+        return 0;
+    }
     const elapsed = timer();
     let count = 0;
     let carSizeBytes;
@@ -385,7 +403,7 @@ export async function runBackfill(opts) {
     const pending = [];
     for (const did of dids) {
         const status = await getRepoStatus(did);
-        if (status !== 'active') {
+        if (backfillEligible(status)) {
             if (!status)
                 await setRepoStatus(did, 'pending');
             pending.push(did);

package/dist/dev-entry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dev-entry.d.ts","sourceRoot":"","sources":["../src/dev-entry.ts"],"names":[],"mappings":"AAgHA,eAAO,MAAM,OAAO,yCAKlB,CAAA;AAEF,yEAAyE;AACzE,wBAAsB,YAAY,kBAEjC;AAED,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA"}
+{"version":3,"file":"dev-entry.d.ts","sourceRoot":"","sources":["../src/dev-entry.ts"],"names":[],"mappings":"AAmHA,eAAO,MAAM,OAAO,yCAKlB,CAAA;AAEF,yEAAyE;AACzE,wBAAsB,YAAY,kBAEjC;AAED,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA"}

package/dist/dev-entry.js

@@ -72,7 +72,11 @@ if (config.oauth) {
 }
 // Start indexer
 const collectionSet = new Set(collections);
-const cursor = await getCursor('relay');
+// Same escape hatch as main.ts: boot from live instead of the saved cursor.
+const ignoreSavedCursor = Boolean(process.env.HATK_IGNORE_SAVED_CURSOR);
+if (ignoreSavedCursor)
+    log('[dev] HATK_IGNORE_SAVED_CURSOR set — starting indexer from live');
+const cursor = ignoreSavedCursor ? null : await getCursor('relay');
 startIndexer({
     relayUrl: config.relay,
     plcUrl: config.plc,

package/dist/indexer.d.ts

@@ -1,3 +1,23 @@
+/** Record the latest firehose sequence seen. Source of truth for cursor persistence and reconnect resume. */
+export declare function noteSeq(seq: number): void;
+/**
+ * Persist the latest firehose sequence when it has advanced past the stored
+ * cursor. Runs on a timer independent of the write path: an app whose
+ * collections rarely appear on the firehose never flushes a batch, so the
+ * flush-path cursor write alone leaves the stored cursor frozen — and a frozen
+ * cursor makes the relay replay its whole retention window at line rate
+ * (~135 GB/hour observed against bsky.network) on every boot and reconnect.
+ */
+export declare function checkpointCursor(): Promise<void>;
+/**
+ * The cursor a reconnect should resume from: the latest seq this process has
+ * seen, falling back to the boot-time cursor before any message has arrived.
+ * Reusing the boot-time cursor on every reconnect would replay everything
+ * received since the process started.
+ */
+export declare function resumeCursor(liveSeq: number | null, bootCursor: string | null | undefined): string | null | undefined;
+/** Reset module cursor state between tests. */
+export declare function _resetCursorStateForTests(): void;
 /**
  * Auto-backfill a DID's repo when first seen on the firehose.
  *

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAuKA;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CA8EjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAoDxE"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AA4KA,6GAA6G;AAC7G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAEzC;AAED;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAStD;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,MAAM,GAAG,IAAI,EACtB,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GACpC,MAAM,GAAG,IAAI,GAAG,SAAS,CAE3B;AAED,+CAA+C;AAC/C,wBAAgB,yBAAyB,IAAI,IAAI,CAGhD;AAED;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CA8EjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CA4DxE"}

package/dist/indexer.js

@@ -12,8 +12,11 @@ import { validateRecord } from '@bigmoves/lexicon';
 let buffer = [];
 let flushTimer = null;
 let lastSeq = null;
+let lastPersistedSeq = null;
+let cursorCheckpointTimer = null;
 const BATCH_SIZE = 100;
 const FLUSH_INTERVAL_MS = 500;
+const CURSOR_CHECKPOINT_INTERVAL_MS = 5_000;
 let writesSinceRebuild = 0;
 let ftsRebuildInterval = 500;
 // Event buffer for DIDs mid-backfill
@@ -57,8 +60,10 @@ async function flushBuffer() {
         }
     }
     if (lastSeq !== null) {
+        const seq = lastSeq;
         try {
-            await setCursor('relay', String(lastSeq));
+            await setCursor('relay', String(seq));
+            lastPersistedSeq = seq;
         }
         catch (err) {
             cursorError = err.message;
@@ -134,6 +139,44 @@ function bufferWrite(item) {
         scheduleFlush();
     }
 }
+/** Record the latest firehose sequence seen. Source of truth for cursor persistence and reconnect resume. */
+export function noteSeq(seq) {
+    lastSeq = seq;
+}
+/**
+ * Persist the latest firehose sequence when it has advanced past the stored
+ * cursor. Runs on a timer independent of the write path: an app whose
+ * collections rarely appear on the firehose never flushes a batch, so the
+ * flush-path cursor write alone leaves the stored cursor frozen — and a frozen
+ * cursor makes the relay replay its whole retention window at line rate
+ * (~135 GB/hour observed against bsky.network) on every boot and reconnect.
+ */
+export async function checkpointCursor() {
+    if (lastSeq === null || lastSeq === lastPersistedSeq)
+        return;
+    const seq = lastSeq;
+    try {
+        await setCursor('relay', String(seq));
+        lastPersistedSeq = seq;
+    }
+    catch (err) {
+        emit('indexer', 'cursor_checkpoint_error', { cursor_seq: seq, error: err.message });
+    }
+}
+/**
+ * The cursor a reconnect should resume from: the latest seq this process has
+ * seen, falling back to the boot-time cursor before any message has arrived.
+ * Reusing the boot-time cursor on every reconnect would replay everything
+ * received since the process started.
+ */
+export function resumeCursor(liveSeq, bootCursor) {
+    return liveSeq !== null ? String(liveSeq) : bootCursor;
+}
+/** Reset module cursor state between tests. */
+export function _resetCursorStateForTests() {
+    lastSeq = null;
+    lastPersistedSeq = null;
+}
 /**
  * Auto-backfill a DID's repo when first seen on the firehose.
  *
@@ -275,6 +318,13 @@ export async function startIndexer(opts) {
         log(`[indexer] Warmed repo status cache with ${statuses.length} entries`);
     }
     // startMemoryDiagnostics()
+    // Checkpoint the cursor on a timer regardless of write activity (see
+    // checkpointCursor). Guarded so reconnects don't stack intervals; unref'd so
+    // it never keeps the process alive.
+    if (!cursorCheckpointTimer) {
+        cursorCheckpointTimer = setInterval(() => void checkpointCursor(), CURSOR_CHECKPOINT_INTERVAL_MS);
+        cursorCheckpointTimer.unref?.();
+    }
     let wsUrl = `${relayUrl}/xrpc/com.atproto.sync.subscribeRepos`;
     if (cursor) {
         wsUrl += `?cursor=${cursor}`;
@@ -299,7 +349,7 @@ export async function startIndexer(opts) {
     ws.addEventListener('open', () => log('[indexer] Connected to relay'));
     ws.addEventListener('close', () => {
         log('[indexer] Disconnected, reconnecting in 3s...');
-        setTimeout(() => startIndexer(opts), 3000);
+        setTimeout(() => startIndexer({ ...opts, cursor: resumeCursor(lastSeq, opts.cursor) }), 3000);
     });
     return ws;
 }
@@ -380,7 +430,7 @@ function processMessage(bytes, collections) {
         return;
     // Track sequence number for cursor
     if (body.value.seq)
-        lastSeq = body.value.seq;
+        noteSeq(body.value.seq);
     const did = body.value.repo;
     if (!did)
         return;

package/dist/main.js

@@ -199,7 +199,14 @@ log(`  Feeds: ${listFeeds()
     .map((f) => f.name)
     .join(', ')}`);
 // 6. Start indexer with cursor
-const cursor = await getCursor('relay');
+// HATK_IGNORE_SAVED_CURSOR=1 boots from the live firehose instead of the saved
+// cursor — the escape hatch when a stale cursor would make the relay replay its
+// retention window at line rate (see checkpointCursor in indexer.ts). Unset it
+// after one boot so later restarts resume normally.
+const ignoreSavedCursor = Boolean(process.env.HATK_IGNORE_SAVED_CURSOR);
+if (ignoreSavedCursor)
+    log('[main] HATK_IGNORE_SAVED_CURSOR set — starting indexer from live');
+const cursor = ignoreSavedCursor ? null : await getCursor('relay');
 startIndexer({
     relayUrl: config.relay,
     plcUrl: config.plc,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatk/hatk",
-  "version": "0.0.1-alpha.67",
+  "version": "0.0.1-alpha.69",
   "license": "MIT",
   "bin": {
     "hatk": "dist/cli.js"