@hatk/hatk 0.0.1-alpha.64 → 0.0.1-alpha.65

package/dist/backfill.d.ts

@@ -1,3 +1,13 @@
+/**
+ * The collections whose local rows a full re-import may purge before
+ * re-inserting from the repo CAR.
+ *
+ * Private collections are AppView-authoritative and never present in a repo:
+ * purging their rows would destroy data the re-import can never restore (this
+ * happened — a login-triggered backfill wiped a deployment's activity rows).
+ * Exported for tests.
+ */
+export declare function purgeableCollections(collections: Iterable<string>): string[];
 import type { BackfillConfig } from './config.ts';
 /** Options passed to {@link runBackfill}. */
 interface BackfillOpts {

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAIjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,wDAAwD;IACxD,MAAM,EAAE,cAAc,CAAA;CACvB;AA+FD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkK/G;AA8BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAkIrE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAiBA;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAE5E;AAGD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAIjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,wDAAwD;IACxD,MAAM,EAAE,cAAc,CAAA;CACvB;AA+FD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkK/G;AA8BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAkIrE"}

package/dist/backfill.js

@@ -2,6 +2,19 @@ import { parseCarStream } from "./car.js";
 import { cborDecode } from "./cbor.js";
 import { walkMst } from "./mst.js";
 import { setRepoStatus, getRepoStatus, getRepoRev, getRepoRetryInfo, listRetryEligibleRepos, listPendingRepos, querySQL, runSQL, getSchema, bulkInsertRecords, } from "./database/db.js";
+import { isPrivateCollection } from "./private-collections.js";
+/**
+ * The collections whose local rows a full re-import may purge before
+ * re-inserting from the repo CAR.
+ *
+ * Private collections are AppView-authoritative and never present in a repo:
+ * purging their rows would destroy data the re-import can never restore (this
+ * happened — a login-triggered backfill wiped a deployment's activity rows).
+ * Exported for tests.
+ */
+export function purgeableCollections(collections) {
+    return [...collections].filter((c) => !isPrivateCollection(c));
+}
 import { emit, timer } from "./logger.js";
 import { validateRecord } from '@bigmoves/lexicon';
 import { getLexiconArray } from "./database/schema.js";
@@ -177,7 +190,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         // Delete existing records for this DID before re-importing so deletions are reflected
         // Only on full imports (no since) — diff CARs only contain changes
         if (!lastRev) {
-            for (const col of collections) {
+            for (const col of purgeableCollections(collections)) {
                 const schema = getSchema(col);
                 if (!schema)
                     continue;
@@ -198,7 +211,7 @@ export async function backfillRepo(did, collections, fetchTimeout) {
         const validationSkips = {};
         for (const entry of entries) {
             const collection = entry.path.split('/')[0];
-            if (!collections.has(collection))
+            if (!collections.has(collection) || isPrivateCollection(collection))
                 continue;
             const blockData = blocks.get(entry.cid);
             if (!blockData)

package/dist/indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAsKA;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CA8EjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAoDxE"}
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAuKA;;;;;;;GAOG;AACH,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGxD;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CA8EjF;AAED,8CAA8C;AAC9C,UAAU,WAAW;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAyBD;;;;;;;;;GASG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAoDxE"}

package/dist/indexer.js

@@ -1,5 +1,6 @@
 import { cborDecode } from "./cbor.js";
 import { parseCarFrame } from "./car.js";
+import { isPrivateCollection } from "./private-collections.js";
 import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses, getDatabasePort, updateRepoHandle, } from "./database/db.js";
 import { backfillRepo } from "./backfill.js";
 import { rebuildAllIndexes } from "./database/fts.js";
@@ -387,7 +388,13 @@ function processMessage(bytes, collections) {
     if (indexerPinnedRepos && !indexerPinnedRepos.has(did))
         return;
     // Check if any ops in this commit are for collections we care about
-    const relevantOps = body.value.ops.filter((op) => collections.has(op.path.split('/')[0]));
+    // Private collections are AppView-authoritative: nothing on the network may
+    // write or delete their rows, so firehose ops naming them are spoofed by
+    // definition and must be dropped.
+    const relevantOps = body.value.ops.filter((op) => {
+        const collection = op.path.split('/')[0];
+        return collections.has(collection) && !isPrivateCollection(collection);
+    });
     if (relevantOps.length === 0)
         return;
     // Copy blocks out of the original buffer before it can be GC'd

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatk/hatk",
-  "version": "0.0.1-alpha.64",
+  "version": "0.0.1-alpha.65",
   "license": "MIT",
   "bin": {
     "hatk": "dist/cli.js"