@hatk/hatk 0.0.1-alpha.12 → 0.0.1-alpha.14

package/dist/backfill.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,qDAAqD;IACrD,MAAM,EAAE,cAAc,CAAA;CACvB;AAuGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAwI/G;AAgCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}
+{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../src/backfill.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,6CAA6C;AAC7C,UAAU,YAAY;IACpB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IACd,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAA;IACd,yEAAyE;IACzE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACxB,qDAAqD;IACrD,MAAM,EAAE,cAAc,CAAA;CACvB;AAuGD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAsJ/G;AAgCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiInE"}

package/dist/backfill.js

@@ -153,10 +153,25 @@ export async function backfillRepo(did, collections, fetchTimeout) {
             throw httpErr;
         }
         resolvedSince = lastRev;
-        const { roots, blocks, byteLength } = await parseCarStream(res.body);
+        let { roots, blocks, byteLength } = await parseCarStream(res.body);
         carSizeBytes = byteLength;
-        // Decode commit to get MST root
-        const rootData = blocks.get(roots[0]);
+        // Decode commit to get MST root — if the diff CAR is missing the root block,
+        // fall back to a full import (the PDS compacted past our `since` rev)
+        let rootData = blocks.get(roots[0]);
+        if (!rootData && lastRev) {
+            lastRev = null;
+            resolvedSince = null;
+            res = await fetch(baseUrl, { signal: controller.signal });
+            if (!res.ok) {
+                const httpErr = new Error(`getRepo failed for ${did}: ${res.status}`);
+                httpErr.httpStatus = res.status;
+                throw httpErr;
+            }
+            ;
+            ({ roots, blocks, byteLength } = await parseCarStream(res.body));
+            carSizeBytes = byteLength;
+            rootData = blocks.get(roots[0]);
+        }
         if (!rootData)
             throw new Error(`No root block for ${did}`);
         const { value: commit } = cborDecode(rootData);

package/dist/config.js

@@ -27,7 +27,7 @@ export function loadConfig(configPath) {
             fetchTimeout: parseInt(env.BACKFILL_FETCH_TIMEOUT || '') || backfillRaw.fetchTimeout || 300,
             maxRetries: parseInt(env.BACKFILL_MAX_RETRIES || '') || backfillRaw.maxRetries || 5,
         },
-        ftsRebuildInterval: parseInt(env.FTS_REBUILD_INTERVAL || '') || parsed.ftsRebuildInterval || 500,
+        ftsRebuildInterval: parseInt(env.FTS_REBUILD_INTERVAL || '') || parsed.ftsRebuildInterval || 5000,
         oauth: null,
         admins: env.ADMINS ? env.ADMINS.split(',').map((s) => s.trim()) : parsed.admins || [],
     };

package/dist/db.js

@@ -115,8 +115,8 @@ async function all(sql, ...params) {
 }
 export async function initDatabase(dbPath, tableSchemas, ddlStatements) {
     instance = await DuckDBInstance.create(dbPath === ':memory:' ? undefined : dbPath, {
-        memory_limit: '512MB',
-        threads: '2',
+        memory_limit: '256MB',
+        threads: '1',
     });
     con = await instance.connect();
     readCon = await instance.connect();

package/dist/fts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB5E"}
+{"version":3,"file":"fts.d.ts","sourceRoot":"","sources":["../src/fts.ts"],"names":[],"mappings":"AAwEA,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAElE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0FrE;AAokBD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIpD;AAED,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B5E"}

package/dist/fts.js

@@ -752,6 +752,11 @@ export async function rebuildAllIndexes(collections) {
             errors.push(`${collection}: ${err.message}`);
         }
     }
+    // Compact WAL to free DuckDB memory after heavy FTS operations
+    try {
+        await runSQL('CHECKPOINT');
+    }
+    catch { }
     emit('fts', 'rebuild', {
         collections_total: collections.length,
         collections_rebuilt: rebuilt,

package/dist/main.js

@@ -19,8 +19,13 @@ import { runBackfill } from "./backfill.js";
 import { initOAuth } from "./oauth/server.js";
 import { loadOnLoginHook } from "./oauth/hooks.js";
 import { initSetup } from "./setup.js";
+function logMemory(phase) {
+    const mem = process.memoryUsage();
+    log(`[mem] ${phase}: heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB rss=${Math.round(mem.rss / 1024 / 1024)}MB external=${Math.round(mem.external / 1024 / 1024)}MB arrayBuffers=${Math.round(mem.arrayBuffers / 1024 / 1024)}MB`);
+}
 const configPath = process.argv[2] || 'config.yaml';
 const configDir = dirname(resolve(configPath));
+logMemory('startup');
 // 1. Load config
 const config = loadConfig(configPath);
 configureRelay(config.relay);
@@ -74,6 +79,7 @@ if (config.database !== ':memory:') {
     mkdirSync(dirname(config.database), { recursive: true });
 }
 await initDatabase(config.database, schemas, ddlStatements);
+logMemory('after-db-init');
 log(`[main] DuckDB initialized (${config.database === ':memory:' ? 'in-memory' : config.database})`);
 // 3b. Run setup hooks (after DB init, before server)
 await initSetup(resolve(configDir, 'setup'));
@@ -104,6 +110,7 @@ if (config.oauth) {
     await initOAuth(config.oauth, config.plc, config.relay);
     log(`[main] OAuth initialized (issuer: ${config.oauth.issuer})`);
 }
+logMemory('before-server');
 // 5. Start server immediately (don't wait for backfill)
 const collectionSet = new Set(collections);
 startServer(config.port, collections, config.publicDir, config.oauth, config.admins);
@@ -115,6 +122,7 @@ log(`  Collections: ${collections.join(', ')}`);
 log(`  Feeds: ${listFeeds()
     .map((f) => f.name)
     .join(', ')}`);
+logMemory('after-server');
 // 6. Start indexer with cursor
 const cursor = await getCursor('relay');
 startIndexer({
@@ -145,3 +153,8 @@ runBackfill({
     .catch((err) => {
     console.error('[main] Backfill error:', err.message);
 });
+// Graceful shutdown
+process.on('SIGTERM', () => {
+    log('[main] Received SIGTERM, shutting down...');
+    process.exit(0);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hatk/hatk",
-  "version": "0.0.1-alpha.12",
+  "version": "0.0.1-alpha.14",
   "license": "MIT",
   "bin": {
     "hatk": "dist/cli.js"