@hatk/hatk 0.0.1-alpha.2 → 0.0.1-alpha.21

package/dist/indexer.js

@@ -18,7 +18,7 @@ let ftsRebuildInterval = 500;
 const pendingBuffers = new Map();
 // Track in-flight backfills to avoid duplicates
 const backfillInFlight = new Set();
-const MAX_CONCURRENT_BACKFILLS = 5;
+const pendingReschedule = new Set();
 // In-memory cache of repo status to avoid flooding the DB read queue
 const repoStatusCache = new Map();
 // Set by startIndexer
@@ -27,6 +27,7 @@ let indexerSignalCollections;
 let indexerPinnedRepos = null;
 let indexerFetchTimeout;
 let indexerMaxRetries;
+let maxConcurrentBackfills = 3;
 async function flushBuffer() {
     if (buffer.length === 0)
         return;
@@ -113,6 +114,16 @@ function bufferWrite(item) {
 export async function triggerAutoBackfill(did, attempt = 0) {
     if (backfillInFlight.has(did))
         return;
+    if (backfillInFlight.size >= maxConcurrentBackfills) {
+        if (!pendingReschedule.has(did)) {
+            pendingReschedule.add(did);
+            setTimeout(() => {
+                pendingReschedule.delete(did);
+                triggerAutoBackfill(did, attempt);
+            }, 10_000);
+        }
+        return;
+    }
     backfillInFlight.add(did);
     pendingBuffers.set(did, []);
     if (attempt === 0)
@@ -193,6 +204,7 @@ export async function startIndexer(opts) {
     indexerPinnedRepos = opts.pinnedRepos || null;
     indexerFetchTimeout = fetchTimeout;
     indexerMaxRetries = opts.maxRetries;
+    maxConcurrentBackfills = opts.parallelism ?? 3;
     // Pre-populate repo status cache from DB so non-signal updates
     // (e.g. profile changes) are processed for already-tracked DIDs
     if (repoStatusCache.size === 0) {
@@ -264,7 +276,7 @@ function processMessage(bytes, collections) {
         repoStatusCache.set(did, 'unknown');
     }
     if (hasSignalOp && (!indexerPinnedRepos || indexerPinnedRepos.has(did))) {
-        if (repoStatus === null && backfillInFlight.size < MAX_CONCURRENT_BACKFILLS) {
+        if (repoStatus === null && backfillInFlight.size < maxConcurrentBackfills) {
             repoStatusCache.set(did, 'pending');
             triggerAutoBackfill(did);
         }

package/dist/main.js

@@ -5,7 +5,7 @@ import { log } from "./logger.js";
 import { loadConfig } from "./config.js";
 import { loadLexicons, storeLexicons, discoverCollections, generateTableSchema, generateCreateTableSQL, } from "./schema.js";
 import { discoverViews } from "./views.js";
-import { initDatabase, getCursor, querySQL, backfillChildTables } from "./db.js";
+import { initDatabase, getCursor, querySQL } from "./db.js";
 import { initFeeds, listFeeds } from "./feeds.js";
 import { initXrpc, listXrpc, configureRelay } from "./xrpc.js";
 import { initOpengraph } from "./opengraph.js";
@@ -19,10 +19,15 @@ import { runBackfill } from "./backfill.js";
 import { initOAuth } from "./oauth/server.js";
 import { loadOnLoginHook } from "./oauth/hooks.js";
 import { initSetup } from "./setup.js";
-const configPath = process.argv[2] || 'config.yaml';
+function logMemory(phase) {
+    const mem = process.memoryUsage();
+    log(`[mem] ${phase}: heap=${Math.round(mem.heapUsed / 1024 / 1024)}MB rss=${Math.round(mem.rss / 1024 / 1024)}MB external=${Math.round(mem.external / 1024 / 1024)}MB arrayBuffers=${Math.round(mem.arrayBuffers / 1024 / 1024)}MB`);
+}
+const configPath = process.argv[2] || 'hatk.config.ts';
 const configDir = dirname(resolve(configPath));
+logMemory('startup');
 // 1. Load config
-const config = loadConfig(configPath);
+const config = await loadConfig(configPath);
 configureRelay(config.relay);
 // 2. Load lexicons, validate schemas, and discover collections
 const lexicons = loadLexicons(resolve(configDir, 'lexicons'));
@@ -74,9 +79,8 @@ if (config.database !== ':memory:') {
     mkdirSync(dirname(config.database), { recursive: true });
 }
 await initDatabase(config.database, schemas, ddlStatements);
+logMemory('after-db-init');
 log(`[main] DuckDB initialized (${config.database === ':memory:' ? 'in-memory' : config.database})`);
-// 3a. Backfill child tables for decomposed arrays (one-time migration)
-await backfillChildTables();
 // 3b. Run setup hooks (after DB init, before server)
 await initSetup(resolve(configDir, 'setup'));
 // Detect orphaned tables
@@ -106,9 +110,34 @@ if (config.oauth) {
     await initOAuth(config.oauth, config.plc, config.relay);
     log(`[main] OAuth initialized (issuer: ${config.oauth.issuer})`);
 }
+logMemory('before-server');
 // 5. Start server immediately (don't wait for backfill)
 const collectionSet = new Set(collections);
-startServer(config.port, collections, config.publicDir, config.oauth, config.admins);
+const backfillOpts = {
+    pdsUrl: relayHttpUrl(config.relay),
+    plcUrl: config.plc,
+    collections: collectionSet,
+    config: config.backfill,
+};
+function runBackfillAndRestart() {
+    runBackfill(backfillOpts)
+        .then((recordCount) => {
+        log('[main] Backfill complete, rebuilding FTS indexes...');
+        return rebuildAllIndexes(collections).then(() => recordCount);
+    })
+        .then((recordCount) => {
+        log('[main] FTS indexes ready');
+        if (recordCount > 0) {
+            logMemory('after-backfill');
+            log('[main] Restarting to reclaim memory...');
+            process.exit(1);
+        }
+    })
+        .catch((err) => {
+        console.error('[main] Backfill error:', err.message);
+    });
+}
+startServer(config.port, collections, config.publicDir, config.oauth, config.admins, undefined, runBackfillAndRestart);
 log(`\nhatk running:`);
 log(`  Relay: ${config.relay}`);
 log(`  Database: ${config.database}`);
@@ -117,6 +146,7 @@ log(`  Collections: ${collections.join(', ')}`);
 log(`  Feeds: ${listFeeds()
     .map((f) => f.name)
     .join(', ')}`);
+logMemory('after-server');
 // 6. Start indexer with cursor
 const cursor = await getCursor('relay');
 startIndexer({
@@ -127,22 +157,13 @@ startIndexer({
     cursor,
     fetchTimeout: config.backfill.fetchTimeout,
     maxRetries: config.backfill.maxRetries,
+    parallelism: config.backfill.parallelism,
     ftsRebuildInterval: config.ftsRebuildInterval,
 });
 // 7. Run backfill in background
-runBackfill({
-    pdsUrl: relayHttpUrl(config.relay),
-    plcUrl: config.plc,
-    collections: collectionSet,
-    config: config.backfill,
-})
-    .then(() => {
-    log('[main] Backfill complete, rebuilding FTS indexes...');
-    return rebuildAllIndexes(collections);
-})
-    .then(() => {
-    log('[main] FTS indexes ready');
-})
-    .catch((err) => {
-    console.error('[main] Backfill error:', err.message);
+runBackfillAndRestart();
+// Graceful shutdown
+process.on('SIGTERM', () => {
+    log('[main] Received SIGTERM, shutting down...');
+    process.exit(0);
 });

package/dist/mst.d.ts

@@ -2,5 +2,7 @@ export interface MstEntry {
     path: string;
     cid: string;
 }
-export declare function walkMst(blocks: Map<string, Uint8Array>, rootCid: string): MstEntry[];
+export declare function walkMst(blocks: {
+    get(cid: string): Uint8Array | undefined;
+}, rootCid: string): Generator<MstEntry>;
 //# sourceMappingURL=mst.d.ts.map

package/dist/mst.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,wBAAgB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE,CAiCpF"}
+{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,wBAAiB,OAAO,CAAC,MAAM,EAAE;IAAE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAA;CAAE,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CA8BnH"}

package/dist/mst.js

@@ -1,14 +1,13 @@
 import { cborDecode } from "./cbor.js";
-export function walkMst(blocks, rootCid) {
-    const entries = [];
-    function visit(cid, prefix) {
+export function* walkMst(blocks, rootCid) {
+    function* visit(cid, prefix) {
         const data = blocks.get(cid);
         if (!data)
             return prefix;
         const { value: node } = cborDecode(data);
         // Visit left subtree
         if (node.l?.$link)
-            visit(node.l.$link, prefix);
+            yield* visit(node.l.$link, prefix);
         let lastKey = prefix;
         for (const entry of node.e || []) {
             const keySuffix = entry.k instanceof Uint8Array ? new TextDecoder().decode(entry.k) : entry.k;
@@ -16,15 +15,14 @@ export function walkMst(blocks, rootCid) {
             const fullKey = lastKey.substring(0, prefixLen) + keySuffix;
             lastKey = fullKey;
             if (entry.v?.$link) {
-                entries.push({ path: fullKey, cid: entry.v.$link });
+                yield { path: fullKey, cid: entry.v.$link };
             }
             // Visit right subtree
             if (entry.t?.$link) {
-                visit(entry.t.$link, lastKey);
+                yield* visit(entry.t.$link, lastKey);
             }
         }
         return lastKey;
     }
-    visit(rootCid, '');
-    return entries;
+    yield* visit(rootCid, '');
 }

package/dist/server.d.ts

@@ -2,5 +2,5 @@ import { type Server, type IncomingMessage } from 'node:http';
 import type { OAuthConfig } from './config.ts';
 export declare function startServer(port: number, collections: string[], publicDir: string | null, oauth: OAuthConfig | null, admins?: string[], resolveViewer?: (req: IncomingMessage) => {
     did: string;
-} | null): Server;
+} | null, onResync?: () => void): Server;
 //# sourceMappingURL=server.d.ts.map

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAE,KAAK,eAAe,EAAE,MAAM,WAAW,CAAA;AAiD3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AA2B9C,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAAE,EACrB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,WAAW,GAAG,IAAI,EACzB,MAAM,GAAE,MAAM,EAAO,EACrB,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,GAC/D,MAAM,CAm7BR"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAE,KAAK,eAAe,EAAE,MAAM,WAAW,CAAA;AAmD3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AA2B9C,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAAE,EACrB,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,EAAE,WAAW,GAAG,IAAI,EACzB,MAAM,GAAE,MAAM,EAAO,EACrB,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,KAAK;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,EAChE,QAAQ,CAAC,EAAE,MAAM,IAAI,GACpB,MAAM,CA28BR"}

package/dist/server.js

@@ -1,4 +1,6 @@
 import { createServer } from 'node:http';
+import { gzipSync } from 'node:zlib';
+import { existsSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { join, extname } from 'node:path';
 import { queryRecords, getRecordByUri, searchRecords, getSchema, reshapeRow, setRepoStatus, getRepoStatus, getRepoRetryInfo, querySQL, insertRecord, deleteRecord, queryLabelsForUris, insertLabels, searchAccounts, listReposPaginated, getCollectionCounts, normalizeValue, getSchemaDump, getPreferences, putPreference, } from "./db.js";
@@ -37,14 +39,15 @@ function readBodyRaw(req) {
         req.on('error', reject);
     });
 }
-export function startServer(port, collections, publicDir, oauth, admins = [], resolveViewer) {
+export function startServer(port, collections, publicDir, oauth, admins = [], resolveViewer, onResync) {
     const coreXrpc = (method) => `/xrpc/dev.hatk.${method}`;
+    const devMode = process.env.DEV_MODE === '1';
     function requireAdmin(viewer, res) {
         if (!viewer) {
             jsonError(res, 401, 'Authentication required');
             return false;
         }
-        if (!admins.includes(viewer.did)) {
+        if (!devMode && !admins.includes(viewer.did)) {
             jsonError(res, 403, 'Admin access required');
             return false;
         }
@@ -447,9 +450,10 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
                 }
                 for (const did of repoList) {
                     await setRepoStatus(did, 'pending');
-                    triggerAutoBackfill(did);
                 }
                 jsonResponse(res, { resyncing: repoList.length });
+                if (onResync)
+                    onResync();
                 return;
             }
             // POST /admin/repos/remove — remove DIDs from tracking
@@ -478,7 +482,14 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
                 const sizeRows = await querySQL(`SELECT database_size, memory_usage, memory_limit FROM pragma_database_size()`);
                 const dbInfo = sizeRows[0] ?? {};
                 const collectionCounts = await getCollectionCounts();
-                jsonResponse(res, { repos: counts, duckdb: dbInfo, collections: collectionCounts });
+                const mem = process.memoryUsage();
+                const node = {
+                    rss: `${(mem.rss / 1024 / 1024).toFixed(1)} MiB`,
+                    heapUsed: `${(mem.heapUsed / 1024 / 1024).toFixed(1)} MiB`,
+                    heapTotal: `${(mem.heapTotal / 1024 / 1024).toFixed(1)} MiB`,
+                    external: `${(mem.external / 1024 / 1024).toFixed(1)} MiB`,
+                };
+                jsonResponse(res, { repos: counts, duckdb: dbInfo, node, collections: collectionCounts });
                 return;
             }
             // GET /admin/info/:did — repo status info
@@ -864,6 +875,21 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
                     throw err;
                 }
             }
+            // GET /robots.txt — serve from user's public dir or fall back to hatk default
+            if (url.pathname === '/robots.txt') {
+                const userRobots = publicDir ? join(publicDir, 'robots.txt') : null;
+                const defaultRobots = join(import.meta.dirname, '../public/robots.txt');
+                const robotsPath = userRobots && existsSync(userRobots) ? userRobots : defaultRobots;
+                try {
+                    const content = await readFile(robotsPath);
+                    res.writeHead(200, { 'Content-Type': 'text/plain' });
+                    res.end(content);
+                    return;
+                }
+                catch {
+                    // fall through
+                }
+            }
             // Static file serving
             if (publicDir) {
                 try {
@@ -912,15 +938,33 @@ export function startServer(port, collections, publicDir, oauth, admins = [], re
     server.listen(port, () => log(`[server] ${oauth?.issuer || `http://localhost:${port}`}`));
     return server;
 }
+function sendJson(res, status, body) {
+    const acceptEncoding = res.req?.headers['accept-encoding'] || '';
+    if (body.length > 1024 && /\bgzip\b/.test(acceptEncoding)) {
+        const compressed = gzipSync(body);
+        res.writeHead(status, {
+            'Content-Type': 'application/json',
+            'Content-Encoding': 'gzip',
+            Vary: 'Accept-Encoding',
+            ...(status === 200 ? { 'Cache-Control': 'no-store' } : {}),
+        });
+        res.end(compressed);
+    }
+    else {
+        res.writeHead(status, {
+            'Content-Type': 'application/json',
+            ...(status === 200 ? { 'Cache-Control': 'no-store' } : {}),
+        });
+        res.end(body);
+    }
+}
 function jsonResponse(res, data) {
-    res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
-    res.end(JSON.stringify(data, (_, v) => normalizeValue(v)));
+    sendJson(res, 200, Buffer.from(JSON.stringify(data, (_, v) => normalizeValue(v))));
 }
 function jsonError(res, status, message) {
     if (res.headersSent)
         return;
-    res.writeHead(status, { 'Content-Type': 'application/json' });
-    res.end(JSON.stringify({ error: message }));
+    sendJson(res, status, Buffer.from(JSON.stringify({ error: message })));
 }
 /** Proxy a request to the user's PDS with DPoP + automatic nonce retry + token refresh. */
 async function proxyToPds(oauthConfig, session, method, pdsUrl, body) {

package/dist/test.js

@@ -14,14 +14,14 @@ import { validateLexicons } from '@bigmoves/lexicon';
 import { packCursor, unpackCursor, isTakendownDid, filterTakendownDids } from "./db.js";
 import { seed as createSeedHelpers } from "./seed.js";
 /**
- * Find the project's config.yaml by walking up from cwd.
- * Returns the resolved config path, or falls back to 'config.yaml'.
+ * Find the project's hatk.config.ts by walking up from cwd.
+ * Returns the resolved config path, or falls back to 'hatk.config.ts'.
  */
 function findConfigPath() {
     const explicit = process.env.APPVIEW_CONFIG;
     if (explicit)
         return resolve(explicit);
-    return resolve('config.yaml');
+    return resolve('hatk.config.ts');
 }
 /**
  * Boot an in-memory hatk context for unit tests.
@@ -34,7 +34,7 @@ function findConfigPath() {
  */
 export async function createTestContext() {
     const configPath = findConfigPath();
-    const config = loadConfig(configPath);
+    const config = await loadConfig(configPath);
     const configDir = dirname(resolve(configPath));
     configureRelay(config.relay);
     // Load and validate lexicons

package/dist/vite-plugin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vite-plugin.d.ts","sourceRoot":"","sources":["../src/vite-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAA;AAKlC,wBAAgB,IAAI,CAAC,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAsFrD"}
+{"version":3,"file":"vite-plugin.d.ts","sourceRoot":"","sources":["../src/vite-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAA;AAKlC,wBAAgB,IAAI,CAAC,IAAI,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAuFrD"}

package/dist/vite-plugin.js

@@ -64,13 +64,14 @@ export function hatk(opts) {
             const mainPath = resolve(import.meta.dirname, 'main.js');
             const watchDirs = ['xrpc', 'feeds', 'labels', 'jobs', 'setup', 'lexicons'].filter((d) => existsSync(d));
             const watchArgs = watchDirs.flatMap((d) => ['--watch-path', d]);
-            serverProcess = spawn('npx', ['tsx', 'watch', ...watchArgs, mainPath, 'config.yaml'], {
+            serverProcess = spawn('npx', ['tsx', 'watch', ...watchArgs, mainPath, 'hatk.config.ts'], {
                 stdio: 'inherit',
                 cwd: process.cwd(),
                 env: {
                     ...process.env,
                     PORT: String(backendPort),
                     OAUTH_ISSUER: process.env.OAUTH_ISSUER || issuer,
+                    DEV_MODE: '1',
                 },
             });
             server.httpServer?.on('close', () => {

package/package.json

@@ -1,9 +1,15 @@
 {
   "name": "@hatk/hatk",
-  "version": "0.0.1-alpha.2",
+  "version": "0.0.1-alpha.21",
+  "license": "MIT",
   "bin": {
     "hatk": "dist/cli.js"
   },
+  "files": [
+    "dist",
+    "fonts",
+    "public"
+  ],
   "type": "module",
   "exports": {
     "./feeds": "./dist/feeds.js",
@@ -17,9 +23,9 @@
     "./setup": "./dist/setup.js",
     "./test": "./dist/test.js",
     "./test/browser": "./dist/test-browser.js",
+    "./config": "./dist/config.js",
     "./vite-plugin": "./dist/vite-plugin.js"
   },
-  "files": ["dist", "fonts", "public"],
   "scripts": {
     "build": "tsc -p tsconfig.build.json",
     "prepublishOnly": "npm run build"

package/public/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Allow: /