@hatk/hatk 0.0.1-alpha.11 → 0.0.1-alpha.111

package/dist/lexicons/dev/hatk/getRecord.json

@@ -0,0 +1,26 @@
+{
+  "lexicon": 1,
+  "id": "dev.hatk.getRecord",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "Fetch a single record by AT URI.",
+      "parameters": {
+        "type": "params",
+        "required": ["uri"],
+        "properties": {
+          "uri": { "type": "string", "format": "at-uri" }
+        }
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "properties": {
+            "record": { "type": "unknown" }
+          }
+        }
+      }
+    }
+  }
+}

package/dist/lexicons/dev/hatk/getRecords.json

@@ -0,0 +1,32 @@
+{
+  "lexicon": 1,
+  "id": "dev.hatk.getRecords",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "List records from a collection with optional filters.",
+      "parameters": {
+        "type": "params",
+        "required": ["collection"],
+        "properties": {
+          "collection": { "type": "string" },
+          "limit": { "type": "integer", "minimum": 1, "maximum": 100, "default": 20 },
+          "cursor": { "type": "string" },
+          "sort": { "type": "string" },
+          "order": { "type": "string" }
+        }
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": ["items"],
+          "properties": {
+            "items": { "type": "array", "items": { "type": "unknown" } },
+            "cursor": { "type": "string" }
+          }
+        }
+      }
+    }
+  }
+}

package/dist/lexicons/dev/hatk/putPreference.json

@@ -0,0 +1,28 @@
+{
+  "lexicon": 1,
+  "id": "dev.hatk.putPreference",
+  "defs": {
+    "main": {
+      "type": "procedure",
+      "description": "Set a single preference by key.",
+      "input": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": ["key", "value"],
+          "properties": {
+            "key": { "type": "string" },
+            "value": { "type": "unknown" }
+          }
+        }
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "properties": {}
+        }
+      }
+    }
+  }
+}

package/dist/lexicons/dev/hatk/putRecord.json

@@ -0,0 +1,41 @@
+{
+  "lexicon": 1,
+  "id": "dev.hatk.putRecord",
+  "defs": {
+    "main": {
+      "type": "procedure",
+      "description": "Create or update a record via the user's PDS.",
+      "input": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": ["collection", "rkey", "record"],
+          "properties": {
+            "collection": { "type": "string" },
+            "rkey": { "type": "string" },
+            "record": { "type": "unknown" },
+            "repo": { "type": "string", "format": "did" }
+          }
+        }
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "properties": {
+            "uri": { "type": "string", "format": "at-uri" },
+            "cid": { "type": "string", "format": "cid" },
+            "commit": {
+              "type": "object",
+              "properties": {
+                "cid": { "type": "string", "format": "cid" },
+                "rev": { "type": "string" }
+              }
+            },
+            "validationStatus": { "type": "string" }
+          }
+        }
+      }
+    }
+  }
+}

package/dist/lexicons/dev/hatk/searchRecords.json

@@ -0,0 +1,32 @@
+{
+  "lexicon": 1,
+  "id": "dev.hatk.searchRecords",
+  "defs": {
+    "main": {
+      "type": "query",
+      "description": "Full-text search across a collection.",
+      "parameters": {
+        "type": "params",
+        "required": ["collection", "q"],
+        "properties": {
+          "collection": { "type": "string" },
+          "q": { "type": "string", "description": "Search query" },
+          "limit": { "type": "integer", "minimum": 1, "maximum": 100, "default": 20 },
+          "cursor": { "type": "string" },
+          "fuzzy": { "type": "boolean", "default": true }
+        }
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": ["items"],
+          "properties": {
+            "items": { "type": "array", "items": { "type": "unknown" } },
+            "cursor": { "type": "string" }
+          }
+        }
+      }
+    }
+  }
+}

package/dist/lexicons/dev/hatk/uploadBlob.json

@@ -0,0 +1,23 @@
+{
+  "lexicon": 1,
+  "id": "dev.hatk.uploadBlob",
+  "defs": {
+    "main": {
+      "type": "procedure",
+      "description": "Upload a blob via the user's PDS.",
+      "input": {
+        "encoding": "*/*"
+      },
+      "output": {
+        "encoding": "application/json",
+        "schema": {
+          "type": "object",
+          "required": ["blob"],
+          "properties": {
+            "blob": { "type": "blob" }
+          }
+        }
+      }
+    }
+  }
+}

package/dist/logger.d.ts

@@ -1,4 +1,33 @@
+/**
+ * Unstructured debug log — use sparingly for human-readable dev output.
+ * Prefer {@link emit} for anything that should be queryable in production.
+ * Disabled when `DEBUG=0`.
+ */
 export declare function log(...args: unknown[]): void;
+/**
+ * Emit a structured wide event as a single JSON line to stdout.
+ *
+ * Each call produces one canonical log line with a timestamp, module, operation,
+ * and arbitrary key-value fields — designed for columnar search and aggregation,
+ * not string grep. Pack as much context as possible into `fields` (request IDs,
+ * durations, status codes, user DIDs, counts) so a single event tells the full
+ * story. See https://loggingsucks.com for the philosophy behind this approach.
+ *
+ * Disabled when `DEBUG=0`.
+ *
+ * @param module - Subsystem emitting the event (e.g. "server", "indexer", "backfill")
+ * @param op - Operation name (e.g. "request", "commit", "memory")
+ * @param fields - High-cardinality key-value context — include everything relevant
+ */
 export declare function emit(module: string, op: string, fields: Record<string, unknown>): void;
+/**
+ * Start a millisecond timer. Call the returned function to get elapsed ms.
+ * Use with {@link emit} to add `duration_ms` to wide events.
+ *
+ * @example
+ * const elapsed = timer()
+ * await doWork()
+ * emit('server', 'request', { path, status_code, duration_ms: elapsed() })
+ */
 export declare function timer(): () => number;
 //# sourceMappingURL=logger.d.ts.map

package/dist/logger.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,wBAAgB,GAAG,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAG5C;AAED,wBAAgB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAWtF;AAED,wBAAgB,KAAK,IAAI,MAAM,MAAM,CAGpC"}
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,GAAG,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAG5C;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAWtF;AAED;;;;;;;;GAQG;AACH,wBAAgB,KAAK,IAAI,MAAM,MAAM,CAGpC"}

package/dist/logger.js

@@ -1,8 +1,28 @@
+/**
+ * Unstructured debug log — use sparingly for human-readable dev output.
+ * Prefer {@link emit} for anything that should be queryable in production.
+ * Disabled when `DEBUG=0`.
+ */
 export function log(...args) {
     if (process.env.DEBUG === '0')
         return;
     console.log(...args);
 }
+/**
+ * Emit a structured wide event as a single JSON line to stdout.
+ *
+ * Each call produces one canonical log line with a timestamp, module, operation,
+ * and arbitrary key-value fields — designed for columnar search and aggregation,
+ * not string grep. Pack as much context as possible into `fields` (request IDs,
+ * durations, status codes, user DIDs, counts) so a single event tells the full
+ * story. See https://loggingsucks.com for the philosophy behind this approach.
+ *
+ * Disabled when `DEBUG=0`.
+ *
+ * @param module - Subsystem emitting the event (e.g. "server", "indexer", "backfill")
+ * @param op - Operation name (e.g. "request", "commit", "memory")
+ * @param fields - High-cardinality key-value context — include everything relevant
+ */
 export function emit(module, op, fields) {
     if (process.env.DEBUG === '0')
         return;
@@ -17,6 +37,15 @@ export function emit(module, op, fields) {
     }
     process.stdout.write(JSON.stringify(entry) + '\n');
 }
+/**
+ * Start a millisecond timer. Call the returned function to get elapsed ms.
+ * Use with {@link emit} to add `duration_ms` to wide events.
+ *
+ * @example
+ * const elapsed = timer()
+ * await doWork()
+ * emit('server', 'request', { path, status_code, duration_ms: elapsed() })
+ */
 export function timer() {
     const start = performance.now();
     return () => Math.round(performance.now() - start);

package/dist/main.js

@@ -1,32 +1,60 @@
 #!/usr/bin/env node
-import { mkdirSync } from 'node:fs';
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { mkdirSync, writeFileSync, existsSync } from 'node:fs';
 import { dirname, resolve } from 'node:path';
+import { registerHatkResolveHook } from "./resolve-hatk.js";
 import { log } from "./logger.js";
 import { loadConfig } from "./config.js";
-import { loadLexicons, storeLexicons, discoverCollections, generateTableSchema, generateCreateTableSQL, } from "./schema.js";
+import { validatableLexicons, loadLexicons, storeLexicons, discoverCollections, buildSchemas, } from "./database/schema.js";
 import { discoverViews } from "./views.js";
-import { initDatabase, getCursor, querySQL, backfillChildTables } from "./db.js";
+import { initDatabase, getCursor, querySQL, getSqlDialect, getSchemaDump } from "./database/db.js";
+import { createAdapter } from "./database/adapter-factory.js";
+import { getDialect } from "./database/dialect.js";
+import { setSearchPort } from "./database/fts.js";
 import { initFeeds, listFeeds } from "./feeds.js";
-import { initXrpc, listXrpc, configureRelay } from "./xrpc.js";
+import { initXrpc, listXrpc, configureRelay, configureCdn, configureOAuth, callXrpc } from "./xrpc.js";
 import { initOpengraph } from "./opengraph.js";
 import { initLabels, getLabelDefinitions } from "./labels.js";
-import { startIndexer } from "./indexer.js";
-import { rebuildAllIndexes } from "./fts.js";
-import { startServer } from "./server.js";
+import { initClassifiers, listClassifiers, configureClef, startScoringOnIndex } from "./classifiers.js";
+import { auxCursorKey, jetstreamCursorKey, startAuxIndexer, startIndexer, sweepReferences } from "./indexer.js";
+import { startJetstreamIndexer } from "./jetstream.js";
+import { rebuildAllIndexes } from "./database/fts.js";
+import { createHandler, registerCoreHandlers } from "./server.js";
+import { setPrivateCollections } from "./private-collections.js";
+import { serve } from "./adapter.js";
 import { validateLexicons } from '@bigmoves/lexicon';
 import { relayHttpUrl } from "./config.js";
-import { runBackfill } from "./backfill.js";
+import { runBackfill, configurePlc, configureSkipPds } from "./backfill.js";
 import { initOAuth } from "./oauth/server.js";
-import { loadOnLoginHook } from "./oauth/hooks.js";
+import { parseSessionCookie, getSessionCookieName } from "./oauth/session.js";
+import { loadOnLoginHook } from "./hooks.js";
+import { enabledPushTransports, initPush, isPushEnabled } from "./push.js";
 import { initSetup } from "./setup.js";
-const configPath = process.argv[2] || 'config.yaml';
+import { initServer } from "./server-init.js";
+import { startSpaces } from "./spaces/index.js";
+import { configureSpaceIdentity } from "./spaces/identity.js";
+const configPath = process.argv[2] || 'hatk.config.ts';
 const configDir = dirname(resolve(configPath));
+registerHatkResolveHook();
 // 1. Load config
-const config = loadConfig(configPath);
+const config = await loadConfig(configPath);
 configureRelay(config.relay);
+configurePlc(config.plc);
+configureSkipPds(config.backfill.skipPds);
+// Space reads resolve authorities too, and ctx.spaceCredential works without
+// `spaces` indexing being configured.
+configureSpaceIdentity(config.plc);
+configureCdn(config.cdn);
 // 2. Load lexicons, validate schemas, and discover collections
 const lexicons = loadLexicons(resolve(configDir, 'lexicons'));
-const lexiconErrors = validateLexicons([...lexicons.values()]);
+const lexiconErrors = validateLexicons(validatableLexicons(lexicons));
 if (lexiconErrors) {
     for (const [nsid, errors] of Object.entries(lexiconErrors)) {
         for (const err of errors) {
@@ -44,44 +72,74 @@ if (collections.length === 0) {
 log(`[main] Loaded config: ${collections.length} collections`);
 // Discover view defs from lexicons
 discoverViews();
-await loadOnLoginHook(resolve(configDir, 'hooks'));
-const schemas = [];
-const ddlStatements = [];
-for (const nsid of collections) {
-    const lexicon = lexicons.get(nsid);
-    if (!lexicon) {
-        log(`[main] No lexicon found for ${nsid}, using generic JSON storage`);
-        const genericDDL = `CREATE TABLE IF NOT EXISTS "${nsid}" (
-      uri TEXT PRIMARY KEY,
-      cid TEXT,
-      did TEXT NOT NULL,
-      indexed_at TIMESTAMP NOT NULL,
-      data JSON
-    );
-    CREATE INDEX IF NOT EXISTS idx_${nsid.replace(/\./g, '_')}_indexed ON "${nsid}"(indexed_at DESC);
-    CREATE INDEX IF NOT EXISTS idx_${nsid.replace(/\./g, '_')}_author ON "${nsid}"(did);`;
-        schemas.push({ collection: nsid, tableName: `"${nsid}"`, columns: [], refColumns: [], children: [], unions: [] });
-        ddlStatements.push(genericDDL);
-        continue;
+const engineDialect = getDialect(config.databaseEngine);
+const { schemas, ddlStatements, indexStatements } = buildSchemas(lexicons, collections, engineDialect);
+for (const s of schemas) {
+    if (s.columns.length === 0) {
+        log(`[main] No lexicon found for ${s.collection}, using generic JSON storage`);
+    }
+    else {
+        log(`[main] Schema for ${s.collection}: ${s.columns.length} columns, ${s.unions.length} unions`);
     }
-    const schema = generateTableSchema(nsid, lexicon, lexicons);
-    schemas.push(schema);
-    ddlStatements.push(generateCreateTableSQL(schema));
-    log(`[main] Schema for ${nsid}: ${schema.columns.length} columns, ${schema.unions.length} unions`);
 }
-// 3. Ensure data directory exists and initialize DuckDB
+// 3. Ensure data directory exists and initialize database
 if (config.database !== ':memory:') {
     mkdirSync(dirname(config.database), { recursive: true });
 }
-await initDatabase(config.database, schemas, ddlStatements);
-log(`[main] DuckDB initialized (${config.database === ':memory:' ? 'in-memory' : config.database})`);
-// 3a. Backfill child tables for decomposed arrays (one-time migration)
-await backfillChildTables();
-// 3b. Run setup hooks (after DB init, before server)
-await initSetup(resolve(configDir, 'setup'));
+const { adapter, searchPort } = await createAdapter(config.databaseEngine);
+setSearchPort(searchPort);
+// Migrates as part of opening, between creating the tables and indexing them.
+const migrationChanges = await initDatabase(adapter, config.database, schemas, ddlStatements, indexStatements);
+log(`[main] Database initialized (${config.databaseEngine}, ${config.database === ':memory:' ? 'in-memory' : config.database})`);
+if (migrationChanges.length > 0) {
+    log(`[main] Applied ${migrationChanges.length} schema migration(s)`);
+}
+// 3b. Run setup hooks, feeds, xrpc, og, labels
+const serverDir = resolve(configDir, 'server');
+if (existsSync(serverDir)) {
+    // New: single server/ directory
+    await initServer(serverDir);
+}
+else {
+    // Legacy: separate directories
+    await initSetup(resolve(configDir, 'setup'));
+    await loadOnLoginHook(resolve(configDir, 'hooks'));
+    await initFeeds(resolve(configDir, 'feeds'));
+    log(`[main] Feeds initialized: ${listFeeds()
+        .map((f) => f.name)
+        .join(', ') || 'none'}`);
+    await initXrpc(resolve(configDir, 'xrpc'));
+    log(`[main] XRPC handlers initialized: ${listXrpc().join(', ') || 'none'}`);
+    await initOpengraph(resolve(configDir, 'og'));
+    log(`[main] OpenGraph initialized`);
+    await initLabels(resolve(configDir, 'labels'));
+    log(`[main] Labels initialized: ${getLabelDefinitions().length} definitions`);
+    await initClassifiers(resolve(configDir, 'classifiers'));
+    log(`[main] Classifiers initialized: ${listClassifiers().join(', ') || 'none'}`);
+}
+// Register built-in dev.hatk.* handlers so callXrpc() can find them
+configureClef(config.clef && {
+    apiToken: config.clef.apiToken,
+    model: config.clef.model,
+    endpoint: `https://api.cloudflare.com/client/v4/accounts/${config.clef.accountId}/ai/run/@cf/cloudflare/${config.clef.model}`,
+});
+if (config.clef?.scoreOnIndex && listClassifiers().length)
+    await startScoringOnIndex();
+setPrivateCollections(config.privateCollections);
+registerCoreHandlers(collections, config.oauth);
+configureOAuth(config.oauth);
+// Write db/schema.sql (after setup, so setup-created tables are included)
+try {
+    const schemaDir = resolve(configDir, 'db');
+    mkdirSync(schemaDir, { recursive: true });
+    const schemaDump = await getSchemaDump();
+    writeFileSync(resolve(schemaDir, 'schema.sql'), `-- This file is auto-generated by hatk on startup. Do not edit.\n-- Database engine: ${config.databaseEngine}\n\n${schemaDump}\n`);
+    log(`[main] Schema written to db/schema.sql`);
+}
+catch { }
 // Detect orphaned tables
 try {
-    const existingTables = await querySQL(`SELECT table_name FROM information_schema.tables WHERE table_schema = 'main' AND table_name NOT LIKE '\\_%' ESCAPE '\\'`);
+    const existingTables = (await querySQL(getSqlDialect().listTablesQuery));
     for (const row of existingTables) {
         const tableName = row.table_name;
         const isChildTable = collections.some((c) => tableName.startsWith(c + '__'));
@@ -91,24 +149,78 @@ try {
     }
 }
 catch { }
-// 4. Initialize feeds, xrpc handlers, og, labels from directories
-await initFeeds(resolve(configDir, 'feeds'));
-log(`[main] Feeds initialized: ${listFeeds()
-    .map((f) => f.name)
-    .join(', ') || 'none'}`);
-await initXrpc(resolve(configDir, 'xrpc'));
-log(`[main] XRPC handlers initialized: ${listXrpc().join(', ') || 'none'}`);
-await initOpengraph(resolve(configDir, 'og'));
-log(`[main] OpenGraph initialized`);
-await initLabels(resolve(configDir, 'labels'));
-log(`[main] Labels initialized: ${getLabelDefinitions().length} definitions`);
 if (config.oauth) {
     await initOAuth(config.oauth, config.plc, config.relay);
     log(`[main] OAuth initialized (issuer: ${config.oauth.issuer})`);
 }
+if (config.push) {
+    initPush(config.push, configDir);
+    if (isPushEnabled()) {
+        log(`[main] Push initialized (${enabledPushTransports().join(', ')})`);
+    }
+    else {
+        log(`[main] Push configured but no usable credential — push disabled`);
+    }
+}
 // 5. Start server immediately (don't wait for backfill)
 const collectionSet = new Set(collections);
-startServer(config.port, collections, config.publicDir, config.oauth, config.admins);
+const backfillOpts = {
+    pdsUrl: relayHttpUrl(config.relay),
+    extraPdsUrls: config.relays.map(relayHttpUrl),
+    plcUrl: config.plc,
+    collections: collectionSet,
+    config: config.backfill,
+};
+function runBackfillAndRestart() {
+    runBackfill(backfillOpts)
+        .then(async (recordCount) => {
+        log('[main] Backfill complete, building FTS indexes...');
+        await rebuildAllIndexes(collections);
+        log('[main] FTS indexes ready');
+        return recordCount;
+    })
+        .then((recordCount) => {
+        if (recordCount > 0 && !process.env.DEV_MODE) {
+            log('[main] Restarting to reclaim memory...');
+            process.exit(1);
+        }
+    })
+        .catch((err) => {
+        console.error('[main] Backfill error:', err.message);
+    });
+}
+const handler = createHandler({
+    collections,
+    publicDir: config.publicDir,
+    oauth: config.oauth,
+    admins: config.admins,
+    adminToken: config.adminToken,
+    onResync: runBackfillAndRestart,
+    ...(config.spaces?.serviceDid
+        ? {
+            spaces: {
+                serviceDid: config.spaces.serviceDid,
+                serviceFragment: config.spaces.serviceFragment ?? 'atproto_space_syncer',
+                // The origin an authority delivers to. The OAuth issuer is already
+                // the public origin an operator has had to get right, so it is the
+                // honest default rather than a second thing to configure wrong.
+                ...(config.oauth?.issuer ? { publicUrl: config.oauth.issuer } : {}),
+            },
+        }
+        : {}),
+});
+globalThis.__hatk_callXrpc = callXrpc;
+globalThis.__hatk_parseSessionCookie = parseSessionCookie;
+globalThis.__hatk_sessionCookieName = getSessionCookieName();
+// Detect SvelteKit build output and use it as fallback handler
+let fallback = undefined;
+const sveltekitHandler = resolve(configDir, 'build', 'handler.js');
+if (existsSync(sveltekitHandler)) {
+    const sk = await import(__rewriteRelativeImportExtension(/* @vite-ignore */ sveltekitHandler));
+    fallback = sk.handler;
+    log(`[main] SvelteKit handler loaded from build/handler.js`);
+}
+serve(handler, config.port, undefined, fallback);
 log(`\nhatk running:`);
 log(`  Relay: ${config.relay}`);
 log(`  Database: ${config.database}`);
@@ -118,32 +230,57 @@ log(`  Feeds: ${listFeeds()
     .map((f) => f.name)
     .join(', ')}`);
 // 6. Start indexer with cursor
-const cursor = await getCursor('relay');
-startIndexer({
-    relayUrl: config.relay,
+// HATK_IGNORE_SAVED_CURSOR=1 boots from the live firehose instead of the saved
+// cursor — the escape hatch when a stale cursor would make the relay replay its
+// retention window at line rate (see checkpointCursor in indexer.ts). Unset it
+// after one boot so later restarts resume normally.
+const ignoreSavedCursor = Boolean(process.env.HATK_IGNORE_SAVED_CURSOR);
+if (ignoreSavedCursor)
+    log('[main] HATK_IGNORE_SAVED_CURSOR set — starting indexer from live');
+// Each source keeps its own cursor row: relay seqs and Jetstream seqs are
+// different coordinate systems, so switching sources must not resume one from
+// the other's value.
+const indexerCore = {
+    plcUrl: config.plc,
     collections: collectionSet,
     signalCollections: config.backfill.signalCollections ? new Set(config.backfill.signalCollections) : undefined,
+    references: config.backfill.references,
     pinnedRepos: config.backfill.repos ? new Set(config.backfill.repos) : undefined,
-    cursor,
     fetchTimeout: config.backfill.fetchTimeout,
     maxRetries: config.backfill.maxRetries,
     parallelism: config.backfill.parallelism,
     ftsRebuildInterval: config.ftsRebuildInterval,
-});
+};
+if (config.jetstream) {
+    const cursor = ignoreSavedCursor ? null : await getCursor(jetstreamCursorKey(config.jetstream.url));
+    startJetstreamIndexer({ ...indexerCore, jetstreamUrl: config.jetstream.url, cursor });
+}
+else {
+    const cursor = ignoreSavedCursor ? null : await getCursor('relay');
+    startIndexer({ ...indexerCore, relayUrl: config.relay, cursor });
+    for (const relayUrl of config.relays) {
+        const auxCursor = ignoreSavedCursor ? null : await getCursor(auxCursorKey(relayUrl));
+        startAuxIndexer({ relayUrl, collections: collectionSet, cursor: auxCursor });
+    }
+}
+// What is already indexed may name repos too; a reference configured after
+// the fact reaches them here rather than on their next write.
+void sweepReferences();
 // 7. Run backfill in background
-runBackfill({
-    pdsUrl: relayHttpUrl(config.relay),
-    plcUrl: config.plc,
-    collections: collectionSet,
-    config: config.backfill,
-})
-    .then(() => {
-    log('[main] Backfill complete, rebuilding FTS indexes...');
-    return rebuildAllIndexes(collections);
-})
-    .then(() => {
-    log('[main] FTS indexes ready');
-})
-    .catch((err) => {
-    console.error('[main] Backfill error:', err.message);
+runBackfillAndRestart();
+// 8. Follow permissioned spaces, if this instance indexes any. Started after
+// the firehose and backfill because it is neither: a space has no stream to
+// tail, so it is a sweep on a timer rather than a source to resume.
+if (config.spaces) {
+    startSpaces({
+        spaces: config.spaces,
+        oauth: config.oauth,
+        plc: config.plc,
+        collections: collectionSet,
+    });
+}
+// Graceful shutdown
+process.on('SIGTERM', () => {
+    log('[main] Received SIGTERM, shutting down...');
+    process.exit(0);
 });

package/dist/mst.d.ts

@@ -1,7 +1,22 @@
+/** A single entry from a Merkle Search Tree — a record path paired with its content CID. */
 export interface MstEntry {
+    /** Record path, e.g. "xyz.marketplace.listing/3mfniulnr7c2g" */
     path: string;
+    /** CID of the record's CBOR block */
     cid: string;
 }
+/**
+ * Walk an AT Protocol Merkle Search Tree (MST) in key order, yielding every record entry.
+ *
+ * The MST is a prefix-compressed B+ tree used by AT Protocol repositories to map
+ * record paths to CIDs. Each node contains a left subtree pointer, an array of entries
+ * (each with a prefix length, key suffix, value CID, and right subtree pointer), and
+ * keys are reconstructed by combining the prefix of the previous key with the suffix.
+ *
+ * @param blocks - Block store that resolves CIDs to raw CBOR bytes
+ * @param rootCid - CID of the MST root node
+ * @yields {MstEntry} Record entries in lexicographic key order
+ */
 export declare function walkMst(blocks: {
     get(cid: string): Uint8Array | undefined;
 }, rootCid: string): Generator<MstEntry>;

package/dist/mst.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,wBAAiB,OAAO,CAAC,MAAM,EAAE;IAAE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAA;CAAE,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CA8BnH"}
+{"version":3,"file":"mst.d.ts","sourceRoot":"","sources":["../src/mst.ts"],"names":[],"mappings":"AAEA,4FAA4F;AAC5F,MAAM,WAAW,QAAQ;IACvB,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAA;IACZ,qCAAqC;IACrC,GAAG,EAAE,MAAM,CAAA;CACZ;AAED;;;;;;;;;;;GAWG;AACH,wBAAiB,OAAO,CAAC,MAAM,EAAE;IAAE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAA;CAAE,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CA+BnH"}