@hatk/hatk 0.0.1-alpha.7 → 0.0.1-alpha.70

package/dist/indexer.js

@@ -1,24 +1,30 @@
 import { cborDecode } from "./cbor.js";
 import { parseCarFrame } from "./car.js";
-import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses } from "./db.js";
+import { isPrivateCollection } from "./private-collections.js";
+import { insertRecord, deleteRecord, setCursor, setRepoStatus, getRepoRetryInfo, listAllRepoStatuses, getDatabasePort, updateRepoHandle, } from "./database/db.js";
 import { backfillRepo } from "./backfill.js";
-import { rebuildAllIndexes } from "./fts.js";
+import { rebuildAllIndexes } from "./database/fts.js";
 import { log, emit, timer } from "./logger.js";
 import { runLabelRules } from "./labels.js";
-import { getLexiconArray } from "./schema.js";
+import { fireOnCommitHooks } from "./hooks.js";
+import { getLexiconArray } from "./database/schema.js";
 import { validateRecord } from '@bigmoves/lexicon';
 let buffer = [];
 let flushTimer = null;
 let lastSeq = null;
+let lastPersistedSeq = null;
+let cursorCheckpointTimer = null;
 const BATCH_SIZE = 100;
 const FLUSH_INTERVAL_MS = 500;
+const CURSOR_CHECKPOINT_INTERVAL_MS = 5_000;
 let writesSinceRebuild = 0;
 let ftsRebuildInterval = 500;
 // Event buffer for DIDs mid-backfill
 const pendingBuffers = new Map();
 // Track in-flight backfills to avoid duplicates
 const backfillInFlight = new Set();
-const MAX_CONCURRENT_BACKFILLS = 5;
+const backfillPromises = new Map();
+const pendingReschedule = new Set();
 // In-memory cache of repo status to avoid flooding the DB read queue
 const repoStatusCache = new Map();
 // Set by startIndexer
@@ -27,6 +33,13 @@ let indexerSignalCollections;
 let indexerPinnedRepos = null;
 let indexerFetchTimeout;
 let indexerMaxRetries;
+let indexerPlcUrl;
+let maxConcurrentBackfills = 3;
+/**
+ * Flush the write buffer — insert all buffered records, update the relay cursor,
+ * run label rules on inserted records, and trigger FTS rebuilds when the write
+ * threshold is reached. Emits a wide event with batch stats.
+ */
 async function flushBuffer() {
     if (buffer.length === 0)
         return;
@@ -47,8 +60,10 @@ async function flushBuffer() {
         }
     }
     if (lastSeq !== null) {
+        const seq = lastSeq;
         try {
-            await setCursor('relay', String(lastSeq));
+            await setCursor('relay', String(seq));
+            lastPersistedSeq = seq;
         }
         catch (err) {
             cursorError = err.message;
@@ -64,6 +79,14 @@ async function flushBuffer() {
             value: item.record,
         }).catch(() => { });
     }
+    // Fire on-commit hooks for inserted records (async, non-blocking)
+    fireOnCommitHooks(inserted.map((item) => ({
+        action: 'create',
+        collection: item.collection,
+        uri: item.uri,
+        authorDid: item.authorDid,
+        record: item.record,
+    })));
     // Aggregate collection counts and unique DIDs for wide event
     const collections = {};
     const dids = new Set();
@@ -86,9 +109,14 @@ async function flushBuffer() {
     writesSinceRebuild += batch.length;
     if (writesSinceRebuild >= ftsRebuildInterval) {
         writesSinceRebuild = 0;
-        rebuildAllIndexes([...indexerCollections]).catch(() => { });
+        // Skip periodic full rebuild for SQLite — it uses incremental FTS updates
+        const port = getDatabasePort();
+        if (port.dialect !== 'sqlite') {
+            rebuildAllIndexes([...indexerCollections]).catch(() => { });
+        }
     }
 }
+/** Schedule a flush after FLUSH_INTERVAL_MS if one isn't already pending. */
 function scheduleFlush() {
     if (flushTimer)
         return;
@@ -97,6 +125,7 @@ function scheduleFlush() {
         await flushBuffer();
     }, FLUSH_INTERVAL_MS);
 }
+/** Add a record to the write buffer. Flushes immediately if BATCH_SIZE is reached. */
 function bufferWrite(item) {
     buffer.push(item);
     if (buffer.length >= BATCH_SIZE) {
@@ -110,11 +139,79 @@ function bufferWrite(item) {
         scheduleFlush();
     }
 }
+/** Record the latest firehose sequence seen. Source of truth for cursor persistence and reconnect resume. */
+export function noteSeq(seq) {
+    lastSeq = seq;
+}
+/**
+ * Persist the latest firehose sequence when it has advanced past the stored
+ * cursor. Runs on a timer independent of the write path: an app whose
+ * collections rarely appear on the firehose never flushes a batch, so the
+ * flush-path cursor write alone leaves the stored cursor frozen — and a frozen
+ * cursor makes the relay replay its whole retention window at line rate
+ * (~135 GB/hour observed against bsky.network) on every boot and reconnect.
+ */
+export async function checkpointCursor() {
+    if (lastSeq === null || lastSeq === lastPersistedSeq)
+        return;
+    const seq = lastSeq;
+    try {
+        await setCursor('relay', String(seq));
+        lastPersistedSeq = seq;
+    }
+    catch (err) {
+        emit('indexer', 'cursor_checkpoint_error', { cursor_seq: seq, error: err.message });
+    }
+}
+/**
+ * The cursor a reconnect should resume from: the latest seq this process has
+ * seen, falling back to the boot-time cursor before any message has arrived.
+ * Reusing the boot-time cursor on every reconnect would replay everything
+ * received since the process started.
+ */
+export function resumeCursor(liveSeq, bootCursor) {
+    return liveSeq !== null ? String(liveSeq) : bootCursor;
+}
+/** Reset module cursor state between tests. */
+export function _resetCursorStateForTests() {
+    lastSeq = null;
+    lastPersistedSeq = null;
+}
+/**
+ * Auto-backfill a DID's repo when first seen on the firehose.
+ *
+ * Fetches the full repo via CAR export, inserts all records, then replays any
+ * firehose events that arrived during the backfill. Concurrency is capped at
+ * `maxConcurrentBackfills`. Failed backfills retry with exponential delay up
+ * to `maxRetries`.
+ */
+/** Wait for a DID's backfill to complete if one is in flight. */
+export function awaitBackfill(did) {
+    const entry = backfillPromises.get(did);
+    return entry ? entry.promise : Promise.resolve();
+}
 export async function triggerAutoBackfill(did, attempt = 0) {
     if (backfillInFlight.has(did))
         return;
+    if (backfillInFlight.size >= maxConcurrentBackfills) {
+        if (!pendingReschedule.has(did)) {
+            pendingReschedule.add(did);
+            setTimeout(() => {
+                pendingReschedule.delete(did);
+                triggerAutoBackfill(did, attempt);
+            }, 10_000);
+        }
+        return;
+    }
     backfillInFlight.add(did);
     pendingBuffers.set(did, []);
+    if (!backfillPromises.has(did)) {
+        let resolveBackfill;
+        const promise = new Promise((r) => {
+            resolveBackfill = r;
+        });
+        backfillPromises.set(did, { promise, resolve: resolveBackfill });
+    }
     if (attempt === 0)
         await setRepoStatus(did, 'pending');
     const elapsed = timer();
@@ -154,6 +251,12 @@ export async function triggerAutoBackfill(did, attempt = 0) {
         error,
         retry_count: currentRetryCount,
     });
+    // Resolve awaiting callers (e.g. on-login hooks)
+    const entry = backfillPromises.get(did);
+    if (entry) {
+        entry.resolve();
+        backfillPromises.delete(did);
+    }
     if (status === 'error' && currentRetryCount < indexerMaxRetries) {
         const delaySecs = Math.min(currentRetryCount * 60, 3600);
         const delayMs = Math.max(delaySecs, 60) * 1000;
@@ -162,7 +265,7 @@ export async function triggerAutoBackfill(did, attempt = 0) {
         }, delayMs);
     }
 }
-// Periodic memory diagnostics
+/** Emit a memory diagnostics wide event every 30s for observability. */
 function startMemoryDiagnostics() {
     setInterval(() => {
         const mem = process.memoryUsage();
@@ -184,6 +287,16 @@ function startMemoryDiagnostics() {
         });
     }, 30_000);
 }
+/**
+ * Connect to the AT Protocol relay firehose and begin indexing.
+ *
+ * Opens a WebSocket to `subscribeRepos`, processes commit messages synchronously
+ * on the event loop to minimize backpressure, and batches writes through
+ * {@link flushBuffer}. New DIDs trigger auto-backfill via {@link triggerAutoBackfill}.
+ * Reconnects automatically on disconnect after a 3s delay.
+ *
+ * @returns The WebSocket connection (for shutdown coordination)
+ */
 export async function startIndexer(opts) {
     const { relayUrl, collections, cursor, fetchTimeout } = opts;
     if (opts.ftsRebuildInterval != null)
@@ -193,6 +306,8 @@ export async function startIndexer(opts) {
     indexerPinnedRepos = opts.pinnedRepos || null;
     indexerFetchTimeout = fetchTimeout;
     indexerMaxRetries = opts.maxRetries;
+    indexerPlcUrl = opts.plcUrl;
+    maxConcurrentBackfills = opts.parallelism ?? 3;
     // Pre-populate repo status cache from DB so non-signal updates
     // (e.g. profile changes) are processed for already-tracked DIDs
     if (repoStatusCache.size === 0) {
@@ -202,7 +317,14 @@ export async function startIndexer(opts) {
         }
         log(`[indexer] Warmed repo status cache with ${statuses.length} entries`);
     }
-    startMemoryDiagnostics();
+    // startMemoryDiagnostics()
+    // Checkpoint the cursor on a timer regardless of write activity (see
+    // checkpointCursor). Guarded so reconnects don't stack intervals; unref'd so
+    // it never keeps the process alive.
+    if (!cursorCheckpointTimer) {
+        cursorCheckpointTimer = setInterval(() => void checkpointCursor(), CURSOR_CHECKPOINT_INTERVAL_MS);
+        cursorCheckpointTimer.unref?.();
+    }
     let wsUrl = `${relayUrl}/xrpc/com.atproto.sync.subscribeRepos`;
     if (cursor) {
         wsUrl += `?cursor=${cursor}`;
@@ -220,27 +342,95 @@ export async function startIndexer(opts) {
             const bytes = new Uint8Array(event.data);
             processMessage(bytes, collections);
         }
-        catch {
-            // Skip unparseable firehose messages silently
+        catch (err) {
+            emit('indexer', 'decode_error', { error: err instanceof Error ? err.message : String(err) });
         }
     });
     ws.addEventListener('open', () => log('[indexer] Connected to relay'));
     ws.addEventListener('close', () => {
         log('[indexer] Disconnected, reconnecting in 3s...');
-        setTimeout(() => startIndexer(opts), 3000);
+        setTimeout(() => startIndexer({ ...opts, cursor: resumeCursor(lastSeq, opts.cursor) }), 3000);
     });
     return ws;
 }
+/**
+ * Handle a `#identity` firehose event for a DID. The `handle` field on the
+ * event is optional per the lexicon, and some emitters omit it (signalling
+ * "re-resolve"). When absent, we re-resolve from the PLC directory so handle
+ * renames propagate even when the relay payload is sparse.
+ *
+ * Only updates DIDs we already track (present in repoStatusCache) to avoid
+ * writing rows for the entire network.
+ */
+async function handleIdentityEvent(did, payloadHandle) {
+    if (!repoStatusCache.has(did))
+        return;
+    let handle = payloadHandle;
+    const payloadHadHandle = handle !== undefined;
+    if (!handle) {
+        try {
+            // Bound the PLC fetch so a slow plc.directory can't pile up unbounded
+            // promises during an identity-event burst (fire-and-forget caller).
+            const res = await fetch(`${indexerPlcUrl}/${did}`, {
+                signal: AbortSignal.timeout(indexerFetchTimeout * 1000),
+            });
+            if (res.ok) {
+                const doc = (await res.json());
+                // First at:// entry is the canonical handle (per @atproto/identity convention)
+                const aka = doc.alsoKnownAs?.find((u) => u.startsWith('at://'));
+                handle = aka ? aka.slice('at://'.length) : undefined;
+            }
+            else {
+                emit('indexer', 'identity_resolve_error', { did, status: res.status });
+            }
+        }
+        catch (err) {
+            emit('indexer', 'identity_resolve_error', {
+                did,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    if (!handle) {
+        emit('indexer', 'identity_no_handle', { did, payload_had_handle: payloadHadHandle });
+        return;
+    }
+    try {
+        await updateRepoHandle(did, handle);
+        emit('indexer', 'identity_handle_update', { did, handle, payload_had_handle: payloadHadHandle });
+    }
+    catch (err) {
+        emit('indexer', 'identity_update_error', {
+            did,
+            handle,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+}
+/**
+ * Process a single firehose message. Decodes the CBOR header/body, filters
+ * for relevant collections, validates records against lexicons, and routes
+ * writes to the buffer (or pending buffer if the DID is mid-backfill).
+ */
 function processMessage(bytes, collections) {
     const header = cborDecode(bytes, 0);
     const body = cborDecode(bytes, header.offset);
+    // Handle identity events (handle changes). Fire-and-forget — keeps
+    // processMessage synchronous so the WS event loop drains without backpressure.
+    if (header.value.t === '#identity') {
+        const did = typeof body.value.did === 'string' ? body.value.did : undefined;
+        const handle = typeof body.value.handle === 'string' ? body.value.handle : undefined;
+        if (did)
+            handleIdentityEvent(did, handle);
+        return;
+    }
     if (header.value.op !== 1 || header.value.t !== '#commit')
         return;
     if (!body.value.blocks || !body.value.ops)
         return;
     // Track sequence number for cursor
     if (body.value.seq)
-        lastSeq = body.value.seq;
+        noteSeq(body.value.seq);
     const did = body.value.repo;
     if (!did)
         return;
@@ -248,7 +438,13 @@ function processMessage(bytes, collections) {
     if (indexerPinnedRepos && !indexerPinnedRepos.has(did))
         return;
     // Check if any ops in this commit are for collections we care about
-    const relevantOps = body.value.ops.filter((op) => collections.has(op.path.split('/')[0]));
+    // Private collections are AppView-authoritative: nothing on the network may
+    // write or delete their rows, so firehose ops naming them are spoofed by
+    // definition and must be dropped.
+    const relevantOps = body.value.ops.filter((op) => {
+        const collection = op.path.split('/')[0];
+        return collections.has(collection) && !isPrivateCollection(collection);
+    });
     if (relevantOps.length === 0)
         return;
     // Copy blocks out of the original buffer before it can be GC'd
@@ -264,7 +460,7 @@ function processMessage(bytes, collections) {
         repoStatusCache.set(did, 'unknown');
     }
     if (hasSignalOp && (!indexerPinnedRepos || indexerPinnedRepos.has(did))) {
-        if (repoStatus === null && backfillInFlight.size < MAX_CONCURRENT_BACKFILLS) {
+        if (repoStatus === null && backfillInFlight.size < maxConcurrentBackfills) {
             repoStatusCache.set(did, 'pending');
             triggerAutoBackfill(did);
         }
@@ -283,34 +479,46 @@ function processMessage(bytes, collections) {
         const uri = `at://${did}/${op.path}`;
         if (op.action === 'delete') {
             deleteRecord(collection, uri);
+            fireOnCommitHooks([
+                {
+                    action: 'delete',
+                    collection,
+                    uri,
+                    authorDid: did,
+                    record: null,
+                },
+            ]);
             continue;
         }
-        for (const [cid, data] of blocks) {
-            try {
-                const { value: record } = cborDecode(data);
-                if (record?.$type === collection) {
-                    const validationError = validateRecord(getLexiconArray(), collection, record);
-                    if (validationError) {
-                        emit('indexer', 'validation_skip', {
-                            uri,
-                            collection,
-                            path: validationError.path,
-                            error: validationError.message,
-                        });
-                        break;
-                    }
-                    const item = { collection, uri, cid, authorDid: did, record };
-                    // If DID is mid-backfill, buffer instead of writing directly
-                    if (pendingBuffers.has(did)) {
-                        pendingBuffers.get(did).push(item);
-                    }
-                    else {
-                        bufferWrite(item);
-                    }
-                    break;
+        const opCid = typeof op.cid === 'string' ? op.cid : op.cid?.$link;
+        if (!opCid)
+            continue;
+        const data = blocks.get(opCid);
+        if (!data)
+            continue;
+        try {
+            const { value: record } = cborDecode(data);
+            if (record?.$type === collection) {
+                const validationError = validateRecord(getLexiconArray(), collection, record);
+                if (validationError) {
+                    emit('indexer', 'validation_skip', {
+                        uri,
+                        collection,
+                        path: validationError.path,
+                        error: validationError.message,
+                    });
+                    continue;
+                }
+                const item = { collection, uri, cid: opCid, authorDid: did, record };
+                // If DID is mid-backfill, buffer instead of writing directly
+                if (pendingBuffers.has(did)) {
+                    pendingBuffers.get(did).push(item);
+                }
+                else {
+                    bufferWrite(item);
                 }
             }
-            catch { }
         }
+        catch { }
     }
 }

package/dist/labels.d.ts

@@ -13,7 +13,36 @@ export interface LabelRuleContext {
         value: Record<string, any>;
     };
 }
+export interface LabelModule {
+    definition?: LabelDefinition;
+    evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
+}
+export declare function defineLabel(module: LabelModule): {
+    definition?: LabelDefinition;
+    evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
+    __type: "labels";
+};
+/**
+ * Discover and load label rule modules from the `labels/` directory.
+ *
+ * Each module should default-export an object with an optional `definition`
+ * (label metadata like severity and blur behavior) and an optional `evaluate`
+ * function that returns label values to apply to a record.
+ *
+ * @param labelsDir - Absolute path to the `labels/` directory
+ */
 export declare function initLabels(labelsDir: string): Promise<void>;
+/** Clear all registered label definitions and rules (for hot-reload). */
+export declare function clearLabels(): void;
+/** Register a single label module from a scanned server/ module. */
+export declare function registerLabelModule(name: string, labelMod: {
+    definition?: LabelDefinition;
+    evaluate?: (ctx: LabelRuleContext) => Promise<string[]>;
+}): void;
+/**
+ * Evaluate all loaded label rules against a record and persist any resulting labels.
+ * Called after each record is indexed. Rule errors are logged but never block indexing.
+ */
 export declare function runLabelRules(record: {
     uri: string;
     cid: string;
@@ -21,9 +50,16 @@ export declare function runLabelRules(record: {
     collection: string;
     value: Record<string, any>;
 }): Promise<void>;
+/**
+ * Re-evaluate all label rules against every existing record in the given collections.
+ * Used by `/admin/rescan-labels` to apply new or updated rules retroactively.
+ *
+ * @returns Count of records scanned and new labels applied
+ */
 export declare function rescanLabels(collections: string[]): Promise<{
     scanned: number;
     labeled: number;
 }>;
+/** Return all label definitions discovered during {@link initLabels}. */
 export declare function getLabelDefinitions(): LabelDefinition[];
 //# sourceMappingURL=labels.d.ts.map

package/dist/labels.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"labels.d.ts","sourceRoot":"","sources":["../src/labels.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC3B,CAAA;CACF;AAWD,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjE;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC3B,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhB;AAED,wBAAsB,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAuCvG;AAED,wBAAgB,mBAAmB,IAAI,eAAe,EAAE,CAEvD"}
+{"version":3,"file":"labels.d.ts","sourceRoot":"","sources":["../src/labels.ts"],"names":[],"mappings":"AA8BA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,wDAAwD;AACxD,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE;QACF,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACtD,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;KACtD,CAAA;IACD,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC3B,CAAA;CACF;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,CAAC,EAAE,eAAe,CAAA;IAC5B,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CACxD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW;iBAJhC,eAAe;eACjB,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;;EAKxD;AAYD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjE;AAED,yEAAyE;AACzE,wBAAgB,WAAW,IAAI,IAAI,CAGlC;AAED,oEAAoE;AACpE,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE;IAAE,UAAU,CAAC,EAAE,eAAe,CAAC;IAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,gBAAgB,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CAAE,GAClG,IAAI,CAON;AAED;;;GAGG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC1C,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC3B,GAAG,OAAO,CAAC,IAAI,CAAC,CAyBhB;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAuCvG;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,IAAI,eAAe,EAAE,CAEvD"}

package/dist/labels.js

@@ -6,13 +6,53 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
     }
     return path;
 };
+/**
+ * Label system for applying moderation labels to records as they are indexed.
+ *
+ * Place label modules in the `labels/` directory. Each module default-exports
+ * an object with a `definition` (label metadata) and/or an `evaluate` function
+ * (rule that returns label values for a given record).
+ *
+ * @example
+ * ```ts
+ * // labels/nsfw.ts
+ * import type { LabelRuleContext } from '@hatk/hatk/labels'
+ *
+ * export default {
+ *   definition: {
+ *     identifier: 'nsfw',
+ *     severity: 'alert',
+ *     blurs: 'media',
+ *     defaultSetting: 'warn',
+ *     locales: [{ lang: 'en', name: 'NSFW', description: 'Not safe for work' }],
+ *   },
+ *
+ *   async evaluate(ctx: LabelRuleContext): Promise<string[]> {
+ *     if (ctx.record.value.nsfw === true) return ['nsfw']
+ *     return []
+ *   },
+ * }
+ * ```
+ */
 import { resolve } from 'node:path';
 import { readdirSync } from 'node:fs';
-import { querySQL, runSQL, insertLabels, getSchema } from "./db.js";
+import { querySQL, runSQL, insertLabels, getSchema } from "./database/db.js";
 import { log, emit } from "./logger.js";
+export function defineLabel(module) {
+    return { __type: 'labels', ...module };
+}
 const rules = [];
 let labelDefs = [];
 let labelSrc = 'self';
+/**
+ * Discover and load label rule modules from the `labels/` directory.
+ *
+ * Each module should default-export an object with an optional `definition`
+ * (label metadata like severity and blur behavior) and an optional `evaluate`
+ * function that returns label values to apply to a record.
+ *
+ * @param labelsDir - Absolute path to the `labels/` directory
+ */
 export async function initLabels(labelsDir) {
     let files;
     try {
@@ -26,7 +66,7 @@ export async function initLabels(labelsDir) {
     for (const file of files) {
         const name = file.replace(/\.(ts|js)$/, '');
         const scriptPath = resolve(labelsDir, file);
-        const mod = await import(__rewriteRelativeImportExtension(scriptPath));
+        const mod = await import(__rewriteRelativeImportExtension(/* @vite-ignore */ `${scriptPath}?t=${Date.now()}`));
         const handler = mod.default;
         if (handler.definition) {
             labelDefs.push(handler.definition);
@@ -45,6 +85,24 @@ export async function initLabels(labelsDir) {
         log(`[labels] ${labelDefs.length} label definitions loaded`);
     }
 }
+/** Clear all registered label definitions and rules (for hot-reload). */
+export function clearLabels() {
+    labelDefs.length = 0;
+    rules.length = 0;
+}
+/** Register a single label module from a scanned server/ module. */
+export function registerLabelModule(name, labelMod) {
+    if (labelMod.definition) {
+        labelDefs.push(labelMod.definition);
+    }
+    if (labelMod.evaluate) {
+        rules.push({ name, evaluate: labelMod.evaluate });
+    }
+}
+/**
+ * Evaluate all loaded label rules against a record and persist any resulting labels.
+ * Called after each record is indexed. Rule errors are logged but never block indexing.
+ */
 export async function runLabelRules(record) {
     if (rules.length === 0)
         return;
@@ -69,15 +127,21 @@ export async function runLabelRules(record) {
         emit('labels', 'applied', { count: allLabels.length, uri: record.uri, vals: allLabels.map((l) => l.val) });
     }
 }
+/**
+ * Re-evaluate all label rules against every existing record in the given collections.
+ * Used by `/admin/rescan-labels` to apply new or updated rules retroactively.
+ *
+ * @returns Count of records scanned and new labels applied
+ */
 export async function rescanLabels(collections) {
-    const beforeRows = await querySQL(`SELECT COUNT(*) as count FROM _labels`);
+    const beforeRows = (await querySQL(`SELECT COUNT(*) as count FROM _labels`));
     const beforeCount = Number(beforeRows[0]?.count || 0);
     let scanned = 0;
     for (const collection of collections) {
         const schema = getSchema(collection);
         if (!schema)
             continue;
-        const rows = await querySQL(`SELECT * FROM ${schema.tableName}`);
+        const rows = (await querySQL(`SELECT * FROM ${schema.tableName}`));
         for (const row of rows) {
             scanned++;
             const value = {};
@@ -85,7 +149,7 @@ export async function rescanLabels(collections) {
                 let v = row[col.name];
                 if (v === null || v === undefined)
                     continue;
-                if (col.duckdbType === 'JSON' && typeof v === 'string') {
+                if (col.isJson && typeof v === 'string') {
                     try {
                         v = JSON.parse(v);
                     }
@@ -102,10 +166,11 @@ export async function rescanLabels(collections) {
             });
         }
     }
-    const afterRows = await querySQL(`SELECT COUNT(*) as count FROM _labels`);
+    const afterRows = (await querySQL(`SELECT COUNT(*) as count FROM _labels`));
     const afterCount = Number(afterRows[0]?.count || 0);
     return { scanned, labeled: afterCount - beforeCount };
 }
+/** Return all label definitions discovered during {@link initLabels}. */
 export function getLabelDefinitions() {
     return labelDefs;
 }

package/dist/lexicon-resolve.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lexicon-resolve.d.ts","sourceRoot":"","sources":["../src/lexicon-resolve.ts"],"names":[],"mappings":"AAsMA,UAAU,OAAO;IACf,OAAO,EAAE,MAAM,CAAA;IACf,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CACnB;AA+DD;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAgChF"}
+{"version":3,"file":"lexicon-resolve.d.ts","sourceRoot":"","sources":["../src/lexicon-resolve.ts"],"names":[],"mappings":"AAoFA,UAAU,OAAO;IACf,OAAO,EAAE,MAAM,CAAA;IACf,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CACnB;AA2FD;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAgChF"}