@raybase/core 0.0.2 → 0.3.4

package/dist/search.js

@@ -0,0 +1,785 @@
+import { normalizeNamedDefinitions, stableJson } from "./internal.js";
+import { nowIso, randomId } from "./sql.js";
+import { chunkRetrievalText } from "./retrieval.js";
+export const raybaseSearchDocumentsTableName = "_search_documents";
+export const raybaseSearchReindexRunsTableName = "_search_reindex_runs";
+export const raybaseSearchFtsTableName = "_search_fts";
+export const raybaseSearchRebuildStateKey = "search.fts.rebuildRequired";
+export const raybaseSearchSchemaSql = `
+CREATE TABLE IF NOT EXISTS "_search_documents" (
+  "rowid" INTEGER PRIMARY KEY AUTOINCREMENT,
+  "id" TEXT NOT NULL UNIQUE,
+  "indexName" TEXT NOT NULL,
+  "sourceType" TEXT NOT NULL,
+  "sourceTable" TEXT,
+  "sourceId" TEXT NOT NULL,
+  "sourceVersion" TEXT,
+  "chunkKey" TEXT NOT NULL,
+  "chunkOrdinal" INTEGER NOT NULL,
+  "title" TEXT NOT NULL,
+  "body" TEXT NOT NULL,
+  "url" TEXT,
+  "textHash" TEXT NOT NULL,
+  "visibility" TEXT NOT NULL,
+  "metadata" TEXT,
+  "status" TEXT NOT NULL,
+  "lastError" TEXT,
+  "indexedAt" TEXT,
+  "updatedAt" TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS "_search_documents_source_idx"
+  ON "_search_documents" ("sourceType", "sourceTable", "sourceId");
+CREATE INDEX IF NOT EXISTS "_search_documents_index_idx"
+  ON "_search_documents" ("indexName", "status", "updatedAt");
+CREATE VIRTUAL TABLE IF NOT EXISTS "_search_fts" USING fts5(
+  "title",
+  "body",
+  content='_search_documents',
+  content_rowid='rowid',
+  tokenize='unicode61'
+);
+CREATE TABLE IF NOT EXISTS "_search_reindex_runs" (
+  "id" TEXT PRIMARY KEY NOT NULL,
+  "indexName" TEXT NOT NULL,
+  "status" TEXT NOT NULL,
+  "actorId" TEXT,
+  "actorKind" TEXT,
+  "totalSources" INTEGER NOT NULL DEFAULT 0,
+  "totalChunks" INTEGER NOT NULL DEFAULT 0,
+  "upsertedChunks" INTEGER NOT NULL DEFAULT 0,
+  "deletedChunks" INTEGER NOT NULL DEFAULT 0,
+  "skippedChunks" INTEGER NOT NULL DEFAULT 0,
+  "errorCount" INTEGER NOT NULL DEFAULT 0,
+  "startedAt" TEXT NOT NULL,
+  "completedAt" TEXT,
+  "summaryJson" TEXT
+);
+CREATE INDEX IF NOT EXISTS "_search_reindex_runs_index_started_idx"
+  ON "_search_reindex_runs" ("indexName", "startedAt");
+`;
+const searchNamePattern = /^[A-Za-z][A-Za-z0-9_.:-]{0,127}$/;
+const defaultSearchChunkMaxChars = 2000;
+const defaultSearchChunkOverlapChars = 0;
+const defaultSearchSnippetChars = 240;
+const ftsShadowTables = new Set([
+    "_search_fts_data",
+    "_search_fts_idx",
+    "_search_fts_content",
+    "_search_fts_docsize",
+    "_search_fts_config"
+]);
+export function defineSearchIndex(definition) {
+    validateSearchName(definition.name);
+    validateSearchIndex(definition);
+    return { ...definition };
+}
+export function defineSearch(definition) {
+    const indexes = normalizeNamedDefinitions("search index", definition.indexes, validateSearchName);
+    for (const index of Object.values(indexes))
+        validateSearchIndex(index);
+    return { ...definition, indexes };
+}
+function validateSearchIndex(index) {
+    if (!index.sources.length)
+        throw new Error(`Search index ${index.name} must declare at least one source`);
+    if (index.tokenizer && index.tokenizer !== "unicode61")
+        throw new Error(`Search index ${index.name} only supports the unicode61 tokenizer in v1`);
+    if (index.ranking && index.ranking !== "bm25")
+        throw new Error(`Search index ${index.name} only supports bm25 ranking in v1`);
+}
+export function tableSearchSource(table, options) {
+    return { kind: "table", table, ...options };
+}
+export function corpusSearchSource(name, options) {
+    validateSearchName(name);
+    return { kind: "corpus", name, ...options };
+}
+export function validateSearch(search, schema) {
+    if (!search)
+        return;
+    for (const index of Object.values(search.indexes)) {
+        validateSearchName(index.name);
+        if (!index.sources.length)
+            throw new Error(`Search index ${index.name} must declare at least one source`);
+        for (const source of index.sources) {
+            if (source.kind === "table") {
+                tableKeyFor(schema, source.table);
+            }
+            else {
+                validateSearchName(source.name);
+            }
+        }
+    }
+}
+export function validateSearchName(name) {
+    if (!searchNamePattern.test(name)) {
+        throw new Error(`Invalid Raybase search index name "${name}". Use letters, numbers, "_", "-", ".", or ":".`);
+    }
+}
+export function isRaybaseSearchInternalTable(name) {
+    return name === raybaseSearchFtsTableName
+        || ftsShadowTables.has(name)
+        || name === "_agent_chapter_fts"
+        || name.startsWith("_agent_chapter_fts_");
+}
+export function isRaybaseSearchTable(name) {
+    return name === raybaseSearchDocumentsTableName || name === raybaseSearchReindexRunsTableName || isRaybaseSearchInternalTable(name);
+}
+export function parseSearchQuery(query) {
+    const raw = query.trim();
+    if (!raw)
+        return null;
+    const clauses = [];
+    const operators = [];
+    let pendingOperator = null;
+    for (const token of searchQueryTokens(raw)) {
+        if (token.kind === "operator") {
+            if (!clauses.length)
+                continue;
+            if (token.value === "NOT") {
+                pendingOperator = null;
+                continue;
+            }
+            pendingOperator = token.value;
+            continue;
+        }
+        if (clauses.length >= 12)
+            break;
+        if (clauses.length)
+            operators.push(pendingOperator ?? "AND");
+        clauses.push(token);
+        pendingOperator = null;
+    }
+    if (!clauses.length)
+        return null;
+    const finalClause = clauses.at(-1);
+    if (finalClause?.prefixMatch)
+        finalClause.match = finalClause.prefixMatch;
+    const terms = clauses.flatMap((clause) => clause.terms).slice(0, 12);
+    let match = clauses[0]?.match ?? "";
+    for (let index = 1; index < clauses.length; index += 1) {
+        match += ` ${operators[index - 1] ?? "AND"} ${clauses[index]?.match ?? ""}`;
+    }
+    return { raw, terms, match };
+}
+function searchQueryTokens(raw) {
+    const tokens = [];
+    const pattern = /"([^"]*)"|([\p{L}\p{N}_]+)(\*)?/gu;
+    for (const found of raw.matchAll(pattern)) {
+        const phrase = found[1];
+        if (phrase !== undefined) {
+            const words = searchWords(phrase, { minLength: 1 });
+            const terms = words.filter((term) => term.length > 1);
+            if (!terms.length)
+                continue;
+            tokens.push({ kind: "phrase", match: quoteFtsPhrase(words), terms });
+            continue;
+        }
+        const word = found[2]?.toLowerCase();
+        if (!word || word.length <= 1)
+            continue;
+        if (word === "and" || word === "or" || word === "not") {
+            tokens.push({ kind: "operator", value: word.toUpperCase() });
+            continue;
+        }
+        const prefixMatch = `${word}*`;
+        tokens.push({ kind: "term", match: found[3] === "*" ? prefixMatch : quoteFtsPhrase([word]), prefixMatch, terms: [word] });
+    }
+    return tokens;
+}
+function searchWords(value, options = {}) {
+    const minLength = options.minLength ?? 2;
+    return Array.from(value.matchAll(/[\p{L}\p{N}_]+/gu))
+        .map((match) => match[0].toLowerCase())
+        .filter((term) => term.length >= minLength);
+}
+function quoteFtsPhrase(terms) {
+    return `"${terms.join(" ").replace(/"/g, '""')}"`;
+}
+export async function searchDocumentIdForChunk(input) {
+    const digest = await sha256Base64Url([input.indexName, input.sourceType, input.sourceTable ?? "", input.sourceId, input.chunkKey].join("|"));
+    return `search_${digest.slice(0, 40)}`;
+}
+export async function chunkSearchText(text, options = {}) {
+    return chunkRetrievalText(text, {
+        maxChars: options.maxChars ?? defaultSearchChunkMaxChars,
+        overlapChars: options.overlapChars ?? defaultSearchChunkOverlapChars
+    });
+}
+export function simpleSearchSnippet(text, terms, maxChars = defaultSearchSnippetChars) {
+    const normalized = text.replace(/\s+/g, " ").trim();
+    if (!normalized)
+        return "";
+    const lower = normalized.toLowerCase();
+    const first = terms
+        .map((term) => lower.indexOf(term.toLowerCase()))
+        .filter((position) => position >= 0)
+        .sort((left, right) => left - right)[0] ?? 0;
+    const half = Math.floor(maxChars / 2);
+    const start = Math.max(0, first - half);
+    const end = Math.min(normalized.length, start + maxChars);
+    const prefix = start > 0 ? "..." : "";
+    const suffix = end < normalized.length ? "..." : "";
+    return `${prefix}${normalized.slice(start, end).trim()}${suffix}`;
+}
+export function createRaybaseSearchClient(options = {}) {
+    return {
+        async search(input) {
+            const request = typeof input === "string" ? { query: input } : input;
+            const fetcher = options.fetch ?? fetch;
+            const baseUrl = options.baseUrl?.replace(/\/$/, "") ?? "";
+            const url = new URL(`${baseUrl}/api/search`, "http://raybase.local");
+            url.searchParams.set("q", request.query);
+            const index = request.index ?? options.defaultIndex;
+            if (index)
+                url.searchParams.set("index", index);
+            if (request.limit !== undefined)
+                url.searchParams.set("limit", String(request.limit));
+            if (request.sourceTable)
+                url.searchParams.append("sourceTable", request.sourceTable);
+            for (const sourceTable of request.sourceTables ?? [])
+                url.searchParams.append("sourceTable", sourceTable);
+            const href = options.baseUrl && /^https?:\/\//.test(options.baseUrl) ? url.toString() : `${url.pathname}${url.search}`;
+            const response = await fetcher(href, { credentials: "same-origin" });
+            const body = await response.json().catch(() => ({}));
+            if (!response.ok) {
+                const message = body && typeof body === "object" && typeof body.error === "string"
+                    ? body.error
+                    : `Search request failed with ${response.status}`;
+                throw new Error(message);
+            }
+            return {
+                results: Array.isArray(body.results) ? body.results : [],
+                query: typeof body.query === "string" ? body.query : request.query
+            };
+        }
+    };
+}
+export async function ensureSearchSchema(control) {
+    for (const statement of splitSqlStatements(raybaseSearchSchemaSql))
+        await control.run(statement);
+}
+export async function reindexSearchIndex(input) {
+    await ensureSearchSchema(input.control);
+    const runId = randomId("search_reindex");
+    const startedAt = nowIso();
+    const counts = { totalSources: 0, totalChunks: 0, upsertedChunks: 0, deletedChunks: 0, skippedChunks: 0, errorCount: 0 };
+    await input.control.run(`INSERT INTO "_search_reindex_runs"
+     ("id", "indexName", "status", "actorId", "actorKind", "startedAt", "summaryJson")
+     VALUES (?, ?, 'running', ?, ?, ?, ?)`, [runId, input.index.name, input.actorId ?? null, input.actorKind ?? "operator", startedAt, JSON.stringify({ sources: input.index.sources.length })]);
+    try {
+        for (const source of input.index.sources) {
+            if (source.kind === "table") {
+                const activeDocumentIds = new Set();
+                const tableKey = tableKeyFor(input.schema, source.table);
+                let offset = 0;
+                let remaining = source.limit ?? Number.POSITIVE_INFINITY;
+                while (remaining > 0) {
+                    const limit = Math.min(500, remaining);
+                    const rows = await input.db.list(tableKey, { limit, offset });
+                    if (!rows.rows.length)
+                        break;
+                    counts.totalSources += rows.rows.length;
+                    for (const row of rows.rows) {
+                        const result = await indexTableSearchRow({
+                            control: input.control,
+                            index: input.index,
+                            source,
+                            row
+                        });
+                        result.documentIds.forEach((id) => activeDocumentIds.add(id));
+                        counts.totalChunks += result.indexed;
+                        counts.upsertedChunks += result.upserted;
+                        counts.skippedChunks += result.skipped;
+                        counts.errorCount += result.errors;
+                    }
+                    offset += rows.rows.length;
+                    remaining -= rows.rows.length;
+                    if (rows.rows.length < limit)
+                        break;
+                }
+                const existing = await input.control.all(`SELECT "id" FROM "_search_documents"
+           WHERE "indexName" = ? AND "sourceType" = 'table' AND "sourceTable" = ? AND "status" != 'deleted'`, [input.index.name, source.table.name]);
+                const stale = existing.filter((row) => !activeDocumentIds.has(row.id));
+                for (const row of stale) {
+                    await deleteSearchDocumentById(input.control, row.id);
+                    counts.deletedChunks += 1;
+                }
+            }
+            else {
+                const result = await indexCorpusSearchSource({
+                    control: input.control,
+                    index: input.index,
+                    source
+                });
+                counts.totalSources += result.sources;
+                counts.totalChunks += result.indexed;
+                counts.upsertedChunks += result.upserted;
+                counts.deletedChunks += result.deleted;
+                counts.skippedChunks += result.skipped;
+                counts.errorCount += result.errors;
+            }
+        }
+        await completeSearchReindexRun(input.control, runId, "succeeded", counts, {});
+        return {
+            ok: true,
+            runId,
+            indexed: counts.totalChunks,
+            upserted: counts.upsertedChunks,
+            deleted: counts.deletedChunks,
+            skipped: counts.skippedChunks,
+            errors: counts.errorCount
+        };
+    }
+    catch (error) {
+        counts.errorCount += 1;
+        await completeSearchReindexRun(input.control, runId, "failed", counts, { error: error instanceof Error ? error.message : String(error) });
+        throw error;
+    }
+}
+export async function updateSearchForDataChange(input) {
+    if (!input.search)
+        return;
+    await ensureSearchSchema(input.control);
+    for (const index of Object.values(input.search.indexes)) {
+        for (const source of index.sources) {
+            if (source.kind !== "table" || source.table.name !== input.change.tableName)
+                continue;
+            if (input.change.op === "delete") {
+                await deleteSearchDocumentsForSource(input.control, {
+                    indexName: index.name,
+                    sourceType: "table",
+                    sourceTable: source.table.name,
+                    sourceId: input.change.rowId
+                });
+                continue;
+            }
+            const row = input.change.newRow;
+            if (!row)
+                continue;
+            await indexTableSearchRow({ control: input.control, index, source, row });
+        }
+    }
+}
+export async function searchIndex(input) {
+    if (!input.search)
+        return { results: [], query: parseSearchQuery(input.query) };
+    const indexName = input.indexName || "app";
+    const index = input.search.indexes[indexName];
+    if (!index)
+        throw new Error(`Unknown search index: ${indexName}`);
+    const requestedSourceTables = input.sourceTables?.filter(Boolean) ?? [];
+    const selectedTableSources = index.sources.filter((source) => source.kind === "table" && (!requestedSourceTables.length || requestedSourceTables.includes(source.table.name)));
+    for (const source of selectedTableSources) {
+        await input.db.assertReadable(tableKeyFor(input.schema, source.table));
+    }
+    await ensureSearchSchema(input.control);
+    const query = parseSearchQuery(input.query);
+    if (!query)
+        return { results: [], query };
+    const limit = Math.min(Math.max(input.limit ?? 20, 1), 50);
+    const params = [query.match, index.name, "indexed"];
+    const where = [`_search_fts MATCH ?`, `d."indexName" = ?`, `d."status" = ?`];
+    if (requestedSourceTables.length) {
+        where.push(`d."sourceTable" IN (${requestedSourceTables.map(() => "?").join(", ")})`);
+        params.push(...requestedSourceTables);
+    }
+    if (input.publicOnly) {
+        where.push(`d."visibility" = 'public'`);
+    }
+    const candidates = await input.control.all(`SELECT d."id", d."indexName", d."sourceType", d."sourceTable", d."sourceId", d."sourceVersion", d."chunkKey",
+            d."chunkOrdinal", d."title", d."body", d."url", d."textHash", d."visibility", d."metadata",
+            bm25(_search_fts) AS "rank"
+     FROM "_search_fts"
+     JOIN "_search_documents" d ON d."rowid" = _search_fts.rowid
+     WHERE ${where.join(" AND ")}
+     ORDER BY "rank" ASC
+     LIMIT ?`, [...params, Math.min(limit * 4, 200)]).catch(() => []);
+    const results = [];
+    const seenRows = new Set();
+    for (const candidate of candidates) {
+        if (results.length >= limit)
+            break;
+        const sourceType = candidate.sourceType === "corpus" ? "corpus" : "table";
+        const sourceTable = asNullableString(candidate.sourceTable);
+        const sourceId = String(candidate.sourceId ?? "");
+        if (sourceType === "table") {
+            if (!sourceTable)
+                continue;
+            const source = findTableSearchSource(index, sourceTable);
+            if (!source)
+                continue;
+            const rowKey = `${sourceTable}:${sourceId}`;
+            if (seenRows.has(rowKey))
+                continue;
+            const tableKey = tableKeyFor(input.schema, source.table);
+            const row = await input.db.get(tableKey, sourceId);
+            if (!row)
+                continue;
+            const indexed = await sourceDocumentsForTableRow(index, source, row);
+            const current = indexed.find((document) => document.id === candidate.id || document.chunkKey === candidate.chunkKey);
+            if (!current || current.textHash !== candidate.textHash) {
+                await input.control.run(`UPDATE "_search_documents" SET "status" = 'pending', "lastError" = ?, "updatedAt" = ? WHERE "id" = ?`, ["Source row changed after indexing", nowIso(), String(candidate.id)]);
+                continue;
+            }
+            seenRows.add(rowKey);
+            results.push({
+                id: String(candidate.id),
+                indexName: index.name,
+                sourceType,
+                sourceTable,
+                sourceId,
+                title: current.title,
+                url: current.url ?? null,
+                snippet: simpleSearchSnippet(current.body, query.terms, index.maxSnippetChars ?? defaultSearchSnippetChars),
+                metadata: current.metadata,
+                visibility: current.visibility
+            });
+        }
+        else {
+            results.push({
+                id: String(candidate.id),
+                indexName: index.name,
+                sourceType,
+                sourceTable: null,
+                sourceId,
+                title: String(candidate.title ?? ""),
+                url: asNullableString(candidate.url),
+                snippet: simpleSearchSnippet(String(candidate.body ?? ""), query.terms, index.maxSnippetChars ?? defaultSearchSnippetChars),
+                metadata: parseJson(candidate.metadata),
+                visibility: candidate.visibility === "public" ? "public" : "private"
+            });
+        }
+    }
+    return { results, query };
+}
+export async function rebuildSearchFts(input) {
+    await ensureSearchSchema(input.control);
+    if (!input.indexName) {
+        await input.control.run(`INSERT INTO "_search_fts"("_search_fts") VALUES('rebuild')`);
+        return { ok: true, rebuilt: -1 };
+    }
+    const docs = await input.control.all(`SELECT "rowid", "title", "body" FROM "_search_documents" WHERE "indexName" = ? AND "status" = 'indexed'`, [input.indexName]);
+    for (const doc of docs) {
+        await input.control.run(`INSERT OR REPLACE INTO "_search_fts" ("rowid", "title", "body") VALUES (?, ?, ?)`, [
+            Number(doc.rowid),
+            String(doc.title ?? ""),
+            String(doc.body ?? "")
+        ]);
+    }
+    return { ok: true, rebuilt: docs.length };
+}
+export async function clearSearchIndex(input) {
+    await ensureSearchSchema(input.control);
+    const docs = await input.control.all(`SELECT "id" FROM "_search_documents" WHERE "indexName" = ?`, [input.indexName]);
+    for (const doc of docs)
+        await deleteSearchDocumentById(input.control, doc.id);
+    return { ok: true, deleted: docs.length };
+}
+export async function searchIndexesSummary(input) {
+    await ensureSearchSchema(input.control).catch(() => undefined);
+    const rebuildRequired = Boolean(await readSearchRebuildRequired(input.control));
+    const indexes = [];
+    for (const index of Object.values(input.search?.indexes ?? {})) {
+        const countRows = await input.control.all(`SELECT "status", COUNT(*) as "count" FROM "_search_documents" WHERE "indexName" = ? GROUP BY "status"`, [index.name]).catch(() => []);
+        const lastRun = await input.control.first(`SELECT * FROM "_search_reindex_runs" WHERE "indexName" = ? ORDER BY "startedAt" DESC LIMIT 1`, [index.name]).catch(() => null);
+        indexes.push({
+            name: index.name,
+            title: index.title,
+            description: index.description,
+            sources: index.sources.map((source) => source.kind === "table"
+                ? { kind: "table", table: source.table.name, label: source.table.label ?? null }
+                : { kind: "corpus", name: source.name, title: source.title ?? null }),
+            counts: Object.fromEntries(countRows.map((row) => [String(row.status), Number(row.count)])),
+            lastRun,
+            rebuildRequired
+        });
+    }
+    return { indexes, rebuildRequired };
+}
+export async function searchIndexDetail(input) {
+    const summary = await searchIndexesSummary(input);
+    const index = summary.indexes.find((item) => item.name === input.indexName);
+    if (!index)
+        throw new Error(`Unknown search index: ${input.indexName}`);
+    const recentErrors = await input.control.all(`SELECT "id", "sourceType", "sourceTable", "sourceId", "chunkKey", "status", "lastError", "updatedAt"
+     FROM "_search_documents"
+     WHERE "indexName" = ? AND ("status" = 'error' OR "lastError" IS NOT NULL)
+     ORDER BY "updatedAt" DESC LIMIT 25`, [index.name]).catch(() => []);
+    const recentRuns = await searchReindexRunsQuery({ control: input.control, indexName: index.name, limit: 25 });
+    return { ...index, recentErrors, recentRuns: recentRuns.runs };
+}
+export async function searchReindexRunsQuery(input) {
+    await ensureSearchSchema(input.control).catch(() => undefined);
+    const where = [];
+    const params = [];
+    if (input.indexName) {
+        where.push(`"indexName" = ?`);
+        params.push(input.indexName);
+    }
+    if (input.status) {
+        where.push(`"status" = ?`);
+        params.push(input.status);
+    }
+    const whereSql = where.length ? `WHERE ${where.join(" AND ")}` : "";
+    const runs = await input.control.all(`SELECT * FROM "_search_reindex_runs" ${whereSql} ORDER BY "startedAt" DESC LIMIT ? OFFSET ?`, [...params, Math.min(Math.max(input.limit ?? 50, 1), 500), Math.max(input.offset ?? 0, 0)]).catch(() => []);
+    return { runs };
+}
+export async function markSearchRebuildRequired(control, reason) {
+    await control.run(`INSERT OR REPLACE INTO "_state" ("key", "value", "updatedAt") VALUES (?, ?, ?)`, [raybaseSearchRebuildStateKey, JSON.stringify({ reason, markedAt: nowIso() }), nowIso()]);
+}
+async function indexTableSearchRow(input) {
+    try {
+        const documents = await sourceDocumentsForTableRow(input.index, input.source, input.row);
+        if (!documents.length) {
+            const sourceId = String(input.row.id ?? "");
+            if (sourceId) {
+                await deleteSearchDocumentsForSource(input.control, {
+                    indexName: input.index.name,
+                    sourceType: "table",
+                    sourceTable: input.source.table.name,
+                    sourceId
+                });
+            }
+            return { documentIds: [], indexed: 0, upserted: 0, skipped: 1, errors: 0 };
+        }
+        const documentIds = new Set();
+        for (const document of documents) {
+            documentIds.add(document.id);
+            await upsertSearchDocument(input.control, document);
+        }
+        const sourceId = documents[0].sourceId;
+        const existing = await input.control.all(`SELECT "id" FROM "_search_documents"
+       WHERE "indexName" = ? AND "sourceType" = 'table' AND "sourceTable" = ? AND "sourceId" = ? AND "status" != 'deleted'`, [input.index.name, input.source.table.name, sourceId]);
+        for (const existingDocument of existing) {
+            if (!documentIds.has(existingDocument.id))
+                await deleteSearchDocumentById(input.control, existingDocument.id);
+        }
+        return { documentIds: [...documentIds], indexed: documents.length, upserted: documents.length, skipped: 0, errors: 0 };
+    }
+    catch (error) {
+        const sourceId = String(input.row.id ?? "");
+        if (sourceId) {
+            await input.control.run(`UPDATE "_search_documents"
+         SET "status" = 'error', "lastError" = ?, "updatedAt" = ?
+         WHERE "indexName" = ? AND "sourceType" = 'table' AND "sourceTable" = ? AND "sourceId" = ?`, [error instanceof Error ? error.message : String(error), nowIso(), input.index.name, input.source.table.name, sourceId]).catch(() => undefined);
+        }
+        return { documentIds: [], indexed: 0, upserted: 0, skipped: 0, errors: 1 };
+    }
+}
+async function sourceDocumentsForTableRow(index, source, row) {
+    const sourceId = String(row.id ?? "");
+    if (!sourceId)
+        return [];
+    const body = source.text(row).trim();
+    if (!body)
+        return [];
+    const title = source.title(row).trim() || sourceId;
+    const version = source.version?.(row) ?? String(row.updatedAt ?? row.createdAt ?? "");
+    const visibility = typeof source.public === "function" ? source.public(row) : source.public === true;
+    const chunks = await chunkSearchText(body, index.chunking);
+    const documents = [];
+    for (const chunk of chunks) {
+        documents.push({
+            id: await searchDocumentIdForChunk({
+                indexName: index.name,
+                sourceType: "table",
+                sourceTable: source.table.name,
+                sourceId,
+                chunkKey: chunk.key
+            }),
+            indexName: index.name,
+            sourceType: "table",
+            sourceTable: source.table.name,
+            sourceId,
+            sourceVersion: version,
+            chunkKey: chunk.key,
+            chunkOrdinal: chunk.ordinal,
+            title,
+            body: chunk.text,
+            url: source.url?.(row) ?? null,
+            textHash: chunk.hash,
+            visibility: visibility ? "public" : "private",
+            metadata: source.metadata ? source.metadata(row) : undefined
+        });
+    }
+    return documents;
+}
+async function indexCorpusSearchSource(input) {
+    const docs = typeof input.source.documents === "function" ? await input.source.documents() : input.source.documents;
+    const activeDocumentIds = new Set();
+    const counts = { sources: 0, indexed: 0, upserted: 0, deleted: 0, skipped: 0, errors: 0 };
+    for (const doc of docs.slice(0, Math.min(input.source.limit ?? docs.length, 5000))) {
+        counts.sources += 1;
+        if (!doc.id || !doc.body.trim()) {
+            counts.skipped += 1;
+            continue;
+        }
+        const chunks = await chunkSearchText(doc.body, input.index.chunking);
+        if (!chunks.length) {
+            counts.skipped += 1;
+            continue;
+        }
+        for (const chunk of chunks) {
+            const document = {
+                id: await searchDocumentIdForChunk({
+                    indexName: input.index.name,
+                    sourceType: "corpus",
+                    sourceTable: input.source.name,
+                    sourceId: doc.id,
+                    chunkKey: chunk.key
+                }),
+                indexName: input.index.name,
+                sourceType: "corpus",
+                sourceTable: input.source.name,
+                sourceId: doc.id,
+                sourceVersion: doc.version ?? "",
+                chunkKey: chunk.key,
+                chunkOrdinal: chunk.ordinal,
+                title: doc.title,
+                body: chunk.text,
+                url: doc.url ?? null,
+                textHash: chunk.hash,
+                visibility: doc.public === true ? "public" : "private",
+                metadata: doc.metadata
+            };
+            activeDocumentIds.add(document.id);
+            await upsertSearchDocument(input.control, document);
+            counts.indexed += 1;
+            counts.upserted += 1;
+        }
+    }
+    const existing = await input.control.all(`SELECT "id" FROM "_search_documents"
+     WHERE "indexName" = ? AND "sourceType" = 'corpus' AND "sourceTable" = ? AND "status" != 'deleted'`, [input.index.name, input.source.name]);
+    for (const row of existing) {
+        if (!activeDocumentIds.has(row.id)) {
+            await deleteSearchDocumentById(input.control, row.id);
+            counts.deleted += 1;
+        }
+    }
+    return counts;
+}
+async function upsertSearchDocument(control, document) {
+    const now = nowIso();
+    const metadata = document.metadata === undefined ? null : stableJson(document.metadata);
+    const existing = await control.first(`SELECT "id", "rowid" FROM "_search_documents" WHERE "id" = ? LIMIT 1`, [document.id]);
+    const values = [
+        document.indexName,
+        document.sourceType,
+        document.sourceTable,
+        document.sourceId,
+        document.sourceVersion ?? null,
+        document.chunkKey,
+        document.chunkOrdinal,
+        document.title,
+        document.body,
+        document.url ?? null,
+        document.textHash,
+        document.visibility,
+        metadata,
+        "indexed",
+        null,
+        now,
+        now
+    ];
+    if (existing) {
+        await deleteFtsRow(control, Number(existing.rowid)).catch(() => undefined);
+        await control.run(`UPDATE "_search_documents"
+       SET "indexName" = ?, "sourceType" = ?, "sourceTable" = ?, "sourceId" = ?, "sourceVersion" = ?,
+           "chunkKey" = ?, "chunkOrdinal" = ?, "title" = ?, "body" = ?, "url" = ?, "textHash" = ?,
+           "visibility" = ?, "metadata" = ?, "status" = ?, "lastError" = ?, "indexedAt" = ?, "updatedAt" = ?
+       WHERE "id" = ?`, [...values, document.id]);
+        await control.run(`INSERT INTO "_search_fts" ("rowid", "title", "body") VALUES (?, ?, ?)`, [
+            Number(existing.rowid),
+            document.title,
+            document.body
+        ]);
+        return;
+    }
+    await control.run(`INSERT INTO "_search_documents"
+     ("id", "indexName", "sourceType", "sourceTable", "sourceId", "sourceVersion", "chunkKey", "chunkOrdinal",
+      "title", "body", "url", "textHash", "visibility", "metadata", "status", "lastError", "indexedAt", "updatedAt")
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [document.id, ...values]);
+    const inserted = await control.first(`SELECT "rowid" FROM "_search_documents" WHERE "id" = ?`, [document.id]);
+    if (inserted) {
+        await control.run(`INSERT INTO "_search_fts" ("rowid", "title", "body") VALUES (?, ?, ?)`, [
+            Number(inserted.rowid),
+            document.title,
+            document.body
+        ]);
+    }
+}
+async function deleteSearchDocumentsForSource(control, input) {
+    const docs = await control.all(`SELECT "id" FROM "_search_documents"
+     WHERE "indexName" = ? AND "sourceType" = ? AND "sourceTable" IS ? AND "sourceId" = ?`, [input.indexName, input.sourceType, input.sourceTable ?? null, input.sourceId]);
+    for (const doc of docs)
+        await deleteSearchDocumentById(control, doc.id);
+}
+async function deleteSearchDocumentById(control, id) {
+    const existing = await control.first(`SELECT "rowid", "title", "body" FROM "_search_documents" WHERE "id" = ? LIMIT 1`, [id]);
+    if (!existing)
+        return;
+    await deleteFtsRow(control, Number(existing.rowid)).catch(() => undefined);
+    await control.run(`UPDATE "_search_documents" SET "status" = 'deleted', "updatedAt" = ? WHERE "id" = ?`, [nowIso(), id]);
+}
+async function deleteFtsRow(control, rowid) {
+    await control.run(`DELETE FROM "_search_fts" WHERE "rowid" = ?`, [rowid]);
+}
+async function completeSearchReindexRun(control, runId, status, counts, summary) {
+    await control.run(`UPDATE "_search_reindex_runs"
+     SET "status" = ?, "totalSources" = ?, "totalChunks" = ?, "upsertedChunks" = ?, "deletedChunks" = ?,
+         "skippedChunks" = ?, "errorCount" = ?, "completedAt" = ?, "summaryJson" = ?
+     WHERE "id" = ?`, [
+        status,
+        counts.totalSources,
+        counts.totalChunks,
+        counts.upsertedChunks,
+        counts.deletedChunks,
+        counts.skippedChunks,
+        counts.errorCount,
+        nowIso(),
+        stableJson(summary),
+        runId
+    ]);
+}
+function findTableSearchSource(index, tableName) {
+    return index.sources.find((source) => source.kind === "table" && source.table.name === tableName) ?? null;
+}
+function tableKeyFor(schema, table) {
+    const entry = Object.entries(schema.tables).find(([, candidate]) => candidate.name === table.name);
+    if (!entry)
+        throw new Error(`Search source table ${table.name} is not in the app schema`);
+    return entry[0];
+}
+function splitSqlStatements(sql) {
+    return sql
+        .split(";")
+        .map((statement) => statement.trim())
+        .filter(Boolean);
+}
+function parseJson(value) {
+    if (typeof value !== "string" || !value)
+        return value;
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return value;
+    }
+}
+function asNullableString(value) {
+    return typeof value === "string" && value ? value : null;
+}
+async function readSearchRebuildRequired(control) {
+    const row = await control.first(`SELECT "value" FROM "_state" WHERE "key" = ?`, [raybaseSearchRebuildStateKey]).catch(() => null);
+    return row?.value ? parseJson(row.value) : null;
+}
+async function sha256Base64Url(value) {
+    const bytes = new TextEncoder().encode(value);
+    const digest = await crypto.subtle.digest("SHA-256", bytes);
+    let binary = "";
+    for (const byte of new Uint8Array(digest))
+        binary += String.fromCharCode(byte);
+    return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
+}
+//# sourceMappingURL=search.js.map