@undefineds.co/xpod 0.3.58 → 0.3.61

package/dist/storage/rdf/RdfTextIndex.js

@@ -1,12 +1,30 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RdfTextIndex = void 0;
+exports.RdfTextIndex = exports.RDF_TEXT_SCHEMA_VERSION = exports.RDF_TEXT_TERM_MAX_INDEX_LENGTH = void 0;
+exports.chunkRdfTextSource = chunkRdfTextSource;
+exports.deterministicRdfTextChunkKey = deterministicRdfTextChunkKey;
+exports.rdfTextSha256 = rdfTextSha256;
+exports.normalizeRdfText = normalizeRdfText;
+exports.normalizedRdfTextTokenCount = normalizedRdfTextTokenCount;
+exports.tokenizeNormalizedRdfText = tokenizeNormalizedRdfText;
+exports.rdfTextTermOccurrences = rdfTextTermOccurrences;
+exports.rdfTextOccurrenceCount = rdfTextOccurrenceCount;
+exports.applyRdfTextResultWindow = applyRdfTextResultWindow;
+exports.compareRdfTextSearchHits = compareRdfTextSearchHits;
+exports.normalizeRdfTextPerSourceLimit = normalizeRdfTextPerSourceLimit;
+exports.escapeRdfTextLikePattern = escapeRdfTextLikePattern;
+exports.parseRdfTextPath = parseRdfTextPath;
+exports.normalizeRdfTextRetrievalKind = normalizeRdfTextRetrievalKind;
+exports.rdfTextIndexBudgetSkip = rdfTextIndexBudgetSkip;
+exports.rdfTextIndexChunkCap = rdfTextIndexChunkCap;
 const node_crypto_1 = require("node:crypto");
 const node_fs_1 = require("node:fs");
 const node_path_1 = require("node:path");
 const HeadingChunker_1 = require("../../document/HeadingChunker");
 const SqliteRuntime_1 = require("../SqliteRuntime");
-const RDF_TEXT_TERM_MAX_INDEX_LENGTH = 256;
+const RdfSearchSourceFilter_1 = require("./RdfSearchSourceFilter");
+exports.RDF_TEXT_TERM_MAX_INDEX_LENGTH = 256;
+exports.RDF_TEXT_SCHEMA_VERSION = 2;
 class RdfTextIndex {
     constructor(options) {
         this.options = options;
@@ -31,19 +49,80 @@ class RdfTextIndex {
         this.db = null;
     }
     clear() {
-        this.requireDb().exec('DELETE FROM rdf_text_terms; DELETE FROM rdf_text_chunks; DELETE FROM rdf_text_sources;');
+        this.requireDb().exec('DELETE FROM rdf_text_rebuild_status; DELETE FROM rdf_text_entities; DELETE FROM rdf_text_terms; DELETE FROM rdf_text_chunks; DELETE FROM rdf_text_sources;');
+    }
+    schemaVersion() {
+        const row = this.requireDb()
+            .prepare("SELECT value FROM rdf_text_metadata WHERE key = 'schema_version'")
+            .get();
+        return Number(row?.value ?? 0) || 0;
+    }
+    sourceMetadata(source) {
+        const row = this.requireDb()
+            .prepare('SELECT * FROM rdf_text_sources WHERE source = ?')
+            .get(source);
+        return row ? rdfTextSourceMetadata(row) : undefined;
+    }
+    recordRebuildStatus(input) {
+        this.requireDb().prepare(`
+      INSERT INTO rdf_text_rebuild_status (
+        source,
+        workspace,
+        local_path,
+        content_type,
+        source_version,
+        source_hash,
+        status,
+        reason,
+        message,
+        updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      ON CONFLICT (source)
+      DO UPDATE SET
+        workspace = excluded.workspace,
+        local_path = excluded.local_path,
+        content_type = excluded.content_type,
+        source_version = excluded.source_version,
+        source_hash = excluded.source_hash,
+        status = excluded.status,
+        reason = excluded.reason,
+        message = excluded.message,
+        updated_at = excluded.updated_at
+    `).run(input.source, input.workspace, input.localPath ?? null, input.contentType ?? null, input.sourceVersion ?? null, input.sourceHash ?? null, input.status, input.reason ?? null, input.message ?? null);
+    }
+    rebuildStatus(source) {
+        const row = this.requireDb()
+            .prepare('SELECT * FROM rdf_text_rebuild_status WHERE source = ?')
+            .get(source);
+        return row ? rdfTextRebuildStatus(row) : undefined;
     }
     indexText(source, text, chunks) {
         const db = this.requireDb();
-        const indexedChunks = chunks ?? this.chunkText(source, text);
+        const sourceHash = source.sourceHash ?? sha256(text);
+        const budgetSkip = rdfTextIndexBudgetSkip(this.options.maxSourceBytes, text);
+        if (budgetSkip) {
+            this.deleteSource(source.source);
+            this.recordRebuildStatus({
+                ...source,
+                sourceHash,
+                status: 'skipped',
+                reason: budgetSkip.reason,
+                message: budgetSkip.message,
+            });
+            return;
+        }
+        const chunkCap = rdfTextIndexChunkCap(this.options.maxChunksPerSource, chunks ?? this.chunkText(source, text));
+        const indexedChunks = chunkCap.chunks;
         const sourceId = this.upsertSource({
             ...source,
-            sourceHash: source.sourceHash ?? sha256(text),
+            sourceHash,
         });
         const insertChunk = db.prepare(`
       INSERT INTO rdf_text_chunks (
         source_id,
         chunk_key,
+        retrieval_kind,
         ordinal,
         level,
         heading,
@@ -55,7 +134,7 @@ class RdfTextIndex {
         token_count,
         updated_at
       )
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
     `);
         const insertTerm = db.prepare(`
       INSERT INTO rdf_text_terms (
@@ -66,16 +145,91 @@ class RdfTextIndex {
         updated_at
       )
       VALUES (?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+    `);
+        const insertEntity = db.prepare(`
+      INSERT INTO rdf_text_entities (
+        entity,
+        source_id,
+        chunk_id,
+        predicate,
+        label,
+        value,
+        datatype,
+        language,
+        policy_role,
+        occurrences,
+        updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
     `);
         db.transaction(() => {
+            db.prepare('DELETE FROM rdf_text_entities WHERE source_id = ?').run(sourceId);
             db.prepare('DELETE FROM rdf_text_terms WHERE source_id = ?').run(sourceId);
             db.prepare('DELETE FROM rdf_text_chunks WHERE source_id = ?').run(sourceId);
             for (const chunk of indexedChunks) {
                 const normalizedText = normalizeText(chunk.content);
-                const result = insertChunk.run(sourceId, chunk.chunkKey, chunk.ordinal, chunk.level, chunk.heading || null, JSON.stringify(chunk.path ?? []), chunk.content, chunk.startOffset, chunk.endOffset, normalizedText, tokenCountNormalized(normalizedText));
-                insertTermOccurrences(insertTerm, sourceId, Number(result.lastInsertRowid), normalizedText);
+                const result = insertChunk.run(sourceId, chunk.chunkKey, chunk.retrievalKind ?? 'file-chunk', chunk.ordinal, chunk.level, chunk.heading || null, JSON.stringify(chunk.path ?? []), chunk.content, chunk.startOffset, chunk.endOffset, normalizedText, tokenCountNormalized(normalizedText));
+                const chunkId = Number(result.lastInsertRowid);
+                insertTermOccurrences(insertTerm, sourceId, chunkId, normalizedText);
+                insertEntityMentions(insertEntity, sourceId, chunkId, chunk.entities);
+            }
+        })();
+        if (chunkCap.capped) {
+            this.recordRebuildStatus({
+                ...source,
+                sourceHash,
+                status: 'capped',
+                reason: chunkCap.capped.reason,
+                message: chunkCap.capped.message,
+            });
+        }
+    }
+    moveSource(oldSource, next) {
+        const db = this.requireDb();
+        let affectedRows = 0;
+        db.transaction(() => {
+            const oldRow = db.prepare('SELECT * FROM rdf_text_sources WHERE source = ?').get(oldSource);
+            if (!oldRow) {
+                return;
+            }
+            const chunkCount = db
+                .prepare('SELECT COUNT(*) AS count FROM rdf_text_chunks WHERE source_id = ?')
+                .get(oldRow.id)?.count ?? 0;
+            const targetRow = db.prepare('SELECT * FROM rdf_text_sources WHERE source = ?').get(next.source);
+            if (targetRow && targetRow.id !== oldRow.id) {
+                db.prepare('DELETE FROM rdf_text_entities WHERE source_id = ?').run(targetRow.id);
+                db.prepare('DELETE FROM rdf_text_terms WHERE source_id = ?').run(targetRow.id);
+                db.prepare('DELETE FROM rdf_text_chunks WHERE source_id = ?').run(targetRow.id);
+                db.prepare('DELETE FROM rdf_text_sources WHERE id = ?').run(targetRow.id);
             }
+            db.prepare(`
+        UPDATE rdf_text_sources
+        SET
+          source_key = ?,
+          source = ?,
+          workspace = ?,
+          local_path = ?,
+          content_type = ?,
+          source_version = ?,
+          source_hash = ?,
+          updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+        WHERE id = ?
+      `).run(next.sourceKey ?? oldRow.source_key ?? oldRow.source, next.source, next.workspace, next.localPath ?? null, next.contentType ?? null, next.sourceVersion ?? null, next.sourceHash ?? null, oldRow.id);
+            db.prepare(`
+        UPDATE rdf_text_rebuild_status
+        SET
+          source = ?,
+          workspace = ?,
+          local_path = ?,
+          content_type = ?,
+          source_version = ?,
+          source_hash = ?,
+          updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+        WHERE source = ?
+      `).run(next.source, next.workspace, next.localPath ?? null, next.contentType ?? null, next.sourceVersion ?? null, next.sourceHash ?? null, oldSource);
+            affectedRows = Math.max(chunkCount, 1);
         })();
+        return affectedRows;
     }
     deleteSource(source) {
         const db = this.requireDb();
@@ -84,6 +238,7 @@ class RdfTextIndex {
             return 0;
         }
         return db.transaction(() => {
+            db.prepare('DELETE FROM rdf_text_entities WHERE source_id = ?').run(row.id);
             db.prepare('DELETE FROM rdf_text_terms WHERE source_id = ?').run(row.id);
             const deletedChunks = db.prepare('DELETE FROM rdf_text_chunks WHERE source_id = ?').run(row.id).changes;
             db.prepare('DELETE FROM rdf_text_sources WHERE id = ?').run(row.id);
@@ -92,92 +247,175 @@ class RdfTextIndex {
     }
     search(options) {
         const query = normalizeText(options.query);
-        if (!query) {
+        const entityFilter = normalizeEntityFilter(options.entities);
+        if (!query && entityFilter.length === 0) {
             return [];
         }
-        const predicate = buildTextSearchPredicate(query);
-        const params = [...predicate.params];
-        const conditions = [predicate.sql];
+        const predicate = query ? buildTextSearchPredicate(query) : undefined;
+        const params = predicate ? [...predicate.params] : [];
+        const conditions = predicate ? [predicate.sql] : ['1 = 1'];
+        appendEntitySearchFilter(entityFilter, conditions, params);
         if (options.workspace) {
             conditions.push('source.workspace = ?');
             params.push(options.workspace);
         }
-        if (options.source) {
-            conditions.push('source.source = ?');
-            params.push(options.source);
-        }
-        if (options.sourcePrefix) {
-            conditions.push('source.source >= ? AND source.source < ?');
-            params.push(options.sourcePrefix, `${options.sourcePrefix}\uffff`);
-        }
-        const sql = `
-      SELECT
-        chunk.id,
-        chunk.source_id,
-        source.source,
-        source.workspace,
-        source.local_path,
-        source.content_type,
-        source.source_version,
-        source.source_hash,
-        chunk.chunk_key,
-        chunk.ordinal,
-        chunk.level,
-        chunk.heading,
-        chunk.path,
-        chunk.content,
-        chunk.start_offset,
-        chunk.end_offset,
-        chunk.normalized_text,
-        chunk.token_count,
-        chunk.updated_at
-      FROM rdf_text_chunks chunk
-      JOIN rdf_text_sources source ON source.id = chunk.source_id
-      WHERE ${conditions.join(' AND ')}
-      ORDER BY chunk.source_id ASC, chunk.ordinal ASC
-    `;
-        const rows = this.requireDb().prepare(sql).all(...params);
-        const results = rows
-            .map((row) => ({ row, score: occurrenceCount(row.normalized_text, query) }))
-            .filter((result) => result.score > 0)
-            .sort((left, right) => compareTextSearchHits(left, right, options.orderBy))
-            .map((result) => this.toSearchResult(result.row, result.score));
-        return results.slice(options.offset ?? 0, options.limit === undefined ? undefined : (options.offset ?? 0) + options.limit);
+        (0, RdfSearchSourceFilter_1.appendRdfSearchSourceFilters)(options, conditions, params);
+        const scoreExpression = query
+            ? `
+        CAST((length(chunk.normalized_text) - length(replace(chunk.normalized_text, ?, ''))) / length(?) AS INTEGER)
+        + CASE
+          WHEN lower(COALESCE(chunk.heading, '')) LIKE ? ESCAPE '\\' THEN 100
+          ELSE 0
+        END
+      `
+            : '1';
+        const orderBy = sqliteTextSearchOrderBy(options.orderBy);
+        const window = sqliteTextSearchWindow(options);
+        const perSourceLimit = normalizeRdfTextPerSourceLimit(options.perSourceLimit);
+        const sql = perSourceLimit === undefined
+            ? `
+        SELECT
+          chunk.id,
+          chunk.source_id,
+          source.source_key,
+          source.source,
+          source.workspace,
+          source.local_path,
+          source.content_type,
+          source.source_version,
+          source.source_hash,
+          chunk.chunk_key,
+          chunk.retrieval_kind,
+          chunk.ordinal,
+          chunk.level,
+          chunk.heading,
+          chunk.path,
+          chunk.content,
+          chunk.start_offset,
+          chunk.end_offset,
+          chunk.normalized_text,
+          chunk.token_count,
+          chunk.updated_at,
+          ${scoreExpression} AS score
+        FROM rdf_text_chunks chunk
+        JOIN rdf_text_sources source ON source.id = chunk.source_id
+        WHERE ${conditions.join(' AND ')}
+        ORDER BY ${orderBy}, chunk.source_id ASC, chunk.ordinal ASC
+        ${window.sql}
+      `
+            : `
+        WITH matched AS (
+          SELECT
+            chunk.id,
+            chunk.source_id,
+            source.source_key,
+            source.source,
+            source.workspace,
+            source.local_path,
+            source.content_type,
+            source.source_version,
+            source.source_hash,
+            chunk.chunk_key,
+            chunk.retrieval_kind,
+            chunk.ordinal,
+            chunk.level,
+            chunk.heading,
+            chunk.path,
+            chunk.content,
+            chunk.start_offset,
+            chunk.end_offset,
+            chunk.normalized_text,
+            chunk.token_count,
+            chunk.updated_at,
+            ${scoreExpression} AS score
+          FROM rdf_text_chunks chunk
+          JOIN rdf_text_sources source ON source.id = chunk.source_id
+          WHERE ${conditions.join(' AND ')}
+        ),
+        ranked AS (
+          SELECT
+            matched.*,
+            ROW_NUMBER() OVER (
+              PARTITION BY source
+              ORDER BY ${sqliteTextSearchResultOrderBy(options.orderBy)}, source_id ASC, ordinal ASC
+            ) AS source_rank
+          FROM matched
+        )
+        SELECT
+          id,
+          source_id,
+          source_key,
+          source,
+          workspace,
+          local_path,
+          content_type,
+          source_version,
+          source_hash,
+          chunk_key,
+          retrieval_kind,
+          ordinal,
+          level,
+          heading,
+          path,
+          content,
+          start_offset,
+          end_offset,
+          normalized_text,
+          token_count,
+          updated_at,
+          score
+        FROM ranked
+        WHERE source_rank <= ?
+        ORDER BY ${sqliteTextSearchResultOrderBy(options.orderBy)}, source_id ASC, ordinal ASC
+        ${window.sql}
+      `;
+        const scoreParams = query ? [query, query, `%${escapeLikePattern(query)}%`] : [];
+        const perSourceParams = perSourceLimit === undefined ? [] : [perSourceLimit];
+        const rows = this.requireDb().prepare(sql).all(...scoreParams, ...params, ...perSourceParams, ...window.params);
+        const entities = this.entitiesForChunks(rows.map((row) => row.id));
+        return rows.map((row) => this.toSearchResult(row, query, row.score, entities.get(row.id) ?? []));
     }
     estimateSearchCardinality(options) {
         const query = normalizeText(options.query);
-        if (!query) {
+        const entityFilter = normalizeEntityFilter(options.entities);
+        if (!query && entityFilter.length === 0) {
             return {
                 rows: 0,
                 source: 'text-normalized-scan',
                 indexChoice: 'text-normalized-scan',
             };
         }
-        const predicate = buildTextSearchPredicate(query);
-        const params = [...predicate.params];
-        const conditions = [predicate.sql];
+        const predicate = query ? buildTextSearchPredicate(query) : undefined;
+        const params = predicate ? [...predicate.params] : [];
+        const conditions = predicate ? [predicate.sql] : ['1 = 1'];
+        appendEntitySearchFilter(entityFilter, conditions, params);
         if (options.workspace) {
             conditions.push('source.workspace = ?');
             params.push(options.workspace);
         }
-        if (options.source) {
-            conditions.push('source.source = ?');
-            params.push(options.source);
-        }
-        if (options.sourcePrefix) {
-            conditions.push('source.source >= ? AND source.source < ?');
-            params.push(options.sourcePrefix, `${options.sourcePrefix}\uffff`);
-        }
-        const rows = this.requireDb().prepare(`
-      SELECT COUNT(*) AS count
-      FROM rdf_text_chunks chunk
-      JOIN rdf_text_sources source ON source.id = chunk.source_id
-      WHERE ${conditions.join(' AND ')}
-    `).get(...params)?.count ?? 0;
+        (0, RdfSearchSourceFilter_1.appendRdfSearchSourceFilters)(options, conditions, params);
+        const perSourceLimit = normalizeRdfTextPerSourceLimit(options.perSourceLimit);
+        const rows = perSourceLimit === undefined
+            ? this.requireDb().prepare(`
+        SELECT COUNT(*) AS count
+        FROM rdf_text_chunks chunk
+        JOIN rdf_text_sources source ON source.id = chunk.source_id
+        WHERE ${conditions.join(' AND ')}
+      `).get(...params)?.count ?? 0
+            : this.requireDb().prepare(`
+        SELECT COALESCE(SUM(CASE WHEN source_count > ? THEN ? ELSE source_count END), 0) AS count
+        FROM (
+          SELECT chunk.source_id, COUNT(*) AS source_count
+          FROM rdf_text_chunks chunk
+          JOIN rdf_text_sources source ON source.id = chunk.source_id
+          WHERE ${conditions.join(' AND ')}
+          GROUP BY chunk.source_id
+        )
+      `).get(perSourceLimit, perSourceLimit, ...params)?.count ?? 0;
         return {
             rows: applyResultWindow(rows, options.offset, options.limit),
-            source: predicate.indexChoice,
-            indexChoice: predicate.indexChoice,
+            source: predicate?.indexChoice ?? 'text-term-posting',
+            indexChoice: predicate?.indexChoice ?? 'text-entity-posting',
         };
     }
     stats() {
@@ -185,6 +423,7 @@ class RdfTextIndex {
         return {
             sourceCount: db.prepare('SELECT COUNT(*) AS count FROM rdf_text_sources').get()?.count ?? 0,
             chunkCount: db.prepare('SELECT COUNT(*) AS count FROM rdf_text_chunks').get()?.count ?? 0,
+            entityMentionCount: db.prepare('SELECT COUNT(*) AS count FROM rdf_text_entities').get()?.count ?? 0,
             databaseBytes: this.estimateDatabaseBytes(),
             termDocumentFrequency: this.termDocumentFrequency(),
         };
@@ -211,8 +450,14 @@ class RdfTextIndex {
     }
     initializeSchema() {
         this.requireDb().exec(`
+      CREATE TABLE IF NOT EXISTS rdf_text_metadata (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+
       CREATE TABLE IF NOT EXISTS rdf_text_sources (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
+        source_key TEXT,
         source TEXT NOT NULL UNIQUE,
         workspace TEXT NOT NULL,
         local_path TEXT,
@@ -222,10 +467,24 @@ class RdfTextIndex {
         updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
       );
 
+      CREATE TABLE IF NOT EXISTS rdf_text_rebuild_status (
+        source TEXT PRIMARY KEY,
+        workspace TEXT NOT NULL,
+        local_path TEXT,
+        content_type TEXT,
+        source_version TEXT,
+        source_hash TEXT,
+        status TEXT NOT NULL,
+        reason TEXT,
+        message TEXT,
+        updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      );
+
       CREATE TABLE IF NOT EXISTS rdf_text_chunks (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
         source_id INTEGER NOT NULL,
         chunk_key TEXT NOT NULL,
+        retrieval_kind TEXT NOT NULL DEFAULT 'file-chunk',
         ordinal INTEGER NOT NULL,
         level INTEGER NOT NULL,
         heading TEXT,
@@ -242,7 +501,7 @@ class RdfTextIndex {
 
       CREATE TABLE IF NOT EXISTS rdf_text_terms (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
-        term TEXT NOT NULL CHECK (length(term) <= ${RDF_TEXT_TERM_MAX_INDEX_LENGTH}),
+        term TEXT NOT NULL CHECK (length(term) <= ${exports.RDF_TEXT_TERM_MAX_INDEX_LENGTH}),
         source_id INTEGER NOT NULL,
         chunk_id INTEGER NOT NULL,
         occurrences INTEGER NOT NULL,
@@ -252,18 +511,82 @@ class RdfTextIndex {
         FOREIGN KEY (chunk_id) REFERENCES rdf_text_chunks(id)
       );
 
+      CREATE TABLE IF NOT EXISTS rdf_text_entities (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        entity TEXT NOT NULL,
+        source_id INTEGER NOT NULL,
+        chunk_id INTEGER NOT NULL,
+        predicate TEXT,
+        label TEXT,
+        value TEXT,
+        datatype TEXT,
+        language TEXT,
+        policy_role TEXT,
+        occurrences INTEGER NOT NULL DEFAULT 1,
+        updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+        FOREIGN KEY (source_id) REFERENCES rdf_text_sources(id),
+        FOREIGN KEY (chunk_id) REFERENCES rdf_text_chunks(id)
+      );
+
       CREATE INDEX IF NOT EXISTS rdf_text_sources_workspace ON rdf_text_sources(workspace);
       CREATE INDEX IF NOT EXISTS rdf_text_sources_source ON rdf_text_sources(source);
+      CREATE INDEX IF NOT EXISTS rdf_text_sources_local_path ON rdf_text_sources(local_path);
+      CREATE INDEX IF NOT EXISTS rdf_text_sources_workspace_local_path ON rdf_text_sources(workspace, local_path);
       CREATE INDEX IF NOT EXISTS rdf_text_chunks_source ON rdf_text_chunks(source_id, ordinal);
-      DELETE FROM rdf_text_terms WHERE length(term) > ${RDF_TEXT_TERM_MAX_INDEX_LENGTH};
+      DELETE FROM rdf_text_terms WHERE length(term) > ${exports.RDF_TEXT_TERM_MAX_INDEX_LENGTH};
       CREATE INDEX IF NOT EXISTS rdf_text_terms_term ON rdf_text_terms(term);
       CREATE INDEX IF NOT EXISTS rdf_text_terms_source_term ON rdf_text_terms(source_id, term);
       CREATE INDEX IF NOT EXISTS rdf_text_terms_chunk ON rdf_text_terms(chunk_id);
+      CREATE INDEX IF NOT EXISTS rdf_text_entities_entity ON rdf_text_entities(entity);
+      CREATE INDEX IF NOT EXISTS rdf_text_entities_source_entity ON rdf_text_entities(source_id, entity);
+      CREATE INDEX IF NOT EXISTS rdf_text_entities_chunk ON rdf_text_entities(chunk_id);
 
       DROP INDEX IF EXISTS rdf_text_chunks_normalized;
+
+      INSERT INTO rdf_text_metadata (key, value)
+      VALUES ('schema_version', '${exports.RDF_TEXT_SCHEMA_VERSION}')
+      ON CONFLICT (key) DO UPDATE SET value = excluded.value;
     `);
+        this.ensureSourceKeyColumn();
+        this.ensureChunkRetrievalColumns();
+        this.ensureEntityProvenanceColumns();
         this.backfillTermPostings();
     }
+    ensureSourceKeyColumn() {
+        const columns = new Set(this.requireDb()
+            .prepare('PRAGMA table_info(rdf_text_sources)')
+            .all()
+            .map((column) => column.name));
+        if (!columns.has('source_key')) {
+            this.requireDb().exec('ALTER TABLE rdf_text_sources ADD COLUMN source_key TEXT;');
+        }
+        this.requireDb().exec('UPDATE rdf_text_sources SET source_key = source WHERE source_key IS NULL;');
+    }
+    ensureChunkRetrievalColumns() {
+        const columns = new Set(this.requireDb()
+            .prepare('PRAGMA table_info(rdf_text_chunks)')
+            .all()
+            .map((column) => column.name));
+        if (!columns.has('retrieval_kind')) {
+            this.requireDb().exec("ALTER TABLE rdf_text_chunks ADD COLUMN retrieval_kind TEXT NOT NULL DEFAULT 'file-chunk';");
+        }
+    }
+    ensureEntityProvenanceColumns() {
+        const columns = new Set(this.requireDb()
+            .prepare('PRAGMA table_info(rdf_text_entities)')
+            .all()
+            .map((column) => column.name));
+        for (const [name, type] of [
+            ['value', 'TEXT'],
+            ['datatype', 'TEXT'],
+            ['language', 'TEXT'],
+            ['policy_role', 'TEXT'],
+        ]) {
+            if (!columns.has(name)) {
+                this.requireDb().exec(`ALTER TABLE rdf_text_entities ADD COLUMN ${name} ${type};`);
+            }
+        }
+    }
     backfillTermPostings() {
         const db = this.requireDb();
         const rows = db.prepare(`
@@ -284,6 +607,22 @@ class RdfTextIndex {
         updated_at
       )
       VALUES (?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+    `);
+        const insertEntity = db.prepare(`
+      INSERT INTO rdf_text_entities (
+        entity,
+        source_id,
+        chunk_id,
+        predicate,
+        label,
+        value,
+        datatype,
+        language,
+        policy_role,
+        occurrences,
+        updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
     `);
         db.transaction(() => {
             for (const row of rows) {
@@ -295,6 +634,7 @@ class RdfTextIndex {
         const db = this.requireDb();
         db.prepare(`
       INSERT INTO rdf_text_sources (
+        source_key,
         source,
         workspace,
         local_path,
@@ -303,16 +643,17 @@ class RdfTextIndex {
         source_hash,
         updated_at
       )
-      VALUES (?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+      VALUES (?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
       ON CONFLICT (source)
       DO UPDATE SET
+        source_key = excluded.source_key,
         workspace = excluded.workspace,
         local_path = excluded.local_path,
         content_type = excluded.content_type,
         source_version = excluded.source_version,
         source_hash = excluded.source_hash,
         updated_at = excluded.updated_at
-    `).run(source.source, source.workspace, source.localPath ?? null, source.contentType ?? null, source.sourceVersion ?? null, source.sourceHash ?? null);
+    `).run(source.sourceKey ?? source.source, source.source, source.workspace, source.localPath ?? null, source.contentType ?? null, source.sourceVersion ?? null, source.sourceHash ?? null);
         const row = db.prepare('SELECT * FROM rdf_text_sources WHERE source = ?').get(source.source);
         if (!row) {
             throw new Error(`Failed to upsert RDF text source: ${source.source}`);
@@ -320,25 +661,7 @@ class RdfTextIndex {
         return row.id;
     }
     chunkText(source, text) {
-        if (!text) {
-            return [];
-        }
-        if (isMarkdownSource(source)) {
-            const chunker = new HeadingChunker_1.HeadingChunker();
-            return chunker.flatten(chunker.chunk(text))
-                .filter((chunk) => chunk.content.trim().length > 0)
-                .map((chunk, index) => ({
-                chunkKey: deterministicChunkKey(source.source, index),
-                ordinal: index,
-                level: chunk.level,
-                heading: chunk.heading || undefined,
-                path: chunk.path,
-                content: chunk.content,
-                startOffset: chunk.startOffset,
-                endOffset: chunk.endOffset,
-            }));
-        }
-        return chunkPlainText(source.source, text);
+        return chunkRdfTextSource(source, text);
     }
     estimateDatabaseBytes() {
         const db = this.requireDb();
@@ -351,7 +674,7 @@ class RdfTextIndex {
             return 0;
         }
     }
-    toSearchResult(row, score) {
+    toSearchResult(row, normalizedQuery, score, entities) {
         return {
             source: row.source,
             workspace: row.workspace,
@@ -359,7 +682,10 @@ class RdfTextIndex {
             contentType: row.content_type ?? undefined,
             sourceVersion: row.source_version ?? undefined,
             sourceHash: row.source_hash ?? undefined,
+            sourceKey: row.source_key ?? row.source,
             chunkKey: row.chunk_key,
+            retrievalPointKey: row.chunk_key,
+            retrievalKind: normalizeRdfTextRetrievalKind(row.retrieval_kind),
             ordinal: row.ordinal,
             level: row.level,
             heading: row.heading ?? undefined,
@@ -367,9 +693,61 @@ class RdfTextIndex {
             content: row.content,
             startOffset: row.start_offset,
             endOffset: row.end_offset,
+            scoreComponents: textScoreComponents(row, normalizedQuery, score),
             score,
+            entities,
         };
     }
+    entitiesForChunks(chunkIds) {
+        const uniqueIds = [...new Set(chunkIds)];
+        const mentions = new Map();
+        if (uniqueIds.length === 0) {
+            return mentions;
+        }
+        const rows = this.requireDb().prepare(`
+      SELECT chunk_id, entity, predicate, label, value, datatype, language, policy_role, occurrences
+      FROM rdf_text_entities
+      WHERE chunk_id IN (${uniqueIds.map(() => '?').join(', ')})
+      ORDER BY chunk_id ASC, entity ASC, predicate ASC, label ASC
+    `).all(...uniqueIds);
+        for (const row of rows) {
+            const chunkId = row.chunk_id;
+            if (chunkId === undefined) {
+                continue;
+            }
+            const chunkMentions = mentions.get(chunkId) ?? [];
+            chunkMentions.push({
+                entity: row.entity,
+                predicate: row.predicate ?? undefined,
+                label: row.label ?? undefined,
+                value: row.value ?? undefined,
+                datatype: row.datatype ?? undefined,
+                language: row.language ?? undefined,
+                policyRole: row.policy_role ?? undefined,
+                occurrences: row.occurrences,
+            });
+            mentions.set(chunkId, chunkMentions);
+        }
+        return mentions;
+    }
+    entitiesForChunk(chunkId) {
+        const rows = this.requireDb().prepare(`
+      SELECT entity, predicate, label, value, datatype, language, policy_role, occurrences
+      FROM rdf_text_entities
+      WHERE chunk_id = ?
+      ORDER BY entity ASC, predicate ASC, label ASC
+    `).all(chunkId);
+        return rows.map((row) => ({
+            entity: row.entity,
+            predicate: row.predicate ?? undefined,
+            label: row.label ?? undefined,
+            value: row.value ?? undefined,
+            datatype: row.datatype ?? undefined,
+            language: row.language ?? undefined,
+            policyRole: row.policy_role ?? undefined,
+            occurrences: row.occurrences,
+        }));
+    }
     requireDb() {
         if (!this.db) {
             throw new Error('RdfTextIndex is not open');
@@ -378,6 +756,80 @@ class RdfTextIndex {
     }
 }
 exports.RdfTextIndex = RdfTextIndex;
+function rdfTextSourceMetadata(row) {
+    return {
+        sourceKey: row.source_key ?? row.source,
+        source: row.source,
+        workspace: row.workspace,
+        localPath: row.local_path ?? undefined,
+        contentType: row.content_type ?? undefined,
+        sourceVersion: row.source_version ?? undefined,
+        sourceHash: row.source_hash ?? undefined,
+        updatedAt: row.updated_at,
+    };
+}
+function rdfTextRebuildStatus(row) {
+    return {
+        source: row.source,
+        workspace: row.workspace,
+        localPath: row.local_path ?? undefined,
+        contentType: row.content_type ?? undefined,
+        sourceVersion: row.source_version ?? undefined,
+        sourceHash: row.source_hash ?? undefined,
+        status: row.status,
+        reason: row.reason ?? undefined,
+        message: row.message ?? undefined,
+        updatedAt: row.updated_at,
+    };
+}
+function chunkRdfTextSource(source, text) {
+    if (!text) {
+        return [];
+    }
+    if (isFolderSource(source)) {
+        return chunkFolderCard(source, text);
+    }
+    if (isMarkdownSource(source)) {
+        const chunker = new HeadingChunker_1.HeadingChunker();
+        return chunker.flatten(chunker.chunk(text))
+            .filter((chunk) => chunk.content.trim().length > 0)
+            .map((chunk, index) => ({
+            chunkKey: deterministicChunkKey(source.source, index),
+            ordinal: index,
+            level: chunk.level,
+            heading: chunk.heading || undefined,
+            path: chunk.path,
+            content: chunk.content,
+            startOffset: chunk.startOffset,
+            endOffset: chunk.endOffset,
+        }));
+    }
+    return chunkPlainText(source.source, text);
+}
+function isFolderSource(source) {
+    const contentType = source.contentType?.split(';', 1)[0]?.trim().toLowerCase();
+    return contentType === 'inode/directory' ||
+        source.localPath?.endsWith('/') === true ||
+        source.source.endsWith('/');
+}
+function chunkFolderCard(source, text) {
+    const content = text.trim();
+    if (!content) {
+        return [];
+    }
+    const path = folderPathParts(source.localPath ?? source.source);
+    return [{
+            chunkKey: deterministicChunkKey(source.source, 0),
+            retrievalKind: 'folder-card',
+            ordinal: 0,
+            level: 0,
+            heading: path[path.length - 1],
+            path,
+            content,
+            startOffset: 0,
+            endOffset: text.length,
+        }];
+}
 function isMarkdownSource(source) {
     const contentType = source.contentType?.split(';', 1)[0]?.trim().toLowerCase();
     if (contentType === 'text/markdown' || contentType === 'text/x-markdown') {
@@ -386,6 +838,18 @@ function isMarkdownSource(source) {
     const path = source.localPath ?? source.source;
     return ['.md', '.markdown', '.mdown'].includes((0, node_path_1.extname)(path).toLowerCase());
 }
+function folderPathParts(value) {
+    const path = value.includes('://') ? safeUrlPathname(value) : value;
+    return path.split('/').map((part) => part.trim()).filter(Boolean);
+}
+function safeUrlPathname(value) {
+    try {
+        return new URL(value).pathname;
+    }
+    catch {
+        return value;
+    }
+}
 function chunkPlainText(source, text) {
     const chunks = [];
     const paragraphPattern = /[^\S\r\n]*(?:\r?\n){2,}[^\S\r\n]*/g;
@@ -432,7 +896,7 @@ function pushPlainChunk(chunks, source, ordinal, text, start, end) {
     });
     return ordinal + 1;
 }
-function deterministicChunkKey(source, ordinal) {
+function deterministicRdfTextChunkKey(source, ordinal) {
     return (0, node_crypto_1.createHash)('sha256')
         .update(source)
         .update('\0')
@@ -440,36 +904,110 @@ function deterministicChunkKey(source, ordinal) {
         .digest('hex')
         .slice(0, 24);
 }
-function sha256(value) {
+const deterministicChunkKey = deterministicRdfTextChunkKey;
+function rdfTextSha256(value) {
     return (0, node_crypto_1.createHash)('sha256').update(value).digest('hex');
 }
-function normalizeText(value) {
-    return value.toLowerCase().replace(/\s+/g, ' ').trim();
+const sha256 = rdfTextSha256;
+function textScoreComponents(row, normalizedQuery, score) {
+    if (!normalizedQuery) {
+        return {
+            sourceType: 'text',
+            algorithm: 'occurrence-heading-boost',
+            normalizedQuery,
+            occurrenceScore: 1,
+            headingBoost: 0,
+            score,
+        };
+    }
+    const occurrenceScore = occurrenceCount(row.normalized_text, normalizedQuery);
+    const headingBoost = normalizeText(row.heading ?? '').includes(normalizedQuery) ? 100 : 0;
+    return {
+        sourceType: 'text',
+        algorithm: 'occurrence-heading-boost',
+        normalizedQuery,
+        occurrenceScore,
+        headingBoost,
+        score,
+    };
+}
+function normalizeRdfText(value) {
+    return value.normalize('NFKC').toLowerCase().replace(/\s+/g, ' ').trim();
 }
-function tokenCountNormalized(value) {
+const normalizeText = normalizeRdfText;
+function normalizedRdfTextTokenCount(value) {
     return tokenizeNormalizedText(value).length;
 }
-function tokenizeNormalizedText(value) {
-    return value ? value.split(' ').filter(Boolean) : [];
+const tokenCountNormalized = normalizedRdfTextTokenCount;
+function tokenizeNormalizedRdfText(value) {
+    const terms = [];
+    let word = '';
+    let cjk = '';
+    const flushWord = () => {
+        if (word) {
+            terms.push(word);
+            word = '';
+        }
+    };
+    const flushCjk = () => {
+        if (!cjk) {
+            return;
+        }
+        const chars = Array.from(cjk);
+        if (chars.length === 1) {
+            terms.push(chars[0]);
+        }
+        else {
+            for (let index = 0; index < chars.length - 1; index += 1) {
+                terms.push(`${chars[index]}${chars[index + 1]}`);
+            }
+        }
+        cjk = '';
+    };
+    for (const char of Array.from(value)) {
+        if (isCjkTextChar(char)) {
+            flushWord();
+            cjk += char;
+            continue;
+        }
+        if (isRdfTextTokenChar(char)) {
+            flushCjk();
+            word += char;
+            continue;
+        }
+        flushWord();
+        flushCjk();
+    }
+    flushWord();
+    flushCjk();
+    return terms.filter(Boolean);
+}
+const tokenizeNormalizedText = tokenizeNormalizedRdfText;
+function isCjkTextChar(char) {
+    return /\p{Script=Han}|\p{Script=Hiragana}|\p{Script=Katakana}|\p{Script=Hangul}/u.test(char);
+}
+function isRdfTextTokenChar(char) {
+    return /[\p{Letter}\p{Mark}\p{Number}]/u.test(char);
 }
 function insertTermOccurrences(insertTerm, sourceId, chunkId, normalizedText) {
     for (const [term, occurrences] of termOccurrences(normalizedText)) {
-        if (term.length > RDF_TEXT_TERM_MAX_INDEX_LENGTH) {
+        if (term.length > exports.RDF_TEXT_TERM_MAX_INDEX_LENGTH) {
             continue;
         }
         insertTerm.run(term, sourceId, chunkId, occurrences);
     }
 }
-function termOccurrences(normalizedText) {
+function rdfTextTermOccurrences(normalizedText) {
     const terms = new Map();
     for (const term of tokenizeNormalizedText(normalizedText)) {
         terms.set(term, (terms.get(term) ?? 0) + 1);
     }
     return terms;
 }
+const termOccurrences = rdfTextTermOccurrences;
 function buildTextSearchPredicate(query) {
     const terms = [...new Set(tokenizeNormalizedText(query))]
-        .filter((term) => term.length <= RDF_TEXT_TERM_MAX_INDEX_LENGTH);
+        .filter((term) => term.length <= exports.RDF_TEXT_TERM_MAX_INDEX_LENGTH);
     const phraseCondition = "chunk.normalized_text LIKE ? ESCAPE '\\'";
     const phrasePattern = `%${escapeLikePattern(query)}%`;
     if (terms.length === 0) {
@@ -487,7 +1025,7 @@ function buildTextSearchPredicate(query) {
           ${terms.map(() => `
             SELECT term.chunk_id, ? AS query_term
             FROM rdf_text_terms term
-            WHERE term.term LIKE ? ESCAPE '\\'
+            WHERE term.term = ?
           `).join(' UNION ALL ')}
         ) candidate
         GROUP BY candidate.chunk_id
@@ -496,14 +1034,14 @@ function buildTextSearchPredicate(query) {
       AND ${phraseCondition}
     `,
         params: [
-            ...terms.flatMap((term) => [term, `%${escapeLikePattern(term)}%`]),
+            ...terms.flatMap((term) => [term, term]),
             terms.length,
             phrasePattern,
         ],
         indexChoice: 'text-term-posting',
     };
 }
-function occurrenceCount(haystack, needle) {
+function rdfTextOccurrenceCount(haystack, needle) {
     if (!needle) {
         return 0;
     }
@@ -519,7 +1057,8 @@ function occurrenceCount(haystack, needle) {
     }
     return count;
 }
-function applyResultWindow(rows, offset, limit) {
+const occurrenceCount = rdfTextOccurrenceCount;
+function applyRdfTextResultWindow(rows, offset, limit) {
     const start = Math.max(0, offset ?? 0);
     if (rows <= start) {
         return 0;
@@ -527,7 +1066,8 @@ function applyResultWindow(rows, offset, limit) {
     const remaining = rows - start;
     return limit === undefined ? remaining : Math.min(remaining, Math.max(0, limit));
 }
-function compareTextSearchHits(left, right, orderBy) {
+const applyResultWindow = applyRdfTextResultWindow;
+function compareRdfTextSearchHits(left, right, orderBy) {
     const order = orderBy?.length ? orderBy : [{ field: 'score', direction: 'desc' }];
     for (const entry of order) {
         const direction = entry.direction === 'desc' ? -1 : 1;
@@ -538,6 +1078,71 @@ function compareTextSearchHits(left, right, orderBy) {
     }
     return left.row.source_id - right.row.source_id || left.row.ordinal - right.row.ordinal;
 }
+const compareTextSearchHits = compareRdfTextSearchHits;
+function sqliteTextSearchOrderBy(orderBy) {
+    const order = orderBy?.length ? orderBy : [{ field: 'score', direction: 'desc' }];
+    return order.map((entry) => `${sqliteTextSearchOrderField(entry.field)} ${entry.direction === 'desc' ? 'DESC' : 'ASC'}`).join(', ');
+}
+function sqliteTextSearchOrderField(field) {
+    switch (field) {
+        case 'score':
+            return 'score';
+        case 'source':
+            return 'source.source';
+        case 'localPath':
+            return 'COALESCE(source.local_path, \'\')';
+        case 'ordinal':
+            return 'chunk.ordinal';
+        case 'startOffset':
+            return 'chunk.start_offset';
+        case 'endOffset':
+            return 'chunk.end_offset';
+        default: {
+            const exhaustive = field;
+            throw new Error(`Unsupported RDF text search order field: ${exhaustive}`);
+        }
+    }
+}
+function sqliteTextSearchResultOrderBy(orderBy) {
+    const order = orderBy?.length ? orderBy : [{ field: 'score', direction: 'desc' }];
+    return order.map((entry) => `${sqliteTextSearchResultOrderField(entry.field)} ${entry.direction === 'desc' ? 'DESC' : 'ASC'}`).join(', ');
+}
+function sqliteTextSearchResultOrderField(field) {
+    switch (field) {
+        case 'score':
+            return 'score';
+        case 'source':
+            return 'source';
+        case 'localPath':
+            return 'COALESCE(local_path, \'\')';
+        case 'ordinal':
+            return 'ordinal';
+        case 'startOffset':
+            return 'start_offset';
+        case 'endOffset':
+            return 'end_offset';
+        default: {
+            const exhaustive = field;
+            throw new Error(`Unsupported RDF text search order field: ${exhaustive}`);
+        }
+    }
+}
+function sqliteTextSearchWindow(options) {
+    const offset = Math.max(0, options.offset ?? 0);
+    if (options.limit === undefined) {
+        return offset > 0 ? { sql: 'LIMIT -1 OFFSET ?', params: [offset] } : { sql: '', params: [] };
+    }
+    return {
+        sql: 'LIMIT ? OFFSET ?',
+        params: [Math.max(0, options.limit), offset],
+    };
+}
+function normalizeRdfTextPerSourceLimit(limit) {
+    if (limit === undefined || !Number.isFinite(limit)) {
+        return undefined;
+    }
+    return Math.max(0, Math.trunc(limit));
+}
 function compareTextSearchField(left, right, field) {
     switch (field) {
         case 'score':
@@ -558,10 +1163,11 @@ function compareTextSearchField(left, right, field) {
         }
     }
 }
-function escapeLikePattern(value) {
+function escapeRdfTextLikePattern(value) {
     return value.replace(/[\\%_]/g, (match) => `\\${match}`);
 }
-function parsePath(value) {
+const escapeLikePattern = escapeRdfTextLikePattern;
+function parseRdfTextPath(value) {
     if (!value) {
         return [];
     }
@@ -573,4 +1179,94 @@ function parsePath(value) {
         return [];
     }
 }
+const parsePath = parseRdfTextPath;
+function normalizeRdfTextRetrievalKind(value) {
+    switch (value) {
+        case 'entity-card':
+        case 'field-chunk':
+        case 'folder-card':
+        case 'file-chunk':
+            return value;
+        default:
+            return 'file-chunk';
+    }
+}
+function rdfTextIndexBudgetSkip(maxSourceBytes, text) {
+    if (maxSourceBytes === undefined || !Number.isFinite(maxSourceBytes)) {
+        return undefined;
+    }
+    const maxBytes = Math.max(0, Math.trunc(maxSourceBytes));
+    const sourceBytes = Buffer.byteLength(text);
+    if (sourceBytes <= maxBytes) {
+        return undefined;
+    }
+    return {
+        reason: 'maxSourceBytes',
+        message: `source text is ${sourceBytes} bytes; maxSourceBytes is ${maxBytes}`,
+    };
+}
+function rdfTextIndexChunkCap(maxChunksPerSource, chunks) {
+    if (maxChunksPerSource === undefined || !Number.isFinite(maxChunksPerSource)) {
+        return { chunks };
+    }
+    const maxChunks = Math.max(0, Math.trunc(maxChunksPerSource));
+    if (chunks.length <= maxChunks) {
+        return { chunks };
+    }
+    return {
+        chunks: chunks.slice(0, maxChunks),
+        capped: {
+            reason: 'maxChunksPerSource',
+            message: `source produced ${chunks.length} chunks; maxChunksPerSource is ${maxChunks}`,
+        },
+    };
+}
+function appendEntitySearchFilter(entities, conditions, params) {
+    if (entities.length === 0) {
+        return;
+    }
+    conditions.push(`
+    chunk.id IN (
+      SELECT entity.chunk_id
+      FROM rdf_text_entities entity
+      WHERE entity.entity IN (${entities.map(() => '?').join(', ')})
+      GROUP BY entity.chunk_id
+      HAVING COUNT(DISTINCT entity.entity) = ?
+    )
+  `);
+    params.push(...entities, entities.length);
+}
+function insertEntityMentions(insertEntity, sourceId, chunkId, entities) {
+    for (const entity of normalizeEntityMentions(entities)) {
+        insertEntity.run(entity.entity, sourceId, chunkId, entity.predicate ?? null, entity.label ?? null, entity.value ?? null, entity.datatype ?? null, entity.language ?? null, entity.policyRole ?? null, entity.occurrences);
+    }
+}
+function normalizeEntityFilter(entities) {
+    return [...new Set((entities ?? []).map((entity) => entity.trim()).filter(Boolean))].sort();
+}
+function normalizeEntityMentions(entities) {
+    const mentions = new Map();
+    for (const entity of entities ?? []) {
+        const iri = entity.entity.trim();
+        if (!iri) {
+            continue;
+        }
+        const predicate = entity.predicate?.trim() || undefined;
+        const label = entity.label?.trim() || undefined;
+        const value = entity.value;
+        const datatype = entity.datatype?.trim() || undefined;
+        const language = entity.language?.trim() || undefined;
+        const policyRole = entity.policyRole?.trim() || undefined;
+        const key = JSON.stringify([iri, predicate ?? '', label ?? '', value ?? '', datatype ?? '', language ?? '', policyRole ?? '']);
+        const existing = mentions.get(key);
+        const occurrences = Math.max(1, Math.trunc(entity.occurrences ?? 1));
+        if (existing) {
+            existing.occurrences += occurrences;
+        }
+        else {
+            mentions.set(key, { entity: iri, predicate, label, value, datatype, language, policyRole, occurrences });
+        }
+    }
+    return [...mentions.values()].sort((left, right) => left.entity.localeCompare(right.entity) || (left.predicate ?? '').localeCompare(right.predicate ?? '') || (left.label ?? '').localeCompare(right.label ?? ''));
+}
 //# sourceMappingURL=RdfTextIndex.js.map