@sparkelf/dsh-knowledge-base 0.1.5 → 0.1.20

package/lib/types/ftsIndex.d.ts

@@ -0,0 +1,497 @@
+/**
+ * SQLite FTS5 index for Agentic RAG.
+ *
+ * The index is deliberately a sidecar of the authoritative tree, never inside the
+ * caller's view: a view is rebuilt from grants, so an index stored there would be
+ * deleted by the next sync. It also must never be reachable through `kb_file_*`.
+ *
+ * Chunks are tokenized with FTS5's `trigram` tokenizer, which is what makes Chinese
+ * text searchable without a word segmenter. Scope columns are applied in SQL before
+ * BM25 ordering; the caller's per-path grant check is applied after the top candidate
+ * window, so a hit can be dropped but never leak.
+ *
+ * @module @sparkelf/dsh-knowledge-base/ftsIndex
+ */
+import { type ScopeLocale } from './scopeAliases.ts';
+import { type OutlineSection, type OutlineTocEntry } from './outlineText.ts';
+import { type OutlineSaveRejection, type OutlineSectionSubmission, type OutlineSummaryKind } from './outlineGuard.ts';
+/** Prefix used for the sidecar directory when no explicit index dir is configured. */
+export declare const DEFAULT_INDEX_DIR = "/nas/kb/index";
+/** One search hit returned to the model. */
+/** Result of one kb_search call, including whether it had to relax. */
+export interface FtsSearchResult {
+    hits: FtsSearchHit[];
+    relaxed: boolean;
+    queries: string[];
+    /** Set when hybrid retrieval ran; absent in pure-FTS answers. */
+    retrieval?: 'fts' | 'vector' | 'hybrid';
+    /** Provider/vector failure that made the answer fall back to FTS. */
+    vectorWarning?: string;
+    /** How many vector-only slots the FTS floor reclaimed. */
+    floorReplacements?: number;
+    /**
+     * Summary chunks that matched the query, for navigation only.
+     *
+     * They deliberately carry no citation token: a summary is generated from the source and
+     * must never become the quoted evidence. Follow one with `kb_outline` or a chunk read.
+     */
+    relatedSections?: FtsRelatedSection[];
+}
+/** One summary chunk returned by the summarised half of the FTS index. */
+export interface FtsRelatedSection {
+    docId: number;
+    kind: 'doc_summary' | 'section_summary';
+    sectionId?: string;
+    /** Original source path (the citable document). */
+    path: string;
+    displayPath?: string;
+    /** Chapter/section heading path. */
+    sectionPath: string;
+    summary: string;
+    /** Source chunk ids this summary was extracted from; navigation only. */
+    sourceChunkIds: number[];
+    score: number;
+}
+export interface FtsChunkBox {
+    page: number;
+    bbox: string;
+    blockIndex?: number;
+    occurrence?: number;
+    confidence?: number;
+    approximate?: boolean;
+}
+/**
+ * Collapse provenance boxes to exactly one union rectangle per page.
+ *
+ * `kb_chunk_boxes` intentionally stores one row per extracted block so provenance stays
+ * inspectable, but every reader (search hit, chunk endpoint, PDF viewer) wants the chunk's
+ * painted region: one rectangle per page. Unioning at the SQL boundary means a citation never
+ * receives a list of paragraph boxes that a client might paint one by one before merging.
+ */
+export declare function unionBoxesByPage(boxes: readonly FtsChunkBox[]): FtsChunkBox[];
+export interface FtsSearchHit {
+    /** Original document path (docx/xlsx/pdf/txt), never the derived Markdown sibling. */
+    path: string;
+    /** Localized virtual-root spelling for display; `path` stays canonical. */
+    displayPath?: string;
+    /** Derived text sibling used internally for snippets/reading. */
+    markdownPath: string;
+    heading?: string;
+    /** Chunk ordinal/paragraph anchor within the source. */
+    paragraph?: number;
+    /** Zero-based chunk ordinal in the document. */
+    chunkOrdinal?: number;
+    /** First prose paragraph anchor, for a viewer outline. */
+    paragraphText?: string;
+    /** Plain-text quote for findController (never Markdown/table syntax). */
+    quote?: string;
+    /** Query terms found in this chunk. */
+    matchedTerms?: string[];
+    /** Snippet/sentence containing the query terms. */
+    highlightText?: string;
+    /** Character ranges of matchedTerms inside highlightText (end-exclusive). */
+    highlightRanges?: Array<{
+        start: number;
+        end: number;
+    }>;
+    /** Spreadsheet anchors, when known. */
+    sheet?: string;
+    cell?: string;
+    /** Present only for text/Markdown sources; PDF/scan hits carry boxes instead. */
+    line?: number;
+    lineStart?: number;
+    lineEnd?: number;
+    snippet: string;
+    score: number;
+    docId: number;
+    chunkId: number;
+    /** Stable content-keyed citation id (survives a full reindex). */
+    chunkKey?: string;
+    /** Stable citation token: kb://doc/<docId>/chunk/<chunkKey or chunkId> */
+    citation: string;
+    /** All provenance boxes for this chunk, grouped by page by the viewer. */
+    boxes?: FtsChunkBox[];
+    /** First box's page, kept for convenience. */
+    page?: number;
+    /** First box bbox, kept for compatibility with earlier consumers. */
+    bbox?: string;
+    blockIndex?: number;
+    occurrence?: number;
+    /** True when any box came from a low-confidence OCR pass. */
+    approximate?: boolean;
+    /** Nearest outline section, when the document has one. */
+    sectionId?: string;
+    /** Full heading path of that section (`第三章 > 第二十二条`). */
+    sectionPath?: string;
+    /** Extractive summary of that section; navigation only, never a citation. */
+    sectionSummary?: string;
+    /** Depth of that section (1 = chapter). */
+    sectionLevel?: number;
+    /** Extractive document summary; navigation only, never a citation. */
+    docSummary?: string;
+    /** Which retrieval channel produced this hit in hybrid mode. */
+    retrieval?: 'fts' | 'vector' | 'hybrid';
+    /** One-based rank in the FTS candidate list. */
+    ftsRank?: number;
+    /** One-based rank in the vector candidate list. */
+    vectorRank?: number;
+    /** Weighted reciprocal-rank-fusion score. */
+    rrfScore?: number;
+    /** Raw BM25 score, when the FTS channel found this chunk. */
+    ftsScore?: number;
+    /** Raw cosine similarity, when the vector channel found this chunk. */
+    vectorScore?: number;
+    /** Set when the FTS floor put this hit back into the final window. */
+    retrievalReason?: 'fts-floor';
+}
+/** Query-side vector inputs for hybrid retrieval. */
+export interface VectorSearchInput {
+    model: string;
+    queryVector: readonly number[];
+    candidateK: number;
+    rrfK: number;
+    ftsWeight: number;
+    vectorWeight: number;
+    /** Share of the corpus that has vectors; scales only the vector contribution. */
+    coverage: number;
+}
+/** Counts reported by an index sync/rebuild. */
+export interface FtsIndexStats {
+    indexPath: string;
+    docs: number;
+    chunks: number;
+    indexed: number;
+    unchanged: number;
+    removed: number;
+    updatedAt: string;
+}
+/** A caller's scope for index filtering. */
+export interface FtsCallerScope {
+    callerId: string;
+    /** Department codes the caller may see, including ancestors/descendants as resolved by the Host. */
+    departmentCodes: readonly string[];
+    /** Global administrators see every scope. */
+    global: boolean;
+}
+/** The index path for one deployment. */
+export declare function indexPathFor(indexDir: string | undefined): string;
+/**
+ * Refuse an index path inside the browsable/authoritative trees.
+ *
+ * The index contains unreviewed snippets; putting it under a view would expose it to
+ * file tools and would let the next sync delete it as an unlinked file.
+ */
+export declare function assertIndexOutsideTrees(indexPath: string, trees: readonly string[]): void;
+/** Derive the scope columns from a view-relative path. */
+export declare function scopeForPath(relPath: string): {
+    scope: 'global' | 'department' | 'personal';
+    ownerId: string | null;
+    departmentCode: string | null;
+};
+/**
+ * Split text into structure-aware, line-addressed chunks for retrieval.
+ *
+ * Chapter/section headings, article headings, Markdown headings and table rows open a new
+ * chunk; a heading is never appended to the body that precedes it. Article bodies may split at
+ * sentence or list boundaries, but a chunk never crosses a chapter/section heading. The
+ * 350–500 CJK character target is a soft packing hint (min 150, max 800); structure wins when
+ * the two disagree. The 60-character overlap is added only inside one chapter and never in
+ * front of a heading, so a citation quote always begins with its own structure. The index
+ * records the first source line of each chunk, so text sources keep an honest `lineStart`.
+ */
+export declare function chunkText(text: string, options?: number | {
+    target?: number;
+    min?: number;
+    max?: number;
+    overlap?: number;
+}): Array<{
+    line: number;
+    content: string;
+}>;
+/** Stable citation key for one chunk: heading + normalized content hash. */
+export declare function stableChunkKey(heading: string | null | undefined, content: string): string;
+/**
+ * Resolve the original document for one indexed text file.
+ *
+ * A derived sibling is named `<source>.<ext>.md`; the source is that name with the
+ * trailing `.md` removed. A plain `.md`/`.txt` is its own source.
+ */
+export declare function sourcePathFor(markdownPath: string): string;
+/** Whether a source path is text that can expose a real line anchor. */
+export declare function isTextSourcePath(sourcePath: string): boolean;
+/** Collapse Markdown/table syntax into plain text suitable for a viewer search. */
+export declare function plainQuoteText(content: string): string;
+/**
+ * Pick the most distinctive prose sentence in a chunk. A title/heading is short and
+ * often repeated; a body sentence is the useful quote for findController when the
+ * caller has no query (`kb_chunk_get`).
+ */
+export declare function quoteForContent(content: string, minChars?: number, maxChars?: number): string;
+/** Strip FTS5 `[highlight]` markers/ellipses from a query-centered snippet. */
+export declare function quoteFromSnippet(snippet: string, minChars?: number, maxChars?: number): string;
+/** Split a chunk into plain-text paragraphs for viewer-side whole-chunk boxes. */
+export declare function paragraphsFor(content: string): Array<{
+    ordinal: number;
+    text: string;
+}>;
+/** Query tokens used by the viewer to draw yellow/orange hit boxes. */
+export declare function queryTerms(query: string): string[];
+/** Locate all term occurrences inside the plain snippet; ranges index highlightText. */
+export declare function highlightRangesFor(text: string, terms: readonly string[]): Array<{
+    start: number;
+    end: number;
+}>;
+/** Build the query-relevance fields a viewer uses for the second (yellow) layer. */
+export declare function queryHighlights(query: string, snippet: string, content: string): {
+    matchedTerms: string[];
+    highlightText: string;
+    highlightRanges: Array<{
+        start: number;
+        end: number;
+    }>;
+};
+/** Append the query-highlight hints to a citation token for the viewer. */
+export declare function citationWithHighlights(citation: string, highlights: {
+    matchedTerms: string[];
+    highlightText: string;
+    highlightRanges: Array<{
+        start: number;
+        end: number;
+    }>;
+}): string;
+/** Backwards-compatible simple quote for callers that only have content. */
+export declare function quoteFor(content: string, maxChars?: number): string;
+export declare const CHUNKER_VERSION = "structure-v3-keyed";
+/** Rebuild/refresh the FTS index incrementally from the authoritative tree. */
+export declare function syncIndex(options: {
+    actualRoot: string;
+    indexPath: string;
+    now?: () => Date;
+    /**
+     * Opt-in legacy extractive summaries. The production config default is false: upload
+     * and extraction create the TOC/section structure only, and a model-authored summary
+     * arrives later through `kb_outline_save`.
+     */
+    autoGenerateSummaries?: boolean;
+}): Promise<FtsIndexStats>;
+/** Read only the counts, without a rebuild. */
+export declare function indexStats(indexPath: string): Promise<{
+    docs: number;
+    chunks: number;
+    indexPath: string;
+}>;
+/**
+ * Search FTS5 and apply the caller's scope in SQL before BM25 ordering.
+ *
+ * `canRead` is the final path-level grant check. The SQL window is `topK * 4`, so a
+ * few unreadable descendants cannot starve the result; the returned list is still
+ * capped at `topK`.
+ */
+export declare function searchIndex(options: {
+    indexPath: string;
+    query: string;
+    topK: number;
+    scope?: string;
+    /** Locale for `displayPath`; the canonical `path` is unchanged. */
+    locale?: ScopeLocale;
+    caller: FtsCallerScope;
+    canRead: (relPath: string) => Promise<boolean>;
+    /** Present only when vector retrieval is enabled and the query embedded successfully. */
+    vector?: VectorSearchInput;
+}): Promise<FtsSearchResult>;
+/** One MinerU block, as extract.ts reports it. */
+export interface ProvenanceBlock {
+    page: number;
+    bbox: readonly [number, number, number, number];
+    text: string;
+    type?: string;
+    blockIndex?: number;
+    occurrence?: number;
+    confidence?: number;
+    pageWidth?: number;
+    pageHeight?: number;
+    coordSpace?: 'normalized' | 'points';
+}
+/**
+ * Attach MinerU page/bbox provenance to the chunks already indexed for one document.
+ *
+ * The Markdown and FTS rows stay pure text; bbox lives only in `kb_chunk_boxes`.
+ */
+export declare function attachProvenance(options: {
+    indexPath: string;
+    markdownPath: string;
+    sourcePath?: string;
+    blocks: readonly ProvenanceBlock[];
+}): Promise<{
+    chunks: number;
+    boxes: number;
+    pages: number;
+}>;
+/** Read one chunk and all of its provenance boxes by stable id. */
+/**
+ * Resolve a path (original or derived Markdown sibling) plus an optional 1-based
+ * line to the chunk that contains it. The viewer needs this for inline
+ * `.docx.md:73` links, whose label is derived text but whose citation must point at
+ * the original document's chunk.
+ */
+export declare function findChunkForPath(options: {
+    indexPath: string;
+    path: string;
+    line?: number;
+    /** The text on the requested markdown line, used when stored line anchors are stale. */
+    lineText?: string;
+}): Promise<{
+    docId: number;
+    chunkId: number;
+} | undefined>;
+/** Resolve a stable chunk_key (or an old numeric id) to the current numeric id. */
+export declare function findChunkId(options: {
+    indexPath: string;
+    docId: number;
+    chunkId: number | string;
+}): Promise<number | undefined>;
+export declare function readChunkProvenance(options: {
+    indexPath: string;
+    docId: number;
+    chunkId: number;
+    query?: string;
+}): ReturnType<typeof readChunkProvenanceUnlocked>;
+declare function readChunkProvenanceUnlocked(options: {
+    indexPath: string;
+    docId: number;
+    chunkId: number;
+    query?: string;
+}): Promise<{
+    docId: number;
+    chunkId: number;
+    chunkKey?: string;
+    path: string;
+    sourcePath: string;
+    markdownPath: string;
+    heading?: string;
+    paragraph?: number;
+    chunkOrdinal?: number;
+    paragraphText?: string;
+    quote?: string;
+    sheet?: string;
+    cell?: string;
+    chunkText: string;
+    paragraphs: Array<{
+        ordinal: number;
+        text: string;
+    }>;
+    snippet: string;
+    line?: number;
+    lineStart?: number;
+    lineEnd?: number;
+    boxes: FtsChunkBox[];
+    unionBoxes: Array<{
+        page: number;
+        bbox: string;
+    }>;
+    matchedTerms?: string[];
+    highlightText?: string;
+    highlightRanges?: Array<{
+        start: number;
+        end: number;
+    }>;
+} | undefined>;
+export declare function optimizeIndex(indexPath: string): Promise<void>;
+/** Absolute path of one document's generated INDEX.md (outside the agent-visible tree). */
+export declare function outlineFilePath(indexPath: string, docId: number): string;
+/** Write one generated outline beside the SQLite index; never into the source tree. */
+export declare function writeOutlineFile(indexPath: string, docId: number, markdown: string): Promise<string>;
+/** One stored outline, as `kb_outline` returns it. */
+export interface DocumentOutlineRecord {
+    docId: number;
+    /** Original source path (the citable document). */
+    path: string;
+    /** Derived Markdown path; internal routing detail. */
+    markdownPath: string;
+    title: string;
+    summary: string;
+    summaryVersion?: string;
+    /** How the current summary was produced. Missing summaries are `none`. */
+    summaryKind: OutlineSummaryKind;
+    /** `missing` until an explicit `kb_outline_save` writes a model-authored summary. */
+    summaryStatus: 'ready' | 'missing';
+    /** Source chunk ids behind the document summary. */
+    sourceChunkIds: number[];
+    toc: OutlineTocEntry[];
+    sections: OutlineSection[];
+    /** Absolute path of the generated INDEX.md. */
+    indexPath: string;
+}
+/** Read one stored outline without rebuilding it. */
+export declare function readDocumentOutline(options: {
+    indexPath: string;
+    docId: number;
+}): Promise<DocumentOutlineRecord | undefined>;
+/**
+ * Rewrite one document's INDEX.md from the summaries already stored in SQLite.
+ *
+ * This is deliberately not a summary generator: it reads `kb_docs`/`kb_sections`
+ * and re-renders the file, so a renderer-only change (for example adding article
+ * headings) never regenerates or rewrites any summary text. Permission is decided
+ * by the caller on the source path before this function runs.
+ */
+export declare function rewriteDocumentOutline(options: {
+    indexPath: string;
+    docId: number;
+}): Promise<string | undefined>;
+/**
+ * Ensure one document has an outline and an INDEX.md.
+ *
+ * Upload and extraction only guarantee the navigation structure: TOC rows and section
+ * records exist, but summaries stay empty unless `autoGenerate` explicitly asks for the
+ * legacy extractive text. A model-written (`llm`) summary is never regenerated here: it is
+ * preserved until the source hash changes or `kb_outline_save` replaces it.
+ */
+export declare function ensureDocumentOutline(options: {
+    indexPath: string;
+    docId: number;
+    force?: boolean;
+    /** Opt-in legacy extractive summaries; production config defaults this to false. */
+    autoGenerate?: boolean;
+}): Promise<DocumentOutlineRecord | undefined>;
+/** Statistics for one successful outline save. */
+export interface OutlineSaveStats {
+    /** Document summary written. */
+    docSummary: boolean;
+    /** Section rows replaced (the full heading tree, not only submitted rows). */
+    sections: number;
+    /** Sections whose non-empty summary was written. */
+    sectionSummaries: number;
+    /** Key facts written across all sections. */
+    keyFacts: number;
+    /** Rows inserted into `kb_summary_chunks` (and its FTS mirror). */
+    summaryChunks: number;
+    /** Generated INDEX.md path. */
+    indexPath: string;
+}
+/** Result of `kb_outline_save`: either the write statistics or the refused fields. */
+export interface OutlineSaveResult {
+    ok: boolean;
+    docId: number;
+    summaryVersion?: string;
+    summaryKind: 'llm' | 'none';
+    written: OutlineSaveStats;
+    rejected: OutlineSaveRejection[];
+}
+/**
+ * Persist one model-authored summary set.
+ *
+ * Validation runs before the transaction, so a refused field writes nothing. The write
+ * itself replaces `kb_docs.summary`/`toc_json`/`summary_version`/`summary_kind`, every
+ * `kb_sections` row, the `kb_summary_chunks` rows and their FTS mirror, then rewrites the
+ * document's INDEX.md. Section structure (ids, heading paths, own chunks) is never
+ * invented: unknown ids and foreign chunk ids are refused rather than created.
+ */
+export declare function saveDocumentOutline(options: {
+    indexPath: string;
+    docId: number;
+    summary: string;
+    sections: readonly OutlineSectionSubmission[];
+    summaryVersion?: string;
+}): Promise<OutlineSaveResult | undefined>;
+export {};