@zenera/rag 1.1.9 → 1.1.10

package/dist/docs/repl.js

@@ -0,0 +1,130 @@
+import { bold, cyan, dim, note, write } from '@zenera/cli/lib';
+import { createInterface } from 'node:readline/promises';
+import { assemble } from "./assemble.js";
+import { CHUNK_KINDS } from "./chunk.js";
+import { renderAssembly } from "./render.js";
+import { SEARCH_MODES } from "./search.js";
+export async function repl(index, initial, settings = {}) {
+    const rl = createInterface({ input: process.stdin, output: process.stderr });
+    let query = { ...initial };
+    note(bold(`${index.manifest.counts.documents} document(s), ${index.manifest.counts.chunks} chunks`));
+    note(dim('  a bare line searches; `help` lists the rest.'));
+    try {
+        if (query.query) {
+            query = await run(index, query, settings);
+        }
+        for (;;) {
+            const line = (await rl.question(cyan('docs> '))).trim();
+            if (line === '') {
+                continue;
+            }
+            if (line === 'quit' || line === 'exit') {
+                return;
+            }
+            if (line === 'help') {
+                help();
+                continue;
+            }
+            if (line === 'reset') {
+                query = {};
+                note(dim('  cleared, narrowings and exclusions alike'));
+                continue;
+            }
+            if (line === 'show') {
+                note(dim(`  ${JSON.stringify(query)}`));
+                continue;
+            }
+            if (line === 'files') {
+                for (const name of index.resolveFiles(query.files, query.exclude_files)) {
+                    note(dim(`  ${name}`));
+                }
+                continue;
+            }
+            const [head, ...rest] = line.split(' ');
+            const tail = rest.join(' ').trim();
+            const narrowed = narrow(query, head ?? '', tail);
+            if (narrowed) {
+                query = narrowed;
+                continue;
+            }
+            query = await run(index, { ...query, query: line }, settings);
+        }
+    }
+    finally {
+        rl.close();
+    }
+}
+/**
+ * The narrowing verbs. Each one replaces its own field rather than adding to
+ * it — `file guides/**` twice in a row means the second pattern, which is what
+ * anyone typing it meant. An empty argument clears the field.
+ */
+function narrow(query, head, tail) {
+    const set = (key, value) => {
+        note(dim(`  ${head} is ${tail || 'anything'}`));
+        return { ...query, [key]: value };
+    };
+    switch (head) {
+        case 'file':
+            return set('files', tail ? [tail] : undefined);
+        case 'not':
+            return set('exclude_files', tail ? [tail] : undefined);
+        case 'section':
+            return set('section', tail ? [tail] : undefined);
+        case 'kind':
+            if (tail && !CHUNK_KINDS.includes(tail)) {
+                note(dim(`  kind is one of ${CHUNK_KINDS.join(', ')}`));
+                return query;
+            }
+            return set('kinds', tail ? [tail] : undefined);
+        case 'mode':
+            if (tail && !SEARCH_MODES.includes(tail)) {
+                note(dim(`  mode is one of ${SEARCH_MODES.join(', ')}`));
+                return query;
+            }
+            return set('mode', (tail || undefined));
+        case 'limit': {
+            const limit = Number(tail);
+            if (!Number.isInteger(limit) || limit < 1) {
+                note(dim('  limit is a whole number of at least 1'));
+                return query;
+            }
+            return set('limit', limit);
+        }
+        default:
+            return undefined;
+    }
+}
+async function run(index, query, settings) {
+    const result = await index.search(query);
+    const excerpt = await assemble(index, result.matches, settings);
+    if (excerpt.files.length > 0) {
+        write(renderAssembly(excerpt, {}));
+    }
+    note(dim(`  ${result.matches.length} passage(s) in ${excerpt.files.length} document(s)` +
+        ` · ${excerpt.shown} line(s) · ${result.files.length} document(s) in scope`));
+    if (result.matches.length === 0) {
+        return query;
+    }
+    const seen = new Set([...(query.exclude_ids ?? []), ...result.matches.map((m) => m.id)]);
+    note(dim(`  ${seen.size} passage(s) now excluded — \`reset\` to see them again`));
+    return { ...query, exclude_ids: [...seen] };
+}
+function help() {
+    for (const line of [
+        '  <text>            search',
+        '  file <pattern>    only documents matching it; blank for all',
+        '  not <pattern>     drop documents matching it',
+        '  section <name>    only under this heading, by title, id or path',
+        `  kind <k>          ${CHUNK_KINDS.join(' | ')}`,
+        `  mode <m>          ${SEARCH_MODES.join(' | ')}`,
+        '  limit <n>         passages per search',
+        '  files             the documents currently in scope',
+        '  show              the query as it stands',
+        '  reset             forget it, narrowings and exclusions alike',
+        '  quit',
+    ]) {
+        note(dim(line));
+    }
+}
+//# sourceMappingURL=repl.js.map

package/dist/docs/search.d.ts

@@ -0,0 +1,92 @@
+import type { Embedder } from '@zenera/neo';
+import { type FileOutline, type HeadingRecord, type Manifest, type OpenDocs, type Outline } from './files.ts';
+import { ChunkStore } from './store.ts';
+export type SearchMode = 'hybrid' | 'vector' | 'text';
+export declare const SEARCH_MODES: readonly SearchMode[];
+export interface DocsQuery {
+    query?: string;
+    mode?: SearchMode;
+    /** patterns over the document name: a glob, a substring, or a regex */
+    files?: readonly string[];
+    exclude_files?: readonly string[];
+    /** a heading title, a structure id, or a structure path */
+    section?: readonly string[];
+    /** one of `CHUNK_KINDS` */
+    kinds?: readonly string[];
+    /** chunk ids already seen, so asking again moves on */
+    exclude_ids?: readonly string[];
+    limit?: number;
+}
+export interface Match {
+    id: string;
+    path: string;
+    kind: string;
+    headings: string;
+    structureId: string;
+    structurePath: string;
+    /** every line this chunk wants shown, headings and table header included */
+    lineNumbers: number[];
+    bodyStart: number;
+    bodyEnd: number;
+    text: string;
+    score: number;
+    /** where it placed in each leg, when it placed at all */
+    ranks: {
+        vector?: number;
+        text?: number;
+    };
+    /**
+     * And how well it did there. Fusion scores a rank, so its numbers say only
+     * that something placed first — a question the corpus cannot answer at all
+     * still returns a confident-looking `1 / (RRF_K + 0)`. These are what tell
+     * a caller whether first place was worth having.
+     */
+    relevance: {
+        vector?: number;
+        text?: number;
+    };
+}
+export interface DocsResult {
+    matches: Match[];
+    mode: SearchMode;
+    /** the documents the narrowings left in play */
+    files: string[];
+    /** the sections they left in play, as structure paths */
+    sections: string[];
+    /** distinct chunks either leg returned, before fusion trimmed them */
+    considered: number;
+}
+export declare const DEFAULT_LIMIT = 8;
+export declare class DocsIndex {
+    #private;
+    readonly manifest: Manifest;
+    readonly outline: Outline;
+    constructor(index: OpenDocs, store: ChunkStore, embedder?: Embedder);
+    /**
+     * An embedder is optional because half of what this index is for needs no
+     * model at all: listing, grepping and reading are exact, and asking for a
+     * credential to do them would be asking for nothing.
+     */
+    static open(dir: string, embedder?: Embedder): Promise<DocsIndex>;
+    /** A document, split the way every line number in this index counts it. */
+    lines(name: string): Promise<string[]>;
+    file(name: string): FileOutline | undefined;
+    /** Document names matching the patterns, or all of them when there are none. */
+    resolveFiles(patterns?: readonly string[], exclude?: readonly string[]): string[];
+    /**
+     * Headings the terms name, over the documents still in play. A term is tried
+     * as a structure id, then a structure path, then as a pattern over the
+     * title — so `--section overview` and `--section sec:3` are the same
+     * argument, and neither has to be explained.
+     */
+    resolveSections(terms: readonly string[], files: readonly string[]): HeadingRecord[];
+    close(): void;
+    search(query: DocsQuery, signal?: AbortSignal): Promise<DocsResult>;
+}
+/**
+ * Inside one of these sections, or anywhere when none were asked for. The
+ * boundary is a path separator and not a prefix, because `doc/sec:1` is a
+ * prefix of `doc/sec:10` and they are different sections.
+ */
+export declare const under: (path: string, sections: readonly string[]) => boolean;
+//# sourceMappingURL=search.d.ts.map

package/dist/docs/search.js

@@ -0,0 +1,251 @@
+import { assertSameEmbedding } from "../common/manifest.js";
+import { loose, PatternError } from "../common/match.js";
+import { CHUNK_KINDS, parseLines } from "./chunk.js";
+import { openIndex, readLines, } from "./files.js";
+import { ChunkStore } from "./store.js";
+export const SEARCH_MODES = ['hybrid', 'vector', 'text'];
+export const DEFAULT_LIMIT = 8;
+/** Reciprocal rank fusion; the constant is the usual one and damps the top. */
+const RRF_K = 60;
+/** How much to ask each leg for, so fusion and the caps have room to work. */
+const OVERFETCH = 6;
+/**
+ * And an absolute ceiling on it. The multiple exists to survive the JavaScript
+ * filter and the diversifier, which is a fixed cost, not one that grows with
+ * the limit — at `--limit 32` the multiple alone asked for 192 rows, which on
+ * a small corpus is most of it. Every row fetched votes in the fusion at full
+ * strength, so an over-deep leg does not add recall, it adds an electorate.
+ */
+const MAX_FETCH = 96;
+/**
+ * Floors for the per-section and per-document caps, and the share of the limit
+ * they grow by once it is large enough to matter.
+ *
+ * The caps were constants, which made them a diversity nudge at the default
+ * limit of eight and a straitjacket above it: `--limit 20` for a question whose
+ * answer is one section of one file returned five chunks of the answer and
+ * fifteen of whatever else scored, because the sixth chunk of the right file
+ * was evicted in favour of the first chunk of a worse one. Asking for more
+ * results is asking to go deeper, and depth on a documented topic means more
+ * of the document that covers it.
+ */
+const MAX_PER_SECTION = 3;
+const MAX_PER_FILE = 5;
+const SECTION_SHARE = 0.25;
+const FILE_SHARE = 0.5;
+export class DocsIndex {
+    manifest;
+    outline;
+    #index;
+    #store;
+    #embedder;
+    constructor(index, store, embedder) {
+        this.manifest = index.manifest;
+        this.outline = index.outline;
+        this.#index = index;
+        this.#store = store;
+        this.#embedder = embedder;
+    }
+    /**
+     * An embedder is optional because half of what this index is for needs no
+     * model at all: listing, grepping and reading are exact, and asking for a
+     * credential to do them would be asking for nothing.
+     */
+    static async open(dir, embedder) {
+        const index = await openIndex(dir);
+        if (embedder) {
+            assertSameEmbedding(index.manifest, embedder.id);
+        }
+        return new DocsIndex(index, await ChunkStore.open(dir), embedder);
+    }
+    /** A document, split the way every line number in this index counts it. */
+    lines(name) {
+        return readLines(this.#index, name);
+    }
+    file(name) {
+        return this.outline.files.find((f) => f.name === name);
+    }
+    /** Document names matching the patterns, or all of them when there are none. */
+    resolveFiles(patterns = [], exclude = []) {
+        const names = this.manifest.sources.map((s) => s.name);
+        const keep = patterns.length === 0 ? names : names.filter(anyMatch(patterns));
+        return exclude.length === 0 ? keep : keep.filter((name) => !anyMatch(exclude)(name));
+    }
+    /**
+     * Headings the terms name, over the documents still in play. A term is tried
+     * as a structure id, then a structure path, then as a pattern over the
+     * title — so `--section overview` and `--section sec:3` are the same
+     * argument, and neither has to be explained.
+     */
+    resolveSections(terms, files) {
+        if (terms.length === 0) {
+            return [];
+        }
+        const within = new Set(files);
+        const out = [];
+        for (const file of this.outline.files) {
+            if (!within.has(file.name)) {
+                continue;
+            }
+            for (const heading of file.headings) {
+                const hit = terms.some((term) => term === heading.id ||
+                    term === heading.path ||
+                    anyMatch([term])(heading.title));
+                if (hit) {
+                    out.push(heading);
+                }
+            }
+        }
+        return out;
+    }
+    close() {
+        this.#store.close();
+    }
+    async search(query, signal) {
+        const mode = query.mode ?? 'hybrid';
+        const text = (query.query ?? '').trim();
+        const limit = Math.max(1, query.limit ?? DEFAULT_LIMIT);
+        const files = this.resolveFiles(query.files, query.exclude_files);
+        const headings = this.resolveSections(query.section ?? [], files);
+        const sections = headings.map((h) => h.path);
+        // Two ways to ask for nothing: a pattern no document answers to, and a
+        // section no document has. Both are told, not silently widened.
+        const impossible = files.length === 0 || ((query.section?.length ?? 0) > 0 && sections.length === 0);
+        if (impossible || !text) {
+            return { matches: [], mode, files, sections, considered: 0 };
+        }
+        const filter = this.#filter(query, files, sections);
+        const excluded = new Set(query.exclude_ids ?? []);
+        const fetch = Math.min(limit * OVERFETCH, MAX_FETCH) + excluded.size;
+        const vector = mode === 'text' ? [] : await this.#nearest(text, filter, fetch, signal);
+        const lexical = mode === 'vector' ? [] : await this.#store.matching(text, filter, fetch);
+        const fused = fuse(vector, lexical).filter((match) => !excluded.has(match.id) &&
+            files.includes(match.path) &&
+            under(match.structurePath, sections));
+        return {
+            matches: diversify(fused, limit),
+            mode,
+            files,
+            sections,
+            considered: fused.length,
+        };
+    }
+    async #nearest(text, filter, limit, signal) {
+        if (!this.#embedder) {
+            throw new Error('this index was opened without an embedder, so it cannot be searched');
+        }
+        const response = await this.#embedder.embed({
+            input: [text],
+            taskType: 'query',
+            signal,
+        });
+        return await this.#store.nearest(Float32Array.from(response.vectors[0]), filter, limit);
+    }
+    /**
+     * The SQL side of the narrowing. A short, safe file list becomes a
+     * predicate; a long or awkward one is left to the JavaScript filter that
+     * runs afterwards regardless, since that is what makes the answer correct.
+     */
+    #filter(query, files, sections) {
+        const all = this.manifest.sources.length;
+        return {
+            kinds: kindsOf(query.kinds),
+            paths: files.length < all && files.length <= 64 ? files : undefined,
+            prefixes: sections.length > 0 && sections.length <= 64 ? sections : undefined,
+        };
+    }
+}
+// ---------------------------------------------------------------------------
+const anyMatch = (patterns) => {
+    const matchers = patterns.map((pattern) => loose(pattern));
+    return (value) => matchers.some((match) => match(value));
+};
+/**
+ * Inside one of these sections, or anywhere when none were asked for. The
+ * boundary is a path separator and not a prefix, because `doc/sec:1` is a
+ * prefix of `doc/sec:10` and they are different sections.
+ */
+export const under = (path, sections) => sections.length === 0 ||
+    sections.some((section) => path === section || path.startsWith(`${section}/`));
+function kindsOf(kinds) {
+    if (!kinds || kinds.length === 0) {
+        return undefined;
+    }
+    for (const kind of kinds) {
+        if (!CHUNK_KINDS.includes(kind)) {
+            throw new PatternError(`${kind} is not a kind of block: it is one of ${CHUNK_KINDS.join(', ')}`);
+        }
+    }
+    return [...kinds];
+}
+/**
+ * Reciprocal rank fusion over the chunk id. Fusing on the id rather than on a
+ * physical row means the ordering survives a compaction of the store, and it is
+ * also the only key the two legs are guaranteed to agree on.
+ */
+function fuse(vector, lexical) {
+    const merged = new Map();
+    const add = (hits, leg) => {
+        for (const hit of hits) {
+            const existing = merged.get(hit.record.id) ?? match(hit);
+            existing.score += 1 / (RRF_K + hit.rank);
+            existing.ranks[leg] = hit.rank;
+            existing.relevance[leg] = hit.relevance;
+            merged.set(hit.record.id, existing);
+        }
+    };
+    add(vector, 'vector');
+    add(lexical, 'text');
+    return [...merged.values()].sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
+}
+function match(hit) {
+    const { record } = hit;
+    return {
+        id: record.id,
+        path: record.path,
+        kind: record.kind,
+        headings: record.headings,
+        structureId: record.structureId,
+        structurePath: record.structurePath,
+        lineNumbers: parseLines(record.lineSpec),
+        bodyStart: record.bodyStart,
+        bodyEnd: record.bodyEnd,
+        text: record.text,
+        score: 0,
+        ranks: {},
+        relevance: {},
+    };
+}
+/**
+ * Caps per section and per document, applied in score order. A question whose
+ * answer really is one section still gets it — the cap only stops a run of
+ * near-identical neighbours from crowding out the second place a thing is said.
+ */
+function diversify(matches, limit) {
+    const sectionCap = Math.max(MAX_PER_SECTION, Math.ceil(limit * SECTION_SHARE));
+    const fileCap = Math.max(MAX_PER_FILE, Math.ceil(limit * FILE_SHARE));
+    const perSection = new Map();
+    const perFile = new Map();
+    const kept = [];
+    const held = [];
+    for (const item of matches) {
+        const section = `${item.path}\u0000${parentOf(item.structurePath)}`;
+        const sections = perSection.get(section) ?? 0;
+        const files = perFile.get(item.path) ?? 0;
+        if (sections >= sectionCap || files >= fileCap) {
+            held.push(item);
+            continue;
+        }
+        perSection.set(section, sections + 1);
+        perFile.set(item.path, files + 1);
+        kept.push(item);
+        if (kept.length === limit) {
+            return kept;
+        }
+    }
+    // Better a crowded answer than a short one: what the caps held back fills
+    // the rest, still in score order.
+    return [...kept, ...held].slice(0, limit);
+}
+const parentOf = (path) => path.slice(0, path.lastIndexOf('/')) || path;
+//# sourceMappingURL=search.js.map

package/dist/docs/store.d.ts

@@ -0,0 +1,55 @@
+import { type Connection, type Table } from '@lancedb/lancedb';
+export interface ChunkRecord {
+    /** `${path}#c${ordinal}` — the fusion key, and stable across compaction */
+    id: string;
+    /** the document's name in the index: its relative path */
+    path: string;
+    ordinal: number;
+    kind: string;
+    /** the full-text document — wider */
+    text: string;
+    /** what the vector was made from — tighter */
+    embedText: string;
+    /** the render set, run-length encoded: `1,5,12-40` */
+    lineSpec: string;
+    bodyStart: number;
+    bodyEnd: number;
+    structureId: string;
+    structurePath: string;
+    headings: string;
+    tokens: number;
+}
+export interface StoreFilter {
+    kinds?: readonly string[];
+    /** document names, exactly as the manifest spells them */
+    paths?: readonly string[];
+    /** structure path prefixes, as `structurePath LIKE 'x%'` */
+    prefixes?: readonly string[];
+}
+export interface Hit {
+    record: ChunkRecord;
+    /** 0-based position in this leg's own result list */
+    rank: number;
+    relevance: number;
+}
+export interface WriteResult {
+    rows: number;
+    fts: boolean;
+    vector: boolean;
+}
+export declare function writeChunks(dir: string, rows: readonly ChunkRecord[], vectors: readonly Float32Array[]): Promise<WriteResult>;
+export declare class ChunkStore {
+    #private;
+    constructor(db: Connection, table: Table);
+    static open(dir: string): Promise<ChunkStore>;
+    /** Nearest neighbours. Every row has a vector, so nothing has to be excluded. */
+    nearest(vector: Float32Array, filter: StoreFilter, limit: number): Promise<Hit[]>;
+    /**
+     * The lexical leg. A missing full-text index, or a query the tokenizer
+     * makes nothing of, answers with nothing rather than throwing: a hybrid
+     * search that loses one leg is a worse search, not a failed one.
+     */
+    matching(text: string, filter: StoreFilter, limit: number): Promise<Hit[]>;
+    close(): void;
+}
+//# sourceMappingURL=store.d.ts.map