@zenera/rag 1.1.9 → 1.1.11

package/dist/schema/search.js

@@ -1,3 +1,4 @@
+import { CliError, EXIT } from '@zenera/cli/lib';
 import { assertSameEmbedding } from "../common/manifest.js";
 import { openIndex } from "./files.js";
 import { EntityStore } from "./store.js";
@@ -21,7 +22,8 @@ export class SchemaIndex {
     static async open(dir, embedder) {
         const index = await openIndex(dir);
         assertSameEmbedding(index.manifest, embedder.id);
-        return new SchemaIndex(index, await EntityStore.open(dir), embedder);
+        const names = index.manifest.sources.map((s) => s.name);
+        return new SchemaIndex(index, await EntityStore.open(dir, names), embedder);
     }
     schemas() {
         return this.#index.schemas();
@@ -33,6 +35,7 @@ export class SchemaIndex {
         this.#store.close();
     }
     async search(query, signal) {
+        this.#assertSources(query.sources);
         const terms = termsOf(query);
         if (terms.length === 0) {
             return { seeds: [], subgraphs: [], empty: [] };
@@ -70,11 +73,24 @@ export class SchemaIndex {
         });
         return { seeds, subgraphs, empty };
     }
+    /** Settled before the embedder is called, so a typo costs no credential. */
+    #assertSources(wanted) {
+        if (!wanted || wanted.length === 0) {
+            return;
+        }
+        const known = this.manifest.sources.map((s) => s.name);
+        const missing = wanted.filter((name) => !known.includes(name));
+        if (missing.length > 0) {
+            throw new CliError(`this index holds no document called ${missing.join(', ')}`, EXIT.failed, `it has: ${known.join(', ')}`);
+        }
+    }
 }
 // ---------------------------------------------------------------------------
 function termsOf(query) {
     const method = methodTypes(query.method_type);
     const loose = query.direction ?? 'any';
+    // A document is a constraint on the whole question, not on one field of it.
+    const sources = query.sources?.length ? query.sources : undefined;
     return [
         ...group(query.all, 'all', { methodTypes: method.mixed }),
         ...group(query.methods, 'methods', { kinds: ['method'], methodTypes: method.only }),
@@ -96,7 +112,7 @@ function termsOf(query) {
             kinds: ['property'],
             directions: sides('output'),
         }),
-    ];
+    ].map((term) => (sources ? { ...term, filter: { ...term.filter, sources } } : term));
 }
 function group(texts, field, filter) {
     return (texts ?? [])

package/dist/schema/store.d.ts

@@ -4,6 +4,7 @@ export interface StoreFilter {
     kinds?: readonly string[];
     directions?: readonly string[];
     methodTypes?: readonly string[];
+    sources?: readonly string[];
 }
 export interface Hit {
     record: EntityRecord;
@@ -17,11 +18,20 @@ export interface WriteResult {
     fts: boolean;
     vector: boolean;
 }
-export declare function writeStore(dir: string, rows: readonly EntityRecord[], vectors: readonly Float32Array[]): Promise<WriteResult>;
+/** A table built a window at a time, so peak memory is not a function of size. */
+export interface EntityWriter {
+    add(rows: readonly EntityRecord[], vectors: readonly Float32Array[]): Promise<void>;
+    /** Builds the indexes and closes. Throws if nothing was ever added. */
+    finish(): Promise<WriteResult>;
+    /** For a build that failed, so the connection does not outlive it. */
+    close(): void;
+}
+export declare function openStore(dir: string): Promise<EntityWriter>;
 export declare class EntityStore {
     #private;
-    constructor(db: Connection, table: Table);
-    static open(dir: string): Promise<EntityStore>;
+    constructor(db: Connection, table: Table, sources?: readonly string[]);
+    /** `sources` is the document vocabulary a `sources` filter is checked against. */
+    static open(dir: string, sources?: readonly string[]): Promise<EntityStore>;
     /**
      * One hybrid query: the same string goes to the full-text side and, as a
      * vector, to the nearest-neighbour side, and LanceDB fuses the two.

package/dist/schema/store.js

@@ -8,47 +8,95 @@ import { lancePath } from "./files.js";
 // carrying both the embedding and the full-text index, and the handful of
 // enum columns a query filters on.
 //
-// Those enums are the *only* thing that reaches the SQL predicate. Exclusion
-// lists — which arrive from a model, or from a shell — are applied afterwards
-// in JavaScript. Escaping them into `where()` would work right up until it did
-// not, and there is nothing here that a filter string buys.
+// Those closed vocabularies are the *only* thing that reaches the SQL
+// predicate. `source` is one of them: its values are the document names the
+// index itself wrote, so the store is opened with that list and checks against
+// it. Exclusion lists — which arrive from a model, or from a shell — are
+// applied afterwards in JavaScript. Escaping those into `where()` would work
+// right up until it did not, and there is nothing here that a filter string
+// buys.
 // ---------------------------------------------------------------------------
 const TABLE = 'entities';
 /** Below this an IVF index has nothing to train on, and a flat scan is faster. */
 const VECTOR_INDEX_MIN_ROWS = 2000;
+/**
+ * Rows per write.
+ *
+ * Arrow addresses a batch's buffers with 32-bit offsets, so one batch carrying
+ * more than 2 GiB does not error — it panics inside the reader, on a thread
+ * whose panic never reaches this one.
+ */
+const WRITE_BATCH = 8192;
 const KINDS = new Set(['method', 'type', 'property']);
 const DIRECTIONS = new Set(['input', 'output', 'both', 'none']);
 const METHOD_TYPES = new Set(['read_only', 'read_write', 'n/a']);
-export async function writeStore(dir, rows, vectors) {
-    if (rows.length === 0) {
-        throw new CliError('the documents describe nothing to index', EXIT.invalid, 'they have no operations and no component schemas');
+export async function openStore(dir) {
+    return new Writer(await connect(lancePath(dir)));
+}
+class Writer {
+    #db;
+    #table;
+    #rows = 0;
+    constructor(db) {
+        this.#db = db;
     }
-    const db = await connect(lancePath(dir));
-    // Every column is always populated — never null — so the Arrow schema is
-    // inferred from the first row without a declaration to keep in step.
-    const table = await db.createTable(TABLE, rows.map((row, i) => ({ ...row, vector: vectors[i] })), { mode: 'overwrite' });
-    await table.createIndex('text', { config: Index.fts() });
-    for (const column of ['kind', 'direction', 'methodType']) {
-        await table.createIndex(column, { config: Index.bitmap() });
+    async add(rows, vectors) {
+        for (let from = 0; from < rows.length; from += WRITE_BATCH) {
+            const batch = rows.slice(from, from + WRITE_BATCH).map((row, i) => ({
+                ...row,
+                vector: vectors[from + i],
+            }));
+            if (this.#table) {
+                await this.#table.add(batch);
+            }
+            else {
+                // Every column is always populated — never null — so the Arrow
+                // schema is inferred from the first row without a declaration.
+                this.#table = await this.#db.createTable(TABLE, batch, { mode: 'overwrite' });
+            }
+            this.#rows += batch.length;
+        }
     }
-    const vector = rows.length >= VECTOR_INDEX_MIN_ROWS;
-    if (vector) {
-        await table.createIndex('vector');
+    async finish() {
+        const table = this.#table;
+        if (!table) {
+            throw new CliError('the documents describe nothing to index', EXIT.invalid, 'they have no operations and no component schemas');
+        }
+        // A writer that panicked did not reject, so a short table is the only
+        // evidence left that the rows never landed.
+        const stored = await table.countRows();
+        if (stored !== this.#rows) {
+            throw new CliError(`the table holds ${stored} of ${this.#rows} entities`, EXIT.failed, 'the write did not finish, and a partial index answers as if it were whole');
+        }
+        await table.createIndex('text', { config: Index.fts() });
+        for (const column of ['kind', 'direction', 'methodType']) {
+            await table.createIndex(column, { config: Index.bitmap() });
+        }
+        const vector = this.#rows >= VECTOR_INDEX_MIN_ROWS;
+        if (vector) {
+            await table.createIndex('vector');
+        }
+        this.#db.close();
+        return { rows: this.#rows, fts: true, vector };
+    }
+    close() {
+        this.#db.close();
     }
-    db.close();
-    return { rows: rows.length, fts: true, vector };
 }
 export class EntityStore {
     #db;
     #table;
-    constructor(db, table) {
+    #sources;
+    constructor(db, table, sources = []) {
         this.#db = db;
         this.#table = table;
+        this.#sources = new Set(sources);
     }
-    static async open(dir) {
+    /** `sources` is the document vocabulary a `sources` filter is checked against. */
+    static async open(dir, sources = []) {
         const db = await connect(lancePath(dir));
         try {
-            return new EntityStore(db, await db.openTable(TABLE));
+            return new EntityStore(db, await db.openTable(TABLE), sources);
         }
         catch {
             db.close();
@@ -60,7 +108,7 @@ export class EntityStore {
      * vector, to the nearest-neighbour side, and LanceDB fuses the two.
      */
     async search(text, vector, filter, limit) {
-        const predicate = where(filter);
+        const predicate = where(filter, this.#sources);
         let query = this.#table.query().nearestToText(text).nearestTo(vector).limit(limit);
         if (predicate) {
             query = query.where(predicate);
@@ -78,11 +126,12 @@ export class EntityStore {
 }
 // ---------------------------------------------------------------------------
 /** Closed vocabularies only. Anything else is a bug, and is treated as one. */
-function where(filter) {
+function where(filter, sources) {
     const clauses = [
         clause('kind', filter.kinds, KINDS),
         clause('direction', filter.directions, DIRECTIONS),
         clause('methodType', filter.methodTypes, METHOD_TYPES),
+        clause('source', filter.sources, sources),
     ].filter(Boolean);
     return clauses.join(' AND ');
 }

package/dist/schema/tools.js

@@ -1,9 +1,9 @@
 import { tool } from '@zenera/neo';
 import { loose, matcher, PatternError } from "../common/match.js";
-import { FORMATS, isFormat, present } from "./present.js";
-import { isEmpty, parseQuery, QueryError } from "./query.js";
 import { toTypeScript } from "./hydrate.js";
 import { fields, grepNodes, listNodes, propertyCount } from "./lookup.js";
+import { FORMATS, isFormat, present } from "./present.js";
+import { isEmpty, parseQuery, QueryError } from "./query.js";
 import { sourceTag } from "./render.js";
 import { stitch } from "./subgraph.js";
 import { chainOf, traceNodes } from "./trace.js";
@@ -47,6 +47,7 @@ export function schemaTools(index, options = {}) {
     // line is prompt spent saying the same word; with several it is the only
     // way to tell two revisions of one API apart.
     const source = options.source ?? index.manifest.sources.length > 1;
+    const names = index.manifest.sources.map((s) => s.name);
     const searchApi = tool({
         name: 'search_api',
         group: GROUP,
@@ -81,6 +82,17 @@ export function schemaTools(index, options = {}) {
                 exclude_methods: list('Operation names to leave out.'),
                 exclude_types: list('Schema names to leave out.'),
                 exclude_properties: list('Field names to leave out.'),
+                // Naming the documents is only a choice when there is more than
+                // one, and an enum is what stops a model inventing a third.
+                ...(names.length > 1
+                    ? {
+                        sources: {
+                            type: 'array',
+                            items: { type: 'string', enum: names },
+                            description: 'Search only these documents. Omit it to search all of them.',
+                        },
+                    }
+                    : {}),
                 limit: {
                     type: 'integer',
                     description: `Results per phrase. Default ${DEFAULT_LIMIT}.`,
@@ -111,6 +123,13 @@ export function schemaTools(index, options = {}) {
             if (isEmpty(query)) {
                 return { error: 'nothing was asked for', hint: 'fill at least one search field' };
             }
+            const absent = (query.sources ?? []).filter((name) => !names.includes(name));
+            if (absent.length > 0) {
+                return {
+                    error: `no document called ${absent.join(', ')}`,
+                    hint: `it has: ${names.join(', ')}`,
+                };
+            }
             const result = await index.search({
                 limit: DEFAULT_LIMIT,
                 max_nodes: DEFAULT_MAX_NODES,

package/package.json

@@ -1,11 +1,12 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.9",
-    "description": "Retrieval over API descriptions: openapi/swagger documents as a searchable graph.",
+    "version": "1.1.11",
+    "description": "Retrieval over a corpus: openapi/swagger documents as a searchable graph, markdown and text as searchable passages.",
     "keywords": [
         "agents",
         "ai-agents",
         "llm",
+        "markdown",
         "openapi",
         "rag",
         "vector-search"
@@ -35,6 +36,14 @@
         "./tools": {
             "types": "./dist/schema/tools.d.ts",
             "default": "./dist/schema/tools.js"
+        },
+        "./docs": {
+            "types": "./dist/docs/index.d.ts",
+            "default": "./dist/docs/index.js"
+        },
+        "./docs/tools": {
+            "types": "./dist/docs/tools.d.ts",
+            "default": "./dist/docs/tools.js"
         }
     },
     "files": [
@@ -52,8 +61,12 @@
     "dependencies": {
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
+        "@zenera/cli": "^1.1.11",
+        "@zenera/neo": "^1.1.11",
         "graphology": "^0.26.0",
-        "@zenera/cli": "^1.1.9",
-        "@zenera/neo": "^1.1.9"
+        "remark-frontmatter": "^5.0.0",
+        "remark-gfm": "^4.0.1",
+        "remark-parse": "^11.0.0",
+        "unified": "^11.0.5"
     }
 }