@zenera/rag 1.1.10 → 1.1.11

package/dist/schema/store.js

@@ -19,27 +19,69 @@ import { lancePath } from "./files.js";
 const TABLE = 'entities';
 /** Below this an IVF index has nothing to train on, and a flat scan is faster. */
 const VECTOR_INDEX_MIN_ROWS = 2000;
+/**
+ * Rows per write.
+ *
+ * Arrow addresses a batch's buffers with 32-bit offsets, so one batch carrying
+ * more than 2 GiB does not error — it panics inside the reader, on a thread
+ * whose panic never reaches this one.
+ */
+const WRITE_BATCH = 8192;
 const KINDS = new Set(['method', 'type', 'property']);
 const DIRECTIONS = new Set(['input', 'output', 'both', 'none']);
 const METHOD_TYPES = new Set(['read_only', 'read_write', 'n/a']);
-export async function writeStore(dir, rows, vectors) {
-    if (rows.length === 0) {
-        throw new CliError('the documents describe nothing to index', EXIT.invalid, 'they have no operations and no component schemas');
+export async function openStore(dir) {
+    return new Writer(await connect(lancePath(dir)));
+}
+class Writer {
+    #db;
+    #table;
+    #rows = 0;
+    constructor(db) {
+        this.#db = db;
     }
-    const db = await connect(lancePath(dir));
-    // Every column is always populated — never null — so the Arrow schema is
-    // inferred from the first row without a declaration to keep in step.
-    const table = await db.createTable(TABLE, rows.map((row, i) => ({ ...row, vector: vectors[i] })), { mode: 'overwrite' });
-    await table.createIndex('text', { config: Index.fts() });
-    for (const column of ['kind', 'direction', 'methodType']) {
-        await table.createIndex(column, { config: Index.bitmap() });
+    async add(rows, vectors) {
+        for (let from = 0; from < rows.length; from += WRITE_BATCH) {
+            const batch = rows.slice(from, from + WRITE_BATCH).map((row, i) => ({
+                ...row,
+                vector: vectors[from + i],
+            }));
+            if (this.#table) {
+                await this.#table.add(batch);
+            }
+            else {
+                // Every column is always populated — never null — so the Arrow
+                // schema is inferred from the first row without a declaration.
+                this.#table = await this.#db.createTable(TABLE, batch, { mode: 'overwrite' });
+            }
+            this.#rows += batch.length;
+        }
     }
-    const vector = rows.length >= VECTOR_INDEX_MIN_ROWS;
-    if (vector) {
-        await table.createIndex('vector');
+    async finish() {
+        const table = this.#table;
+        if (!table) {
+            throw new CliError('the documents describe nothing to index', EXIT.invalid, 'they have no operations and no component schemas');
+        }
+        // A writer that panicked did not reject, so a short table is the only
+        // evidence left that the rows never landed.
+        const stored = await table.countRows();
+        if (stored !== this.#rows) {
+            throw new CliError(`the table holds ${stored} of ${this.#rows} entities`, EXIT.failed, 'the write did not finish, and a partial index answers as if it were whole');
+        }
+        await table.createIndex('text', { config: Index.fts() });
+        for (const column of ['kind', 'direction', 'methodType']) {
+            await table.createIndex(column, { config: Index.bitmap() });
+        }
+        const vector = this.#rows >= VECTOR_INDEX_MIN_ROWS;
+        if (vector) {
+            await table.createIndex('vector');
+        }
+        this.#db.close();
+        return { rows: this.#rows, fts: true, vector };
+    }
+    close() {
+        this.#db.close();
     }
-    db.close();
-    return { rows: rows.length, fts: true, vector };
 }
 export class EntityStore {
     #db;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.10",
+    "version": "1.1.11",
     "description": "Retrieval over a corpus: openapi/swagger documents as a searchable graph, markdown and text as searchable passages.",
     "keywords": [
         "agents",
@@ -61,8 +61,8 @@
     "dependencies": {
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
-        "@zenera/cli": "^1.1.10",
-        "@zenera/neo": "^1.1.10",
+        "@zenera/cli": "^1.1.11",
+        "@zenera/neo": "^1.1.11",
         "graphology": "^0.26.0",
         "remark-frontmatter": "^5.0.0",
         "remark-gfm": "^4.0.1",