@zenera/rag 1.1.10 → 1.1.12

package/dist/schema/readme.js

@@ -1,6 +1,6 @@
 import { basename } from 'node:path';
 import { INTERVAL_MS } from "../common/progress.js";
-import { duration, fields, grid, message, plural, searched } from "../common/prose.js";
+import { breakdown, duration, fields, grid, message, plural, searched } from "../common/prose.js";
 export const PHASES = {
     reading: 'reading the documents',
     graph: 'building the graph',
@@ -22,9 +22,10 @@ function building(state) {
         rows.push(['found', entities(state.summary)]);
     }
     if (state.total > 0) {
-        const percent = Math.round((state.done / state.total) * 100);
+        const percent = Math.floor((state.done / state.total) * 100);
         rows.push(['embedded', `${state.done} of ${state.total} · ${percent}%`]);
     }
+    rows.push(['timing', breakdown(state.timings)]);
     rows.push(['updated', new Date(state.now).toISOString()]);
     return [
         '# Schema index — being built',
@@ -60,6 +61,8 @@ function complete(state) {
         `${entities(manifest.counts)},`,
         `${searched(manifest.indexes)}.`,
         '',
+        `Time: ${breakdown(state.timings)}.`,
+        '',
         '## Files',
         '',
         ...fields([
@@ -97,6 +100,7 @@ function failed(state) {
             ['step', state.step],
             ['reason', message(state.reason)],
             ['started', new Date(state.started).toISOString()],
+            ['timing', breakdown(state.timings)],
             [
                 'failed',
                 `${new Date().toISOString()} (after ${duration(Date.now() - state.started)})`,

package/dist/schema/store.d.ts

@@ -18,7 +18,15 @@ export interface WriteResult {
     fts: boolean;
     vector: boolean;
 }
-export declare function writeStore(dir: string, rows: readonly EntityRecord[], vectors: readonly Float32Array[]): Promise<WriteResult>;
+/** A table built a window at a time, so peak memory is not a function of size. */
+export interface EntityWriter {
+    add(rows: readonly EntityRecord[], vectors: readonly Float32Array[]): Promise<void>;
+    /** Builds the indexes and closes. Throws if nothing was ever added. */
+    finish(): Promise<WriteResult>;
+    /** For a build that failed, so the connection does not outlive it. */
+    close(): void;
+}
+export declare function openStore(dir: string): Promise<EntityWriter>;
 export declare class EntityStore {
     #private;
     constructor(db: Connection, table: Table, sources?: readonly string[]);

package/dist/schema/store.js

@@ -19,27 +19,69 @@ import { lancePath } from "./files.js";
 const TABLE = 'entities';
 /** Below this an IVF index has nothing to train on, and a flat scan is faster. */
 const VECTOR_INDEX_MIN_ROWS = 2000;
+/**
+ * Rows per write.
+ *
+ * Arrow addresses a batch's buffers with 32-bit offsets, so one batch carrying
+ * more than 2 GiB does not error — it panics inside the reader, on a thread
+ * whose panic never reaches this one.
+ */
+const WRITE_BATCH = 8192;
 const KINDS = new Set(['method', 'type', 'property']);
 const DIRECTIONS = new Set(['input', 'output', 'both', 'none']);
 const METHOD_TYPES = new Set(['read_only', 'read_write', 'n/a']);
-export async function writeStore(dir, rows, vectors) {
-    if (rows.length === 0) {
-        throw new CliError('the documents describe nothing to index', EXIT.invalid, 'they have no operations and no component schemas');
+export async function openStore(dir) {
+    return new Writer(await connect(lancePath(dir)));
+}
+class Writer {
+    #db;
+    #table;
+    #rows = 0;
+    constructor(db) {
+        this.#db = db;
     }
-    const db = await connect(lancePath(dir));
-    // Every column is always populated — never null — so the Arrow schema is
-    // inferred from the first row without a declaration to keep in step.
-    const table = await db.createTable(TABLE, rows.map((row, i) => ({ ...row, vector: vectors[i] })), { mode: 'overwrite' });
-    await table.createIndex('text', { config: Index.fts() });
-    for (const column of ['kind', 'direction', 'methodType']) {
-        await table.createIndex(column, { config: Index.bitmap() });
+    async add(rows, vectors) {
+        for (let from = 0; from < rows.length; from += WRITE_BATCH) {
+            const batch = rows.slice(from, from + WRITE_BATCH).map((row, i) => ({
+                ...row,
+                vector: vectors[from + i],
+            }));
+            if (this.#table) {
+                await this.#table.add(batch);
+            }
+            else {
+                // Every column is always populated — never null — so the Arrow
+                // schema is inferred from the first row without a declaration.
+                this.#table = await this.#db.createTable(TABLE, batch, { mode: 'overwrite' });
+            }
+            this.#rows += batch.length;
+        }
     }
-    const vector = rows.length >= VECTOR_INDEX_MIN_ROWS;
-    if (vector) {
-        await table.createIndex('vector');
+    async finish() {
+        const table = this.#table;
+        if (!table) {
+            throw new CliError('the documents describe nothing to index', EXIT.invalid, 'they have no operations and no component schemas');
+        }
+        // A writer that panicked did not reject, so a short table is the only
+        // evidence left that the rows never landed.
+        const stored = await table.countRows();
+        if (stored !== this.#rows) {
+            throw new CliError(`the table holds ${stored} of ${this.#rows} entities`, EXIT.failed, 'the write did not finish, and a partial index answers as if it were whole');
+        }
+        await table.createIndex('text', { config: Index.fts() });
+        for (const column of ['kind', 'direction', 'methodType']) {
+            await table.createIndex(column, { config: Index.bitmap() });
+        }
+        const vector = this.#rows >= VECTOR_INDEX_MIN_ROWS;
+        if (vector) {
+            await table.createIndex('vector');
+        }
+        this.#db.close();
+        return { rows: this.#rows, fts: true, vector };
+    }
+    close() {
+        this.#db.close();
     }
-    db.close();
-    return { rows: rows.length, fts: true, vector };
 }
 export class EntityStore {
     #db;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.10",
+    "version": "1.1.12",
     "description": "Retrieval over a corpus: openapi/swagger documents as a searchable graph, markdown and text as searchable passages.",
     "keywords": [
         "agents",
@@ -61,8 +61,8 @@
     "dependencies": {
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
-        "@zenera/cli": "^1.1.10",
-        "@zenera/neo": "^1.1.10",
+        "@zenera/cli": "^1.1.12",
+        "@zenera/neo": "^1.1.12",
         "graphology": "^0.26.0",
         "remark-frontmatter": "^5.0.0",
         "remark-gfm": "^4.0.1",