@zenera/rag 1.1.34 → 1.1.35

package/dist/common/progress.d.ts

@@ -69,4 +69,19 @@ export interface Journal<S, M, P extends string> {
  * interleave their store writes and leave one that neither of them describes.
  */
 export declare function beginBuild<S, M, P extends string>(plan: BuildPlan<S, M, P>): Journal<S, M, P>;
+export declare function span(ms: number): string;
+/**
+ * How long is left at the pace since `first`, or undefined until there is a
+ * pace. Measured from the first report rather than the start, so vectors the
+ * cache already held do not make the rest look free.
+ */
+export declare function remaining(first: {
+    at: number;
+    done: number;
+}, done: number, total: number, now: number): number | undefined;
+/**
+ * The embedding narration. A model that takes one text per request reports
+ * once per text, so it is throttled; the last report always gets through.
+ */
+export declare function embedReporter(say: (line: string) => void, started: number, clock?: () => number): (done: number, total: number) => void;
 //# sourceMappingURL=progress.d.ts.map

package/dist/common/progress.js

@@ -142,4 +142,43 @@ export function beginBuild(plan) {
 function claim(path, lock) {
     claimLock(path, lock, (held) => new CliError(`this index is already being built (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `wait for it, or build elsewhere with --out`));
 }
+export function span(ms) {
+    const seconds = Math.round(ms / 1000);
+    if (seconds < 60) {
+        return `${seconds}s`;
+    }
+    const minutes = Math.floor(seconds / 60);
+    return minutes < 60
+        ? `${minutes}m${seconds % 60}s`
+        : `${Math.floor(minutes / 60)}h${minutes % 60}m`;
+}
+/**
+ * How long is left at the pace since `first`, or undefined until there is a
+ * pace. Measured from the first report rather than the start, so vectors the
+ * cache already held do not make the rest look free.
+ */
+export function remaining(first, done, total, now) {
+    const went = done - first.done;
+    const took = now - first.at;
+    return went > 0 && took > 0 && done < total ? ((total - done) * took) / went : undefined;
+}
+/**
+ * The embedding narration. A model that takes one text per request reports
+ * once per text, so it is throttled; the last report always gets through.
+ */
+export function embedReporter(say, started, clock = Date.now) {
+    let first;
+    let saidAt = -Infinity;
+    return (done, total) => {
+        const now = clock();
+        first ??= { at: now, done };
+        if (done < total && now - saidAt < INTERVAL_MS) {
+            return;
+        }
+        saidAt = now;
+        const left = remaining(first, done, total, now);
+        say(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${span(now - started)}` +
+            (left === undefined ? '' : ` · ~${span(left)} left`));
+    };
+}
 //# sourceMappingURL=progress.js.map

package/dist/docs/command.js

@@ -5,7 +5,7 @@ import { join, relative, resolve } from 'node:path';
 import { resolveEmbedder } from "../common/embedder.js";
 import { locateIndex, outputDir } from "../common/locate.js";
 import { assertSameEmbedding, inspectIndex } from "../common/manifest.js";
-import { INTERVAL_MS } from "../common/progress.js";
+import { embedReporter, INTERVAL_MS, span } from "../common/progress.js";
 import { breakdown, grid, indexing } from "../common/prose.js";
 import { announce, reportReady, stagingFor, swapIn } from "../common/restore.js";
 import { assemble, DEFAULT_MAX_LINES } from "./assemble.js";
@@ -282,7 +282,7 @@ async function index(args, ctx) {
         cacheDir: values['cache-dir'] ? cacheDir : undefined,
         onReading: loud
             ? throttled((done, total, pending) => note(dim(`  parsed ${done}/${total} · ${Math.floor((done / total) * 100)}% · ` +
-                `${elapsed(started)}${still(pending)}`)))
+                `${span(Date.now() - started)}${still(pending)}`)))
             : undefined,
         onRead: loud
             ? (summary) => {
@@ -296,9 +296,7 @@ async function index(args, ctx) {
                 note(dim(`  embedding ${summary.counts.chunks} chunks with ${chosen.id} …`));
             }
             : undefined,
-        onProgress: loud
-            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
-            : undefined,
+        onProgress: loud ? embedReporter((line) => note(dim(line)), started) : undefined,
         onWriting: loud ? (rows) => notes(indexing(rows, 'chunks').map(aside)) : undefined,
     });
     if (ctx.json) {
@@ -376,9 +374,7 @@ async function restore(args, ctx) {
         dimensions,
         cache: !values['no-cache'],
         cacheDir: values['cache-dir'] ? cacheDir : undefined,
-        onProgress: loud
-            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
-            : undefined,
+        onProgress: loud ? embedReporter((line) => note(dim(line)), started) : undefined,
         onWriting: loud ? (rows) => notes(indexing(rows, 'chunks').map(aside)) : undefined,
     });
     await swapIn(dir, staging);
@@ -428,10 +424,6 @@ function printSources(sources) {
     note();
 }
 const column = (source, at) => [source.lines, source.sections, source.tables, source.chunks][at] ?? 0;
-function elapsed(since) {
-    const seconds = Math.round((Date.now() - since) / 1000);
-    return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m${seconds % 60}s`;
-}
 /** Reading fires per document, and a line per document is not a narration. */
 function throttled(say) {
     let saidAt = 0;

package/dist/schema/command.js

@@ -5,6 +5,7 @@ import { join, relative, resolve } from 'node:path';
 import { resolveEmbedder } from "../common/embedder.js";
 import { locateIndex, outputDir } from "../common/locate.js";
 import { assertSameEmbedding, inspectIndex } from "../common/manifest.js";
+import { embedReporter } from "../common/progress.js";
 import { breakdown, indexing } from "../common/prose.js";
 import { announce, reportReady, stagingFor, swapIn } from "../common/restore.js";
 import { buildIndex } from "./build.js";
@@ -267,9 +268,7 @@ async function index(args, ctx) {
                 note(dim(`  embedding ${summary.counts.entities} entities with ${chosen.id} …`));
             }
             : undefined,
-        onProgress: loud
-            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
-            : undefined,
+        onProgress: loud ? embedReporter((line) => note(dim(line)), started) : undefined,
         onWriting: loud ? (rows) => notes(indexing(rows, 'entities').map(aside)) : undefined,
     });
     if (ctx.json) {
@@ -330,9 +329,7 @@ async function restore(args, ctx) {
         dimensions,
         cache: !values['no-cache'],
         cacheDir: values['cache-dir'] ? cacheDir : undefined,
-        onProgress: loud
-            ? (done, total) => note(dim(`  embedded ${done}/${total} · ${Math.floor((done / total) * 100)}% · ${elapsed(started)}`))
-            : undefined,
+        onProgress: loud ? embedReporter((line) => note(dim(line)), started) : undefined,
         onWriting: loud ? (rows) => notes(indexing(rows, 'entities').map(aside)) : undefined,
     });
     await swapIn(dir, staging);
@@ -373,10 +370,6 @@ async function ready(args, ctx) {
     reportReady(health, ctx, values.quiet === true);
 }
 const HEADERS = ['PATHS', 'OPERATIONS', 'SCHEMAS', 'FIELDS'];
-function elapsed(since) {
-    const seconds = Math.round((Date.now() - since) / 1000);
-    return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m${seconds % 60}s`;
-}
 function printSources(sources) {
     const rows = sources.map((s) => ({
         name: s.file,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zenera/rag",
-    "version": "1.1.34",
+    "version": "1.1.35",
     "description": "Retrieval over a corpus: openapi/swagger documents as a searchable graph, markdown and text as searchable passages.",
     "keywords": [
         "agents",
@@ -61,8 +61,8 @@
     "dependencies": {
         "@apidevtools/swagger-parser": "^12.0.0",
         "@lancedb/lancedb": "^0.38.0",
-        "@zenera/cli": "^1.1.34",
-        "@zenera/neo": "^1.1.34",
+        "@zenera/cli": "^1.1.35",
+        "@zenera/neo": "^1.1.35",
         "graphology": "^0.26.0",
         "remark-frontmatter": "^5.0.0",
         "remark-gfm": "^4.0.1",