agenr 0.7.18 → 0.7.19

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [0.7.19] - 2026-02-21
+
+### Fixed
+- fix(ingest): WriteQueue backpressure deadlock when processing large session files; raised default highWatermark from 500 to 2000 and added configurable backpressure timeout (default 120s) that surfaces a clear error instead of hanging forever (#125)
+
+### Added
+- feat(ingest): --queue-high-watermark and --queue-backpressure-timeout-ms CLI flags for tuning write queue behavior on large ingest jobs
+- feat(ingest): verbose mode now logs "[X/N] file -- starting" before extraction begins, eliminating the silent gap during large-file processing (#124)
+
 ## [0.7.18] - 2026-02-21
 
 ### Fixed

package/dist/cli-main.js

@@ -12448,6 +12448,7 @@ var WriteQueue = class {
   dbPath;
   batchSize;
   highWatermark;
+  backpressureTimeoutMs;
   retryOnFailure;
   isShutdownRequested;
   queue = [];
@@ -12467,7 +12468,8 @@ var WriteQueue = class {
     this.llmClient = options.llmClient;
     this.dbPath = options.dbPath;
     this.batchSize = Math.max(1, Math.floor(options.batchSize ?? 40));
-    this.highWatermark = Math.max(1, Math.floor(options.highWatermark ?? 500));
+    this.highWatermark = Math.max(1, Math.floor(options.highWatermark ?? 2e3));
+    this.backpressureTimeoutMs = Math.max(1, options.backpressureTimeoutMs ?? 12e4);
     this.retryOnFailure = options.retryOnFailure !== false;
     this.isShutdownRequested = options.isShutdownRequested;
     void this.runWriterLoop();
@@ -12479,10 +12481,16 @@ var WriteQueue = class {
     if (this.destroyed) {
       throw new ShutdownError("WriteQueue has been destroyed and cannot accept new items.");
     }
+    const backpressureStart = Date.now();
     while (this.pendingEntries > 0 && this.pendingEntries + entries.length > this.highWatermark) {
       if (this.destroyed) {
         throw new ShutdownError("WriteQueue has been destroyed and cannot accept new items.");
       }
+      if (Date.now() - backpressureStart > this.backpressureTimeoutMs) {
+        throw new Error(
+          `WriteQueue backpressure timeout after ${this.backpressureTimeoutMs}ms: ${this.pendingEntries} pending entries, limit=${this.highWatermark} (--queue-high-watermark). The writer may be stalled. Try reducing --workers or --concurrency, increase --queue-high-watermark, or increase --queue-backpressure-timeout-ms if the writer is just slow.`
+        );
+      }
       await sleep(50);
     }
     if (entries.length === 0) {
@@ -13162,6 +13170,12 @@ async function runIngestCommand(inputPaths, options, deps) {
   const globPattern = options.glob?.trim() || DEFAULT_GLOB;
   const llmConcurrency = parsePositiveInt3(options.concurrency, 5, "--concurrency");
   const requestedFileWorkers = parsePositiveInt3(options.workers, 10, "--workers");
+  const queueHighWatermark = parsePositiveInt3(options.queueHighWatermark, 2e3, "--queue-high-watermark");
+  const queueBackpressureTimeoutMs = parsePositiveInt3(
+    options.queueBackpressureTimeoutMs,
+    12e4,
+    "--queue-backpressure-timeout-ms"
+  );
   const retryEnabled = options.retry !== false;
   const maxRetries = retryEnabled ? parsePositiveInt3(options.maxRetries, 3, "--max-retries") : 0;
   const platformRaw = options.platform?.trim();
@@ -13268,7 +13282,8 @@ async function runIngestCommand(inputPaths, options, deps) {
       llmClient: client,
       dbPath,
       batchSize: 40,
-      highWatermark: 500,
+      highWatermark: queueHighWatermark,
+      backpressureTimeoutMs: queueBackpressureTimeoutMs,
       retryOnFailure: retryEnabled,
       isShutdownRequested: resolvedDeps.shouldShutdownFn
     });
@@ -13558,11 +13573,14 @@ async function runIngestCommand(inputPaths, options, deps) {
             if (!target) {
               return;
             }
+            const label = `[${target.index + 1}/${sortedTargets.length}] ${path26.basename(target.file)} (${formatBytes(target.size)})`;
+            if (verbose) {
+              clack4.log.step(`${label} -- starting`, clackOutput);
+            }
             const result = await processTarget(target);
             results[target.index] = result;
             completed += 1;
             updateProgress(completed, total, passVerb);
-            const label = `[${target.index + 1}/${sortedTargets.length}] ${path26.basename(target.file)} (${formatBytes(target.size)})`;
             if (verbose) {
               if (result.error) {
                 clack4.log.warn(`${label} -- ${formatError(result.error)}`, clackOutput);
@@ -18105,6 +18123,22 @@ function createProgram() {
     "Number of files to process in parallel (default: 10). Each worker uses --concurrency chunk parallelism. Total concurrent LLM calls = workers x concurrency. Reduce if hitting rate limits. Writes retry once per sub-batch unless --no-retry is set.",
     parseIntOption,
     10
+  ).option(
+    "--queue-high-watermark <n>",
+    "Max pending entries in write queue before workers apply backpressure (default: 2000)",
+    (v) => {
+      const n = Number.parseInt(v, 10);
+      if (!Number.isInteger(n) || n < 1) throw new Error("--queue-high-watermark must be a positive integer");
+      return n;
+    }
+  ).option(
+    "--queue-backpressure-timeout-ms <n>",
+    "Max milliseconds workers wait for write-queue backpressure to clear before failing with guidance (default: 120000ms = 2 min)",
+    (v) => {
+      const n = Number.parseInt(v, 10);
+      if (!Number.isInteger(n) || n < 1) throw new Error("--queue-backpressure-timeout-ms must be a positive integer");
+      return n;
+    }
   ).option("--skip-ingested", "Skip already-ingested files", true).option("--no-retry", "Disable auto-retry for failed files").option("--no-pre-fetch", "Disable elaborative encoding pre-fetch").option("--max-retries <n>", "Maximum auto-retry attempts", parseIntOption, 3).option("--force", "Clean re-ingest: delete previous rows for each file before processing", false).action(async (paths, opts) => {
     const result = await runIngestCommand(paths, {
       ...opts,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agenr",
-  "version": "0.7.18",
+  "version": "0.7.19",
   "openclaw": {
     "extensions": [
       "dist/openclaw-plugin/index.js"
@@ -11,13 +11,6 @@
   "bin": {
     "agenr": "dist/cli.js"
   },
-  "scripts": {
-    "build": "tsup src/cli.ts src/cli-main.ts src/openclaw-plugin/index.ts --format esm --dts",
-    "dev": "tsup src/cli.ts src/cli-main.ts --format esm --watch",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit"
-  },
   "dependencies": {
     "@clack/prompts": "^1.0.1",
     "@libsql/client": "^0.17.0",
@@ -61,9 +54,11 @@
     "README.md"
   ],
   "author": "agenr-ai",
-  "pnpm": {
-    "overrides": {
-      "fast-xml-parser": "^5.3.6"
-    }
+  "scripts": {
+    "build": "tsup src/cli.ts src/cli-main.ts src/openclaw-plugin/index.ts --format esm --dts",
+    "dev": "tsup src/cli.ts src/cli-main.ts --format esm --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
   }
-}
+}