agenr 0.7.16 → 0.7.17

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## [0.7.17] - 2026-02-21
+
+### Performance
+- perf(ingest): two-phase extract+write pipeline eliminates SQLite write-lock contention; extraction workers run in parallel while a single background writer drains entries in batched transactions (#107)
+- feat(ingest): add `--workers` flag (default 10) for file-level parallelism; previously hardcoded to 1
+- The write queue retries each write sub-batch once on transient failure (2s delay) before surfacing the error to the outer file-level retry loop. Use `--no-retry` to disable all retries including the inner write retry.
+
+### Changed
+- ingest: `entriesStored` now counts `added + superseded` (previously only `added`); superseded entries are written before the previous entry is marked superseded
+
 ## [0.7.16] - 2026-02-21
 
 ### Fixed

package/dist/cli-main.js

@@ -10202,7 +10202,7 @@ async function extractKnowledgeFromChunks(params) {
   );
   let dynamicDelay = baseDelay;
   let lastThrottleNoticeDelayMs = null;
-  const sleep3 = params.sleepImpl ?? sleepMs2;
+  const sleep4 = params.sleepImpl ?? sleepMs2;
   const llmConcurrency = Math.max(1, Math.trunc(params.llmConcurrency ?? 1));
   const bufferStreamDeltas = llmConcurrency > 1 && Boolean(params.onStreamDelta);
   let cursor = 0;
@@ -10276,7 +10276,7 @@ async function extractKnowledgeFromChunks(params) {
               warnings.push(
                 `Chunk ${chunk.chunk_index + 1}: attempt ${attempt} failed (${error instanceof Error ? error.message : String(error)}), retrying in ${backoffMs}ms.`
               );
-              await sleep3(backoffMs);
+              await sleep4(backoffMs);
               continue;
             }
             break;
@@ -10328,9 +10328,9 @@ async function extractKnowledgeFromChunks(params) {
         if (dynamicDelay > 0 && cursor < params.chunks.length && !isShutdownRequested()) {
           if (llmConcurrency > 1) {
             const jitterMs = Math.max(0, Math.trunc(dynamicDelay * (0.5 + Math.random())));
-            await sleep3(jitterMs);
+            await sleep4(jitterMs);
           } else if (currentIndex < params.chunks.length - 1) {
-            await sleep3(dynamicDelay);
+            await sleep4(dynamicDelay);
           }
         }
       }
@@ -12389,6 +12389,374 @@ async function parseTranscriptFile(filePath, options) {
   };
 }
 
+// src/ingest/write-queue.ts
+var CancelledError = class extends Error {
+  constructor(message = "Write queue item was cancelled.") {
+    super(message);
+    this.name = "CancelledError";
+  }
+};
+var ShutdownError = class extends Error {
+  constructor(message = "Write queue is shutting down.") {
+    super(message);
+    this.name = "ShutdownError";
+  }
+};
+var ZERO_RESULT = {
+  added: 0,
+  updated: 0,
+  skipped: 0,
+  superseded: 0,
+  llm_dedup_calls: 0
+};
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function mergeBatchResult(a, b) {
+  return {
+    added: a.added + b.added,
+    updated: a.updated + b.updated,
+    skipped: a.skipped + b.skipped,
+    superseded: a.superseded + b.superseded,
+    llm_dedup_calls: a.llm_dedup_calls + b.llm_dedup_calls
+  };
+}
+function chunkEntries2(entries, size) {
+  if (entries.length <= size) {
+    return [entries];
+  }
+  const chunks = [];
+  for (let i = 0; i < entries.length; i += size) {
+    chunks.push(entries.slice(i, i + size));
+  }
+  return chunks;
+}
+function toError(error) {
+  if (error instanceof Error) {
+    return error;
+  }
+  return new Error(String(error));
+}
+var WriteQueue = class {
+  db;
+  storeEntriesFn;
+  apiKey;
+  llmClient;
+  dbPath;
+  batchSize;
+  highWatermark;
+  isShutdownRequested;
+  queue = [];
+  pendingEntries = 0;
+  destroyed = false;
+  writerStopping = false;
+  writerStopped = false;
+  inflightWrites = 0;
+  activeWorkItems = 0;
+  activeByFileKey = /* @__PURE__ */ new Map();
+  fileWaiters = /* @__PURE__ */ new Map();
+  drainWaiters = [];
+  writerWaiters = [];
+  constructor(options) {
+    this.db = options.db;
+    this.storeEntriesFn = options.storeEntriesFn;
+    this.apiKey = options.apiKey;
+    this.llmClient = options.llmClient;
+    this.dbPath = options.dbPath;
+    this.batchSize = Math.max(1, Math.floor(options.batchSize ?? 40));
+    this.highWatermark = Math.max(1, Math.floor(options.highWatermark ?? 500));
+    this.isShutdownRequested = options.isShutdownRequested;
+    void this.runWriterLoop();
+  }
+  get pendingCount() {
+    return this.pendingEntries;
+  }
+  async push(entries, fileKey, fileHash) {
+    if (this.destroyed) {
+      throw new ShutdownError("WriteQueue has been destroyed and cannot accept new items.");
+    }
+    while (this.pendingEntries >= this.highWatermark) {
+      if (this.destroyed) {
+        throw new ShutdownError("WriteQueue has been destroyed and cannot accept new items.");
+      }
+      await sleep(50);
+    }
+    if (entries.length === 0) {
+      return { ...ZERO_RESULT };
+    }
+    return await new Promise((resolve, reject) => {
+      if (this.destroyed) {
+        reject(new ShutdownError("WriteQueue has been destroyed and cannot accept new items."));
+        return;
+      }
+      this.pendingEntries += entries.length;
+      this.queue.push({
+        kind: "write",
+        entries,
+        fileKey,
+        fileHash,
+        resolve,
+        reject
+      });
+      this.wakeWriter();
+    });
+  }
+  async cancel(fileKey) {
+    const kept = [];
+    const cancelled = [];
+    for (const item of this.queue) {
+      if (item.kind === "write" && item.fileKey === fileKey) {
+        cancelled.push(item);
+        this.pendingEntries = Math.max(0, this.pendingEntries - item.entries.length);
+      } else {
+        kept.push(item);
+      }
+    }
+    if (cancelled.length > 0) {
+      this.queue.splice(0, this.queue.length, ...kept);
+      for (const item of cancelled) {
+        item.reject(new CancelledError(`Cancelled queued write for ${fileKey}.`));
+      }
+      this.resolveDrainIfIdle();
+    }
+    await this.waitForFileIdle(fileKey);
+  }
+  async runExclusive(fn) {
+    if (this.destroyed) {
+      throw new ShutdownError("WriteQueue has been destroyed and cannot accept new items.");
+    }
+    return await new Promise((resolve, reject) => {
+      if (this.destroyed) {
+        reject(new ShutdownError("WriteQueue has been destroyed and cannot accept new items."));
+        return;
+      }
+      this.queue.push({
+        kind: "exclusive",
+        fn,
+        resolve,
+        reject
+      });
+      this.wakeWriter();
+    });
+  }
+  async drain() {
+    if (this.isIdle()) {
+      return;
+    }
+    await new Promise((resolve) => {
+      this.drainWaiters.push(resolve);
+    });
+  }
+  destroy() {
+    if (this.destroyed) {
+      return;
+    }
+    this.destroyed = true;
+    this.writerStopping = true;
+    const pending = this.queue.splice(0, this.queue.length);
+    this.pendingEntries = 0;
+    for (const item of pending) {
+      item.reject(new ShutdownError("WriteQueue destroyed before item was processed."));
+    }
+    this.wakeWriter();
+    this.resolveDrainIfIdle();
+  }
+  async runWriterLoop() {
+    while (true) {
+      if (this.isShutdownRequested?.() && this.hasPendingWriteItems()) {
+        this.shutdownFromSignal();
+      }
+      if ((this.writerStopping || this.destroyed) && this.queue.length === 0) {
+        break;
+      }
+      if (this.queue.length === 0) {
+        await this.waitForWork();
+        continue;
+      }
+      const batch = this.queue.splice(0, this.queue.length);
+      this.markBatchAsDispatched(batch);
+      await this.processBatch(batch);
+    }
+    this.writerStopped = true;
+    this.resolveDrainIfIdle();
+  }
+  hasPendingWriteItems() {
+    return this.queue.some((item) => item.kind === "write");
+  }
+  async processBatch(batch) {
+    let index = 0;
+    while (index < batch.length) {
+      const item = batch[index];
+      if (!item) {
+        index += 1;
+        continue;
+      }
+      if (item.kind === "exclusive") {
+        await this.processExclusive(item);
+        index += 1;
+        continue;
+      }
+      const segment = [];
+      while (index < batch.length) {
+        const segmentItem = batch[index];
+        if (!segmentItem || segmentItem.kind !== "write") {
+          break;
+        }
+        segment.push(segmentItem);
+        index += 1;
+      }
+      await this.processWriteSegment(segment);
+    }
+  }
+  async processExclusive(item) {
+    try {
+      const result = await item.fn();
+      item.resolve(result);
+    } catch (error) {
+      item.reject(toError(error));
+    } finally {
+      this.activeWorkItems = Math.max(0, this.activeWorkItems - 1);
+      this.resolveDrainIfIdle();
+    }
+  }
+  async processWriteSegment(segment) {
+    const grouped = /* @__PURE__ */ new Map();
+    for (const item of segment) {
+      const group = grouped.get(item.fileKey);
+      if (group) {
+        group.push(item);
+      } else {
+        grouped.set(item.fileKey, [item]);
+      }
+    }
+    for (const items of grouped.values()) {
+      for (const item of items) {
+        await this.processWriteItem(item);
+      }
+    }
+  }
+  async processWriteItem(item) {
+    let result = { ...ZERO_RESULT };
+    const subBatches = chunkEntries2(item.entries, this.batchSize);
+    try {
+      for (const subBatch of subBatches) {
+        const writeResult = await this.writeSubBatchWithRetry(subBatch, item.fileKey, item.fileHash);
+        result = mergeBatchResult(result, writeResult);
+      }
+      item.resolve(result);
+    } catch (error) {
+      item.reject(toError(error));
+    } finally {
+      this.activeWorkItems = Math.max(0, this.activeWorkItems - 1);
+      this.decrementActiveFile(item.fileKey);
+      this.resolveDrainIfIdle();
+    }
+  }
+  async writeSubBatchWithRetry(entries, fileKey, fileHash) {
+    let lastError = null;
+    for (let attempt = 1; attempt <= 2; attempt += 1) {
+      this.inflightWrites += 1;
+      try {
+        const result = await this.storeEntriesFn(this.db, entries, this.apiKey, {
+          sourceFile: fileKey,
+          ingestContentHash: fileHash,
+          skipIngestLog: true,
+          onlineDedup: true,
+          skipLlmDedup: false,
+          llmClient: this.llmClient,
+          dbPath: this.dbPath
+        });
+        return {
+          added: result.added,
+          updated: result.updated,
+          skipped: result.skipped,
+          superseded: result.superseded,
+          llm_dedup_calls: result.llm_dedup_calls
+        };
+      } catch (error) {
+        lastError = toError(error);
+        if (attempt < 2) {
+          await sleep(2e3);
+        }
+      } finally {
+        this.inflightWrites = Math.max(0, this.inflightWrites - 1);
+        this.resolveDrainIfIdle();
+      }
+    }
+    throw lastError ?? new Error("Write queue failed to store sub-batch.");
+  }
+  markBatchAsDispatched(batch) {
+    for (const item of batch) {
+      this.activeWorkItems += 1;
+      if (item.kind === "write") {
+        this.pendingEntries = Math.max(0, this.pendingEntries - item.entries.length);
+        this.activeByFileKey.set(item.fileKey, (this.activeByFileKey.get(item.fileKey) ?? 0) + 1);
+      }
+    }
+    this.resolveDrainIfIdle();
+  }
+  decrementActiveFile(fileKey) {
+    const current = this.activeByFileKey.get(fileKey) ?? 0;
+    if (current <= 1) {
+      this.activeByFileKey.delete(fileKey);
+      const waiters = this.fileWaiters.get(fileKey) ?? [];
+      this.fileWaiters.delete(fileKey);
+      for (const waiter of waiters) {
+        waiter();
+      }
+      return;
+    }
+    this.activeByFileKey.set(fileKey, current - 1);
+  }
+  async waitForFileIdle(fileKey) {
+    if ((this.activeByFileKey.get(fileKey) ?? 0) === 0) {
+      return;
+    }
+    await new Promise((resolve) => {
+      const existing = this.fileWaiters.get(fileKey);
+      if (existing) {
+        existing.push(resolve);
+      } else {
+        this.fileWaiters.set(fileKey, [resolve]);
+      }
+    });
+  }
+  waitForWork() {
+    return new Promise((resolve) => {
+      this.writerWaiters.push(resolve);
+    });
+  }
+  wakeWriter() {
+    while (this.writerWaiters.length > 0) {
+      const waiter = this.writerWaiters.shift();
+      waiter?.();
+    }
+  }
+  shutdownFromSignal() {
+    this.destroyed = true;
+    this.writerStopping = true;
+    const pending = this.queue.splice(0, this.queue.length);
+    this.pendingEntries = 0;
+    for (const item of pending) {
+      item.reject(new ShutdownError("Shutdown requested. Dropping queued writes."));
+    }
+    this.wakeWriter();
+    this.resolveDrainIfIdle();
+  }
+  isIdle() {
+    return this.pendingEntries === 0 && this.activeWorkItems === 0 && this.inflightWrites === 0;
+  }
+  resolveDrainIfIdle() {
+    if (!this.isIdle()) {
+      return;
+    }
+    while (this.drainWaiters.length > 0) {
+      const waiter = this.drainWaiters.shift();
+      waiter?.();
+    }
+  }
+};
+
 // src/watch/pid.ts
 import fs26 from "fs/promises";
 import path25 from "path";
@@ -12565,7 +12933,7 @@ function parsePositiveInt3(value, fallback, label) {
   }
   return Math.floor(parsed);
 }
-async function sleep(ms) {
+async function sleep2(ms) {
   await new Promise((resolve) => setTimeout(resolve, ms));
 }
 function retryBackoffMs(attempt) {
@@ -12744,8 +13112,9 @@ async function runIngestCommand(inputPaths, options, deps) {
     readWatcherPidFn: deps?.readWatcherPidFn ?? readWatcherPid,
     resolveWatcherPidPathFn: deps?.resolveWatcherPidPathFn ?? resolveWatcherPidPath,
     nowFn: deps?.nowFn ?? (() => /* @__PURE__ */ new Date()),
-    sleepFn: deps?.sleepFn ?? sleep,
-    shouldShutdownFn: deps?.shouldShutdownFn ?? isShutdownRequested
+    sleepFn: deps?.sleepFn ?? sleep2,
+    shouldShutdownFn: deps?.shouldShutdownFn ?? isShutdownRequested,
+    createWriteQueueFn: deps?.createWriteQueueFn ?? ((opts) => new WriteQueue(opts))
   };
   const clackOutput = { output: process.stderr };
   clack4.intro(banner(), clackOutput);
@@ -12788,6 +13157,7 @@ async function runIngestCommand(inputPaths, options, deps) {
   const skipIngested = force ? false : options.skipIngested !== false;
   const globPattern = options.glob?.trim() || DEFAULT_GLOB;
   const llmConcurrency = parsePositiveInt3(options.concurrency, 5, "--concurrency");
+  const requestedFileWorkers = parsePositiveInt3(options.workers, 10, "--workers");
   const retryEnabled = options.retry !== false;
   const maxRetries = retryEnabled ? parsePositiveInt3(options.maxRetries, 3, "--max-retries") : 0;
   const platformRaw = options.platform?.trim();
@@ -12818,8 +13188,9 @@ async function runIngestCommand(inputPaths, options, deps) {
     })
   );
   const sortedTargets = targetsWithSizes.sort((a, b) => a.size - b.size || a.file.localeCompare(b.file)).map((item, index) => ({ ...item, index }));
+  const fileWorkerCount = Math.min(requestedFileWorkers, Math.max(1, sortedTargets.length));
   clack4.log.info(
-    `Ingesting: ${ui.bold(String(sortedTargets.length))} file(s) | Glob: ${globPattern} | Chunk concurrency: ${ui.bold(String(llmConcurrency))} | Skip ingested: ${skipIngested ? "yes" : "no"}`,
+    `Ingesting: ${ui.bold(String(sortedTargets.length))} file(s) | Glob: ${globPattern} | File workers: ${ui.bold(String(fileWorkerCount))} | Chunk concurrency: ${ui.bold(String(llmConcurrency))} | Skip ingested: ${skipIngested ? "yes" : "no"}`,
     clackOutput
   );
   if (sortedTargets.length === 0) {
@@ -12865,6 +13236,41 @@ async function runIngestCommand(inputPaths, options, deps) {
   }
   const db = resolvedDeps.getDbFn(dbPath);
   await resolvedDeps.initDbFn(db);
+  const cleanupDbResources = () => {
+    if (shouldLockDb) {
+      releaseDbLock();
+    }
+    resolvedDeps.closeDbFn(db);
+  };
+  let embeddingApiKey = null;
+  try {
+    if (!embeddingApiKey && !options.noPreFetch) {
+      embeddingApiKey = resolvedDeps.resolveEmbeddingApiKeyFn(config, process.env);
+    }
+  } catch (error) {
+    cleanupDbResources();
+    throw error;
+  }
+  if ((!embeddingApiKey || embeddingApiKey.trim().length === 0) && !dryRun) {
+    cleanupDbResources();
+    throw new Error("Embedding API key is required for ingest. Run 'agenr setup' to configure.");
+  }
+  let queue;
+  try {
+    queue = resolvedDeps.createWriteQueueFn({
+      db,
+      storeEntriesFn: resolvedDeps.storeEntriesFn,
+      apiKey: embeddingApiKey ?? "",
+      llmClient: client,
+      dbPath,
+      batchSize: 40,
+      highWatermark: 500,
+      isShutdownRequested: resolvedDeps.shouldShutdownFn
+    });
+  } catch (error) {
+    cleanupDbResources();
+    throw error;
+  }
   const results = new Array(sortedTargets.length);
   let totalEntriesExtracted = 0;
   let totalEntriesStored = 0;
@@ -12878,22 +13284,6 @@ async function runIngestCommand(inputPaths, options, deps) {
   let forceDeletedEntryRows = 0;
   let forceDeletedEntrySourceRows = 0;
   let completed = 0;
-  let embeddingApiKey = null;
-  if (!options.noPreFetch) {
-    try {
-      embeddingApiKey = resolvedDeps.resolveEmbeddingApiKeyFn(config, process.env);
-    } catch (error) {
-      embeddingApiKey = null;
-      if (verbose) {
-        clack4.log.warn(
-          formatWarn(
-            `Pre-fetch disabled - embedding API key not available: ${error instanceof Error ? error.message : String(error)}`
-          ),
-          clackOutput
-        );
-      }
-    }
-  }
   let watchStateLoaded = false;
   let watchState = createEmptyWatchState();
   let cursor = 0;
@@ -12901,26 +13291,11 @@ async function runIngestCommand(inputPaths, options, deps) {
   let filesWithChunkFailures = 0;
   const chunkStatsByFile = /* @__PURE__ */ new Map();
   let firstPassFailedIndexSet = /* @__PURE__ */ new Set();
-  let dbChain = Promise.resolve();
-  const withDbLock = async (fn) => {
-    const previous = dbChain;
-    let release;
-    dbChain = new Promise((resolve) => {
-      release = resolve;
-    });
-    await previous;
-    try {
-      return await fn();
-    } finally {
-      release();
-    }
-  };
   const updateProgress = (completedCount, totalCount, verb) => {
     if (verbose) {
       return;
     }
-    const suffix = llmConcurrency > 1 ? ` (${llmConcurrency} chunks active)...` : "...";
-    process.stderr.write(`\r${ui.dim(`${verb} ${completedCount}/${totalCount}${suffix}`)}`);
+    process.stderr.write(`\r${ui.dim(`${verb} ${completedCount}/${totalCount} (queue: ${queue.pendingCount})...`)}`);
   };
   const clearProgressLine = () => {
     if (verbose) {
@@ -12988,8 +13363,9 @@ async function runIngestCommand(inputPaths, options, deps) {
       const rawContent = await fs27.readFile(target.file, "utf8");
       const ingestByteOffset = Buffer.byteLength(rawContent, "utf8");
       fileHash = resolvedDeps.hashTextFn(rawContent);
+      const chunkTickets = [];
       if (skipIngested && !force) {
-        const alreadyIngested = await withDbLock(() => isAlreadyIngested(db, target.file, fileHash));
+        const alreadyIngested = await isAlreadyIngested(db, target.file, fileHash);
         if (alreadyIngested) {
           fileResult.skipped = true;
           fileResult.skipReason = "already ingested";
@@ -12997,12 +13373,12 @@ async function runIngestCommand(inputPaths, options, deps) {
         }
       }
       if (force) {
-        const cleanupStats = await withDbLock(() => cleanupForForceReingest(db, target.file, dryRun));
+        const cleanupStats = await queue.runExclusive(() => cleanupForForceReingest(db, target.file, dryRun));
         forceDeletedIngestLogRows += cleanupStats.ingestLogRows;
         forceDeletedEntryRows += cleanupStats.entryRows;
         forceDeletedEntrySourceRows += cleanupStats.entrySourceRows;
       } else {
-        baselineEntryIds = await withDbLock(() => getSourceEntryIds(db, target.file));
+        baselineEntryIds = await getSourceEntryIds(db, target.file);
       }
       const parsed = await resolvedDeps.parseTranscriptFileFn(target.file, { raw: options.raw === true, verbose });
       if (verbose && parsed.warnings.length > 0) {
@@ -13014,8 +13390,8 @@ async function runIngestCommand(inputPaths, options, deps) {
           }
         }
       }
-      const processChunkEntries = async (chunkEntries2) => {
-        const normalizedEntries = chunkEntries2.map((entry) => ({
+      const processChunkEntries = async (chunkEntries3) => {
+        const normalizedEntries = chunkEntries3.map((entry) => ({
           ...entry,
           ...platform ? { platform } : {},
           ...project ? { project } : {},
@@ -13026,41 +13402,32 @@ async function runIngestCommand(inputPaths, options, deps) {
         }));
         fileResult.entriesExtracted += normalizedEntries.length;
         totalEntriesExtracted += normalizedEntries.length;
-        const deduped = resolvedDeps.deduplicateEntriesFn(normalizedEntries);
-        if (dryRun || deduped.length === 0) {
+        if (dryRun || normalizedEntries.length === 0) {
+          chunkTickets.push(
+            Promise.resolve({
+              added: 0,
+              updated: 0,
+              skipped: 0,
+              superseded: 0,
+              llm_dedup_calls: 0
+            })
+          );
           return;
         }
-        if (!embeddingApiKey) {
-          embeddingApiKey = resolvedDeps.resolveEmbeddingApiKeyFn(config, process.env);
+        const deduped = resolvedDeps.deduplicateEntriesFn(normalizedEntries);
+        if (deduped.length === 0) {
+          chunkTickets.push(
+            Promise.resolve({
+              added: 0,
+              updated: 0,
+              skipped: 0,
+              superseded: 0,
+              llm_dedup_calls: 0
+            })
+          );
+          return;
         }
-        const storeResult = await withDbLock(
-          () => resolvedDeps.storeEntriesFn(db, deduped, embeddingApiKey ?? "", {
-            sourceFile: target.file,
-            ingestContentHash: fileHash,
-            skipIngestLog: true,
-            onlineDedup: true,
-            skipLlmDedup: false,
-            llmClient: client,
-            dbPath
-          })
-        );
-        const reinforced = storeResult.updated;
-        const stored = storeResult.added + storeResult.superseded;
-        fileResult.entriesStored += stored;
-        fileResult.entriesSkippedDuplicate += storeResult.skipped;
-        fileResult.entriesReinforced += reinforced;
-        totalEntriesStored += stored;
-        totalEntriesAdded += storeResult.added;
-        totalEntriesUpdated += 0;
-        totalEntriesSkipped += storeResult.skipped;
-        totalEntriesReinforced += reinforced;
-        totalEntriesSuperseded += storeResult.superseded;
-        totalDedupLlmCalls += storeResult.llm_dedup_calls;
-        fileStoreStats.added += storeResult.added;
-        fileStoreStats.updated += storeResult.updated;
-        fileStoreStats.skipped += storeResult.skipped;
-        fileStoreStats.superseded += storeResult.superseded;
-        fileStoreStats.llmDedupCalls += storeResult.llm_dedup_calls;
+        chunkTickets.push(queue.push(deduped, target.file, fileHash));
       };
       const extracted = await resolvedDeps.extractKnowledgeFromChunksFn({
         file: target.file,
@@ -13093,10 +13460,49 @@ async function runIngestCommand(inputPaths, options, deps) {
           `All chunks failed during extraction (${chunkLabel}). This is often caused by API rate limits or timeouts; check provider limits/logs and re-run ingest.`
         );
       }
-      await withDbLock(() => syncWatchStateOffset(target.file, ingestByteOffset));
+      const writeResults = await Promise.allSettled(chunkTickets);
+      const writeErrors = [];
+      let cancelledTickets = 0;
+      for (const writeResult of writeResults) {
+        if (writeResult.status === "fulfilled") {
+          const result = writeResult.value;
+          const stored = result.added + result.superseded;
+          fileResult.entriesStored += stored;
+          fileResult.entriesSkippedDuplicate += result.skipped;
+          fileResult.entriesReinforced += result.updated;
+          totalEntriesStored += stored;
+          totalEntriesAdded += result.added;
+          totalEntriesSuperseded += result.superseded;
+          totalEntriesSkipped += result.skipped;
+          totalEntriesReinforced += result.updated;
+          totalDedupLlmCalls += result.llm_dedup_calls;
+          fileStoreStats.added += result.added;
+          fileStoreStats.updated += result.updated;
+          fileStoreStats.skipped += result.skipped;
+          fileStoreStats.superseded += result.superseded;
+          fileStoreStats.llmDedupCalls += result.llm_dedup_calls;
+          continue;
+        }
+        const reason = writeResult.reason;
+        if (reason instanceof CancelledError) {
+          cancelledTickets += 1;
+          continue;
+        }
+        writeErrors.push(errorMessage2(reason));
+      }
+      if (writeErrors.length > 0) {
+        throw new Error(writeErrors.join(" | "));
+      }
+      if (cancelledTickets > 0 && verbose) {
+        clack4.log.warn(
+          formatWarn(`Cancelled ${cancelledTickets} pending write chunk(s) for ${path26.basename(target.file)}.`),
+          clackOutput
+        );
+      }
+      await syncWatchStateOffset(target.file, ingestByteOffset);
       if (!dryRun) {
         const fileDurationMs = Math.max(0, resolvedDeps.nowFn().getTime() - fileStartedAt.getTime());
-        await withDbLock(
+        await queue.runExclusive(
           () => insertIngestLogForFile(db, {
             filePath: target.file,
             contentHash: fileHash,
@@ -13109,7 +13515,8 @@ async function runIngestCommand(inputPaths, options, deps) {
     } catch (error) {
       if (fileHash.length > 0) {
         try {
-          await withDbLock(() => cleanupFailedFileIngest(db, target.file, fileHash, baselineEntryIds, dryRun));
+          await queue.cancel(target.file);
+          await queue.runExclusive(() => cleanupFailedFileIngest(db, target.file, fileHash, baselineEntryIds, dryRun));
         } catch (cleanupError) {
           fileResult.error = `${errorMessage2(error)} | cleanup failed: ${errorMessage2(cleanupError)}`;
           return fileResult;
@@ -13129,7 +13536,7 @@ async function runIngestCommand(inputPaths, options, deps) {
       cursor = 0;
       completed = 0;
       const total = targets.length;
-      const workerCount = 1;
+      const workerCount = Math.min(fileWorkerCount, total);
       await Promise.all(
         Array.from({ length: workerCount }, async () => {
           while (true) {
@@ -13247,6 +13654,17 @@ async function runIngestCommand(inputPaths, options, deps) {
     }
   } finally {
     clearProgressLine();
+    try {
+      await queue.drain();
+    } catch (error) {
+      if (error instanceof ShutdownError) {
+        clack4.log.warn(formatWarn(`Write queue shutdown before full drain: ${error.message}`), clackOutput);
+      } else {
+        clack4.log.warn(formatWarn(`Write queue drain failed: ${errorMessage2(error)}`), clackOutput);
+      }
+    } finally {
+      queue.destroy();
+    }
     if (!dryRun) {
       try {
         await walCheckpoint(db);
@@ -15844,7 +16262,7 @@ var DEFAULT_WAL_CHECKPOINT_INTERVAL_MS = 3e4;
 function isFileNotFound(error) {
   return error.code === "ENOENT";
 }
-async function sleep2(ms) {
+async function sleep3(ms) {
   await new Promise((resolve) => setTimeout(resolve, ms));
 }
 function formatError2(error) {
@@ -15931,7 +16349,7 @@ async function runWatcher(options, deps) {
     rmFn: deps?.rmFn ?? fs33.rm,
     watchFn: deps?.watchFn ?? watchFs,
     nowFn: deps?.nowFn ?? (() => /* @__PURE__ */ new Date()),
-    sleepFn: deps?.sleepFn ?? sleep2,
+    sleepFn: deps?.sleepFn ?? sleep3,
     shouldShutdownFn: deps?.shouldShutdownFn ?? isShutdownRequested
   };
   const directoryMode = options.directoryMode === true;
@@ -16192,13 +16610,13 @@ async function runWatcher(options, deps) {
             options.onWarn?.(warning);
           }
         }
-        const processChunkEntries = async (chunkEntries2) => {
+        const processChunkEntries = async (chunkEntries3) => {
           const platformTag = currentPlatform && currentPlatform !== "mtime" ? normalizeKnowledgePlatform(currentPlatform) ?? void 0 : void 0;
-          const taggedEntries = platformTag || cachedProject ? chunkEntries2.map((entry) => ({
+          const taggedEntries = platformTag || cachedProject ? chunkEntries3.map((entry) => ({
             ...entry,
             ...platformTag ? { platform: platformTag } : {},
             ...cachedProject ? { project: cachedProject } : {}
-          })) : chunkEntries2;
+          })) : chunkEntries3;
           cycleResult.entriesExtracted += taggedEntries.length;
           const deduped = resolvedDeps.deduplicateEntriesFn(taggedEntries);
           if (options.dryRun || deduped.length === 0) {
@@ -17677,7 +18095,12 @@ function createProgram() {
     const result = await runTodoCommand(subcommand, subject, { db: opts.db });
     process.exitCode = result.exitCode;
   });
-  program.command("ingest").description("Bulk-ingest knowledge from files and directories").argument("<paths...>", "Files or directories to process").option("--glob <pattern>", "File filter glob", "**/*.{jsonl,md,txt}").option("--db <path>", "Database path override").option("--model <model>", "LLM model to use").option("--provider <name>", "LLM provider: anthropic, openai, openai-codex").option("--platform <name>", "Platform tag: openclaw, claude-code, codex").option("--project <name>", "Project tag (lowercase).", (val, prev) => [...prev, val], []).option("--verbose", "Show per-file details", false).option("--raw", "Bypass adapter filtering (pass transcripts through unmodified)", false).option("--dry-run", "Extract without storing", false).option("--json", "Output JSON results", false).option("--concurrency <n>", "Parallel chunk extractions", parseIntOption, 5).option("--skip-ingested", "Skip already-ingested files", true).option("--no-retry", "Disable auto-retry for failed files").option("--no-pre-fetch", "Disable elaborative encoding pre-fetch").option("--max-retries <n>", "Maximum auto-retry attempts", parseIntOption, 3).option("--force", "Clean re-ingest: delete previous rows for each file before processing", false).action(async (paths, opts) => {
+  program.command("ingest").description("Bulk-ingest knowledge from files and directories").argument("<paths...>", "Files or directories to process").option("--glob <pattern>", "File filter glob", "**/*.{jsonl,md,txt}").option("--db <path>", "Database path override").option("--model <model>", "LLM model to use").option("--provider <name>", "LLM provider: anthropic, openai, openai-codex").option("--platform <name>", "Platform tag: openclaw, claude-code, codex").option("--project <name>", "Project tag (lowercase).", (val, prev) => [...prev, val], []).option("--verbose", "Show per-file details", false).option("--raw", "Bypass adapter filtering (pass transcripts through unmodified)", false).option("--dry-run", "Extract without storing", false).option("--json", "Output JSON results", false).option("--concurrency <n>", "Parallel chunk extractions", parseIntOption, 5).option(
+    "--workers <n>",
+    "Number of files to process in parallel (default: 10). Each worker uses --concurrency chunk parallelism. Total concurrent LLM calls = workers x concurrency. Reduce if hitting rate limits.",
+    parseIntOption,
+    10
+  ).option("--skip-ingested", "Skip already-ingested files", true).option("--no-retry", "Disable auto-retry for failed files").option("--no-pre-fetch", "Disable elaborative encoding pre-fetch").option("--max-retries <n>", "Maximum auto-retry attempts", parseIntOption, 3).option("--force", "Clean re-ingest: delete previous rows for each file before processing", false).action(async (paths, opts) => {
     const result = await runIngestCommand(paths, {
       ...opts,
       noPreFetch: opts.noPreFetch === true

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agenr",
-  "version": "0.7.16",
+  "version": "0.7.17",
   "openclaw": {
     "extensions": [
       "dist/openclaw-plugin/index.js"
@@ -11,6 +11,13 @@
   "bin": {
     "agenr": "dist/cli.js"
   },
+  "scripts": {
+    "build": "tsup src/cli.ts src/cli-main.ts src/openclaw-plugin/index.ts --format esm --dts",
+    "dev": "tsup src/cli.ts src/cli-main.ts --format esm --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
+  },
   "dependencies": {
     "@clack/prompts": "^1.0.1",
     "@libsql/client": "^0.17.0",
@@ -54,11 +61,9 @@
     "README.md"
   ],
   "author": "agenr-ai",
-  "scripts": {
-    "build": "tsup src/cli.ts src/cli-main.ts src/openclaw-plugin/index.ts --format esm --dts",
-    "dev": "tsup src/cli.ts src/cli-main.ts --format esm --watch",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit"
+  "pnpm": {
+    "overrides": {
+      "fast-xml-parser": "^5.3.6"
+    }
   }
-}
+}