@lincoln504/pi-research 1.3.1 → 1.3.2

package/src/infrastructure/knowledge-store-service.ts

@@ -53,7 +53,23 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
   // Initialization promise to prevent concurrent initialization
   private _initializationPromise: Promise<void> | null = null;
 
+  // Disposal promise: lets a concurrent initialize() wait out an in-flight
+  // dispose() instead of racing it. See dispose()'s own comment for the
+  // failure this closes — none of initialize()'s lifecycle guards below
+  // recognize DISPOSING, so without this an initialize() arriving while
+  // dispose() is mid-teardown would start rebuilding immediately, and the
+  // two would race to close/null each other's components.
+  private _disposalPromise: Promise<void> | null = null;
+
   async initialize(ctx?: any): Promise<void> {
+    // A dispose() in flight must settle before anything below inspects
+    // lifecycle or decides what to do — while() (not if()) because a newer
+    // disposal can start again during the await (mirrors the
+    // _initializationPromise wait loop further down).
+    while (this._disposalPromise) {
+      await this._disposalPromise;
+    }
+
     // Only an EXPLICIT ctx.cwd may re-scope the store. The lazy callers
     // (getStore/getEmbedder/getWriterQueue) call initialize() with no ctx — they
     // must NOT flip the scope. Resolving the missing cwd to process.cwd() here was
@@ -277,6 +293,13 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
       return;
     }
 
+    // Join an already-in-flight disposal rather than starting a second one —
+    // two concurrent dispose() calls would otherwise both try to close/null
+    // the same handles.
+    if (this._disposalPromise) {
+      return this._disposalPromise;
+    }
+
     // If an initialization is still in flight, let it finish before we tear down.
     // Otherwise the init closure would re-assign this._embedder/_store/_writerQueue
     // and flip lifecycle back to INITIALIZED *after* dispose() nulled everything —
@@ -293,42 +316,65 @@ export class KnowledgeStoreService implements IKnowledgeStoreService {
     this.lifecycle = ServiceLifecycle.DISPOSING;
     logger.debug('[KnowledgeStoreService] Disposing...');
 
-    try {
-      if (this._writerQueue) {
-        await this._writerQueue.dispose?.();
-      }
-      
-      if (this._store) {
-        await this._store.close();
-      }
+    // Capture the components to tear down NOW, as local references — not
+    // re-read via this._embedder/this._store/etc. at each step below. A
+    // concurrent initialize() is blocked on _disposalPromise (see above) and
+    // cannot publish anything while we run, but capturing locally means this
+    // disposal only ever acts on what it was actually invoked to dispose,
+    // regardless of what the instance fields point at by the time each
+    // `await` below resumes — the same identity-capture discipline the
+    // embedding-leader CAS and file-lock PID+startTime checks elsewhere in
+    // this codebase already use for exactly this class of race.
+    const embedder = this._embedder;
+    const store = this._store;
+    const writerQueue = this._writerQueue;
+    const initLock = this._initLock;
+
+    this._disposalPromise = (async () => {
+      try {
+        if (writerQueue) {
+          await writerQueue.dispose?.();
+        }
 
-      if (this._embedder) {
-        await this._embedder.dispose?.();
-        // The embedding factory's module-level cache still points at the instance
-        // just disposed — its fast path has no liveness check, so a later re-init
-        // (cwd/mode re-scope) would be handed the dead instance and burn every
-        // warm-up/init retry on it. clearEmbeddingInstance() re-disposes
-        // (idempotent) and nulls the cache; when this process was the leader, its
-        // shutdown deregisters via the serverId CAS, and a client instance's
-        // dispose is a no-op — a FOREIGN leader's registration is never touched.
-        await clearEmbeddingInstance();
-      }
+        if (store) {
+          await store.close();
+        }
 
-      if (this._initLock) {
-        await this._initLock.dispose();
-      }
+        if (embedder) {
+          await embedder.dispose?.();
+          // The embedding factory's module-level cache still points at the instance
+          // just disposed — its fast path has no liveness check, so a later re-init
+          // (cwd/mode re-scope) would be handed the dead instance and burn every
+          // warm-up/init retry on it. clearEmbeddingInstance() re-disposes
+          // (idempotent) and nulls the cache; when this process was the leader, its
+          // shutdown deregisters via the serverId CAS, and a client instance's
+          // dispose is a no-op — a FOREIGN leader's registration is never touched.
+          await clearEmbeddingInstance();
+        }
 
-      this._embedder = null;
-      this._store = null;
-      this._writerQueue = null;
-      this._initLock = null;
+        if (initLock) {
+          await initLock.dispose();
+        }
 
-      logger.debug('[KnowledgeStoreService] Disposed');
-    } catch (err) {
-      logger.error('[KnowledgeStoreService] Error during disposal:', err);
-    } finally {
-      this.lifecycle = ServiceLifecycle.DISPOSED;
-    }
+        // Only clear fields that still point at what we just disposed — a
+        // concurrent initialize() cannot have published anything while
+        // blocked on _disposalPromise, but this keeps the intent explicit
+        // rather than relying on that invariant holding forever.
+        if (this._embedder === embedder) this._embedder = null;
+        if (this._store === store) this._store = null;
+        if (this._writerQueue === writerQueue) this._writerQueue = null;
+        if (this._initLock === initLock) this._initLock = null;
+
+        logger.debug('[KnowledgeStoreService] Disposed');
+      } catch (err) {
+        logger.error('[KnowledgeStoreService] Error during disposal:', err);
+      } finally {
+        this.lifecycle = ServiceLifecycle.DISPOSED;
+        this._disposalPromise = null;
+      }
+    })();
+
+    return this._disposalPromise;
   }
 
   /**

package/src/infrastructure/state/state-browser-api.ts

@@ -25,28 +25,6 @@ export class StateBrowserApi {
     return this.browserManager.getBrowserServer(state);
   }
 
-  /**
-   * Set the current browser server information (atomic: only overwrites if no live server exists)
-   * @param port The browser server port
-   * @param pid The browser server process ID
-   * @param schedulerId Optional scheduler ID
-   * @param updateState Function to update state atomically
-   * @param getStartTime Function to get process start time
-   */
-  async setBrowserServer(
-    port: number,
-    pid: number,
-    schedulerId: string | undefined,
-    updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>,
-    getStartTime: (pid: number) => Promise<number | null>,
-    authSecret?: string
-  ): Promise<void> {
-    const startTime = await getStartTime(pid);
-    await updateState((state) => {
-      return this.browserManager.setBrowserServer(state, port, pid, schedulerId, startTime, authSecret);
-    });
-  }
-
   /**
    * Clear the browser server information
    * @param updateState Function to update state atomically

package/src/infrastructure/state/state-browser-manager.ts

@@ -23,27 +23,6 @@ export class StateBrowserManager implements IService {
     return state.browserServer ?? null;
   }
 
-  /**
-   * Set the current browser server information
-   * @param state The current state
-   * @param port The browser server port
-   * @param pid The browser server process ID
-   * @param schedulerId Optional scheduler ID
-   * @param startTime Optional process start time
-   * @returns Updated state with browser server info
-   */
-  setBrowserServer(
-    state: SingletonState,
-    port: number,
-    pid: number,
-    schedulerId?: string,
-    startTime?: number | null,
-    authSecret?: string
-  ): SingletonState {
-    state.browserServer = { port, pid, schedulerId, startTime: startTime ?? undefined, authSecret };
-    return state;
-  }
-
   /**
    * Clear the browser server information
    * @param state The current state

package/src/infrastructure/state/state-manager-service.ts

@@ -178,13 +178,6 @@ export class StateManagerService implements IStateManager {
     return this.getStateManager().getBrowserServer();
   }
 
-  /**
-   * Set the current browser server information
-   */
-  async setBrowserServer(port: number, pid: number, schedulerId?: string, authSecret?: string): Promise<void> {
-    return this.getStateManager().setBrowserServer(port, pid, schedulerId, authSecret);
-  }
-
   /**
    * Clear the browser server information
    */

package/src/infrastructure/state/state-manager.ts

@@ -435,17 +435,6 @@ export class StateManager {
     return this.browserApi.getBrowserServer(this.readState.bind(this));
   }
 
-  public async setBrowserServer(port: number, pid: number, schedulerId?: string, authSecret?: string): Promise<void> {
-    await this.browserApi.setBrowserServer(
-      port,
-      pid,
-      schedulerId,
-      this.updateState.bind(this),
-      this.processLifecycle.getProcessStartTime.bind(this.processLifecycle),
-      authSecret
-    );
-  }
-
   public async clearBrowserServer(expected?: { pid?: number; schedulerId?: string }): Promise<void> {
     await this.browserApi.clearBrowserServer(this.updateState.bind(this), expected);
   }

package/src/infrastructure/state/state-path-configuration.ts

@@ -7,7 +7,7 @@
  */
 
 import * as path from 'node:path';
-import { getGlobalConfigDir } from '../../config.ts';
+import { getStateDir } from '../../config.ts';
 import type { IService } from '../../core/service-registry.ts';
 import { ServiceLifecycle } from '../../core/service-registry.ts';
 import { ServiceNames } from '../../core/interfaces/service-names.ts';
@@ -39,8 +39,12 @@ export class StatePathConfiguration implements IService {
 
   constructor(stateDir?: string) {
     // pi-research's own namespace (~/.pi/research/state), beside config.env and
-    // knowledge_db — not the host pi config root (~/.pi/state).
-    const resolvedStateDir = stateDir || path.join(getGlobalConfigDir(), 'state');
+    // knowledge_db — not the host pi config root (~/.pi/state). A non-absolute
+    // explicit override is ignored in favor of getStateDir()'s own resolution
+    // (which re-derives from PI_RESEARCH_STATE_DIR/the default and warns) — see
+    // getStateDir()'s doc comment for why a relative state dir must never be
+    // resolved against whatever process.cwd() happens to be at construction time.
+    const resolvedStateDir = stateDir && path.isAbsolute(stateDir) ? stateDir : getStateDir();
     this.stateDir = resolvedStateDir;
     this.stateFilePath = path.join(resolvedStateDir, 'research-state.json');
     this.lockDirPath = path.join(resolvedStateDir, '.locks');

package/src/knowledge/store.ts

@@ -885,8 +885,27 @@ export class KnowledgeStore implements IKnowledgeStore {
       metrics.increment('knowledge_store_cache_hits_total', 1, { status: 'miss' });
       return null;
     }
-    // Newest generation wins, never row order.
-    results.sort((a: any, b: any) => Number(b.timestamp) - Number(a.timestamp));
+    // Newest generation wins, never row order. Ties (multiple chunk rows from
+    // the same write batch share one timestamp — see writer-queue.ts's
+    // insertedAt) break on chunkIndex ascending: only the chunk-0 row carries
+    // the complete `content` field this method promises callers ("...→
+    // unfragmented original markdown, not stitched vector chunks" per
+    // rebuildDocuments()'s doc comment in research-knowledge-search.ts).
+    // LanceDB's fragment-scan order is not insertion order, so without this a
+    // non-zero chunk sorting first would silently return just that one
+    // chunk's own fragment in place of the full document.
+    const chunkIndexOf = (row: any): number => {
+      try {
+        const parsed = JSON.parse(row.metadata as string);
+        return typeof parsed.chunkIndex === 'number' ? parsed.chunkIndex : 0;
+      } catch {
+        return 0;
+      }
+    };
+    results.sort((a: any, b: any) => {
+      const tsDiff = Number(b.timestamp) - Number(a.timestamp);
+      return tsDiff !== 0 ? tsDiff : chunkIndexOf(a) - chunkIndexOf(b);
+    });
 
     const r = results[0];
     // Read-time freshness. The stored timestamp (ms) is otherwise discarded here;
@@ -908,7 +927,12 @@ export class KnowledgeStore implements IKnowledgeStore {
     try {
       const metadata = JSON.parse(r.metadata as string);
       const description: string | null = typeof metadata.description === 'string' ? metadata.description : null;
-      const textToReturn = (r.content as string) || (r.text as string) || description || '';
+      // description is complete on every chunk row (spread verbatim from
+      // item.metadata at ingest time — see writer-queue.ts's docs.map), unlike
+      // r.text, which is only that ONE row's own chunk fragment. Prefer it
+      // over r.text so a selected non-content, non-chunk-0 row still returns
+      // a complete (if shorter) document instead of a silently truncated one.
+      const textToReturn = (r.content as string) || description || (r.text as string) || '';
       metrics.increment('knowledge_store_cache_hits_total', 1, { status: 'hit' });
       logger.log(`[store] Cache hit: synthesis-description for ${url} (${textToReturn.length} chars, ~${ageDays ?? '?'}d old)`);
       return { text: textToReturn, description, metadata, ageDays };
@@ -1091,7 +1115,12 @@ export class KnowledgeStore implements IKnowledgeStore {
       if (scopeFilter === '1 = 0') return;
 
       const ttlDays = this.options.ttlDays ?? 30;
-      const cutoffTimestamp = Date.now() - (ttlDays * 24 * 60 * 60 * 1000);
+      // Math.trunc: ttlDays is schema-typed as a plain number (fractional values
+      // are accepted), and BigInt() throws a RangeError on a non-integer — which
+      // would otherwise silently disable eviction entirely for this run (caught
+      // by the catch below and only logged) whenever the configured value or the
+      // current timestamp happens to leave a fractional cutoff.
+      const cutoffTimestamp = Math.trunc(Date.now() - (ttlDays * 24 * 60 * 60 * 1000));
       // Scope eviction to this workspace only — do not evict other projects' data.
       await table.delete(`timestamp < ${BigInt(cutoffTimestamp)} AND (${scopeFilter})`);
       logger.log(`[store] Ran eviction for records older than ${ttlDays} days [scope: ${scopeFilter}]`);

package/src/orchestration/researcher-executor.ts

@@ -454,11 +454,21 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
       observer?.onResearcherComplete?.(id, responseText);
       return;
     } catch (err) {
-      const researcherDuration = Date.now() - researcherExecutionStartMs;
-      metrics.observe('researcher_execution_latency_ms', researcherDuration, { mode: 'deep', complexity: String(complexity), round: String(round), status: 'error' });
-      metrics.increment('researcher_errors_total', 1, { mode: 'deep', complexity: String(complexity), round: String(round) });
       lastError = err;
       const errMsg = err instanceof Error ? err.message : String(err);
+      // Computed up front (not just at the retry-skip check below) so the
+      // metrics right after don't mislabel a clean cancellation as an error.
+      // The sentinel has TWO forms (see the retry-skip check's own comment);
+      // isAbortSentinel matches both.
+      const tornDown = Boolean(signal?.aborted || isAbortSentinel(errMsg) || errMsg.includes('during container disposal') || container?.isDisposing);
+      const researcherDuration = Date.now() - researcherExecutionStartMs;
+      metrics.observe('researcher_execution_latency_ms', researcherDuration, { mode: 'deep', complexity: String(complexity), round: String(round), status: tornDown ? 'cancelled' : 'error' });
+      // A clean cancellation/teardown is not a researcher fault — mirrors
+      // quick-research-orchestrator's equivalent catch, which computes this
+      // same distinction before touching its own error-only counters.
+      if (!tornDown) {
+        metrics.increment('researcher_errors_total', 1, { mode: 'deep', complexity: String(complexity), round: String(round) });
+      }
 
       // Attempt to salvage partial content on timeout or error
       try {
@@ -490,8 +500,8 @@ export async function runResearcher(options: RunResearcherOptions): Promise<void
       // Retrying during disposal is futile — services are being destroyed — and it
       // relaunches search bursts into a WorkerPool that dispose() is simultaneously
       // destroying, producing a storm of "Cannot execute a task on destroying pool"
-      // errors.
-      if (signal?.aborted || isAbortSentinel(errMsg) || errMsg.includes('during container disposal') || container?.isDisposing) {
+      // errors. (tornDown computed above, alongside the metrics it also gates.)
+      if (tornDown) {
         logger.debug(`[ResearcherExecutor] Researcher ${id} aborted or container disposing, skipping retries.`);
         break; // Break out of the attempt loop
       }

package/src/tools/research-tool-definition.ts

@@ -305,8 +305,6 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
               researchId: startResearchSession(piSessionId, researchId),
               piSessionId,
               masterWidgetId: tuiManager.masterWidgetId,
-              panelState,
-              waveTimer: null,
               unsubOrder: null,
             };
             registerSessionAbort(piSessionId, cleanupCtx.researchId, internalAbort);

package/src/types/index.ts

@@ -34,8 +34,6 @@ export interface CleanupContext {
   researchId: string;
   piSessionId: string;
   masterWidgetId: string;
-  panelState: import('./research-panel-types.ts').ResearchPanelState;
-  waveTimer: NodeJS.Timeout | null;
   unsubOrder: (() => void) | null;
   unsubOrderRef?: { value: (() => void) | null };
 }

package/src/types/research-panel-types.ts

@@ -49,8 +49,6 @@ export interface ResearchPanelState {
   waveFrame?: number;
   /** Last-visit frame per position for wave animation decay */
   waveColors?: number[];
-  /** Timer for wave animation */
-  waveTimer?: NodeJS.Timeout | null;
   /** Flag indicating whether completed researchers should be cleared */
   needsClear?: boolean;
   /** 1-2 word topic title derived from the query after the coordinator LLM responds */

package/src/utils/log-utils.ts

@@ -129,8 +129,20 @@ const CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
  * neither sink receives clear-text secrets or unbounded network data.
  */
 export function redactSecrets(message: string): string {
+  // slice() counts UTF-16 code units, so a naive cut at MAX_LOG_MESSAGE_LENGTH
+  // can land between the two halves of a surrogate pair (non-BMP characters —
+  // routine in scraped CJK/emoji content). The console sink then writes a
+  // lone surrogate, silently mangled to U+FFFD on UTF-8 encode; the JSON-file
+  // sink emits an unpaired \uXXXX escape that many strict JSON consumers
+  // reject outright. Same back-off truncateWithMarker (text-utils.ts) already
+  // applies for the same reason.
+  let keep = MAX_LOG_MESSAGE_LENGTH;
+  if (message.length > keep) {
+    const lastKeptUnit = message.charCodeAt(keep - 1);
+    if (lastKeptUnit >= 0xd800 && lastKeptUnit <= 0xdbff) keep -= 1;
+  }
   let out = message.length > MAX_LOG_MESSAGE_LENGTH
-    ? `${message.slice(0, MAX_LOG_MESSAGE_LENGTH)}…[truncated ${message.length - MAX_LOG_MESSAGE_LENGTH} chars]`
+    ? `${message.slice(0, keep)}…[truncated ${message.length - keep} chars]`
     : message;
   out = out.replace(ANSI_PATTERN, '');
   out = out.replace(URL_CREDENTIALS_PATTERN, '$1[REDACTED]@');

package/src/utils/url-utils.ts

@@ -31,18 +31,26 @@ export function stripTrailingLlmPunctuation(input: string): string {
       continue;
     }
     // Balance-aware removal of a trailing closing bracket: strip it only when it
-    // has no matching opener anywhere in the string (so a markdown/parenthetical
+    // has no matching opener earlier in the string (so a markdown/parenthetical
     // noise bracket is removed, but a balanced one that's part of the URL —
     // Wikipedia "..._(programming_language)", IPv6 "http://[::1]/" — is kept).
+    //
+    // A running balance over everything BEFORE this trailing bracket — not a
+    // flat total-count comparison. A flat count misjudges a string that both
+    // ends in a genuinely balanced pair AND carries an earlier, unrelated
+    // unbalanced bracket: "http://x.com/a)b(c)" has 2 ')' vs 1 '(' overall,
+    // so a flat count sees closes > opens and strips the real, balanced
+    // trailing ')' that closes "(c)" — corrupting the URL. balance > 0 here
+    // means an opener earlier in the string is still unmatched, and this
+    // trailing bracket is the one that closes it.
     if (last === ')' || last === ']' || last === '}') {
       const open = last === ')' ? '(' : last === ']' ? '[' : '{';
-      let opens = 0;
-      let closes = 0;
-      for (const ch of s) {
-        if (ch === open) opens++;
-        else if (ch === last) closes++;
+      let balance = 0;
+      for (const ch of s.slice(0, -1)) {
+        if (ch === open) balance++;
+        else if (ch === last) balance = Math.max(0, balance - 1);
       }
-      if (closes > opens) {
+      if (balance === 0) {
         s = s.slice(0, -1);
         continue;
       }

package/src/web-research/scraper-utils.ts

@@ -31,15 +31,19 @@ import { isCloudflareBlockError, isTaskTimeoutError, isPoolShutdownError } from
  * failures at debug instead of error — keeping the ERROR log (and, via
  * logger.error's error-tracker re-track, the diagnostic error count) focused on
  * genuine faults. The messages matched here are produced by validateContent
- * (`Fetch blocked:` / `Fetch returned stub:`) and scrapeWithFetch (`HTTP <code>`);
- * the remaining cases delegate to the shared browser-layer predicates.
+ * (`Fetch blocked:` / `Fetch returned stub:`), scrapeWithFetch (`HTTP <code>`)
+ * and extractPdfToMarkdown (`Could not extract content from PDF`) — a
+ * malformed/encrypted/scanned-image-only PDF is routine on the open web and
+ * just as non-actionable as a stub page or a 404; the remaining cases
+ * delegate to the shared browser-layer predicates.
  */
 export function isBenignScrapeFailure(error: unknown): boolean {
   const msg = error instanceof Error ? error.message : String(error);
   return (
-    msg.startsWith('Fetch blocked:') ||        // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
-    msg.startsWith('Fetch returned stub:') ||  // nav-only / near-empty page
-    /^HTTP \d{3}\b/.test(msg) ||               // remote 4xx/5xx status
+    msg.startsWith('Fetch blocked:') ||                  // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
+    msg.startsWith('Fetch returned stub:') ||             // nav-only / near-empty page
+    msg.startsWith('Could not extract content from PDF') || // malformed/encrypted/scanned-only PDF
+    /^HTTP \d{3}\b/.test(msg) ||                          // remote 4xx/5xx status
     isCloudflareBlockError(error) ||
     isTaskTimeoutError(error) ||
     isPoolShutdownError(error)

package/src/web-research/web-scraper.ts

@@ -110,7 +110,17 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
     return markdown;
   } catch (e) {
     const msg = e instanceof Error ? e.message : String(e);
-    logger.error(`[Scrapers] PDF extraction failed: ${msg}`);
+    // debug, not error: a malformed/encrypted/scanned-image-only PDF is an
+    // expected, non-actionable per-URL outcome, not a genuine fault — the same
+    // reason isBenignScrapeFailure (scraper-utils.ts) keeps stub/blocked/HTTP-error
+    // content out of ERROR. This is called from both the fetch layer (where a
+    // browser-layer fallback always follows — the fetch layer's own catch at line
+    // ~529 already logs ANY of its failures at debug for that reason) and the
+    // browser layer (the last resort — isBenignScrapeFailure now recognizes this
+    // message so that terminal call site's own classification stays consistent
+    // too). Logging ERROR here, before either caller gets a chance to classify,
+    // pre-empted both of those and always alarmed regardless.
+    logger.debug(`[Scrapers] PDF extraction failed: ${msg}`);
     metrics.increment('scrape_pdf_errors_total', 1, { error_type: 'extraction_failed' });
     errorTracker.trackError(e instanceof Error ? e : String(e), {
       component: 'scrapers',
@@ -590,7 +600,22 @@ export async function scrape(urls: string[], maxConcurrency = 5, signal?: AbortS
     // settle via scrapeSingle's own cancellation classification. Without this,
     // every remaining batch still ran its full fetch + browser-fallback cycle
     // after a user cancel.
-    if (signal?.aborted) break;
+    if (signal?.aborted) {
+      // scrapeSingle reports {success:false, error:'Aborted'} for a URL that
+      // was already in flight when the signal fired — the sub-batches that
+      // never even got dispatched (everything from here on) need the same
+      // treatment. Without this, the caller's own results.length silently
+      // fell short of urls.length with no indication a run was cut short —
+      // the same "cancellation resolves through a normal, non-throwing path
+      // and is invisible to the caller" gap the v1.3.1 research-tool fix
+      // closed at the orchestrator level, recurring one layer down here.
+      for (const url of urls.slice(i)) {
+        const aborted: ScrapeResult = { url, success: false, error: 'Aborted', markdown: '' };
+        onUrlComplete?.(aborted);
+        results.push(aborted);
+      }
+      break;
+    }
     const batch = urls.slice(i, i + maxConcurrency);
     const batchRes = await Promise.all(
       batch.map(async url => {
@@ -601,10 +626,10 @@ export async function scrape(urls: string[], maxConcurrency = 5, signal?: AbortS
     );
     results.push(...batchRes);
   }
-  
+
   const batchDuration = Date.now() - batchStart;
   metrics.observe('scrape_batch_latency_ms', batchDuration);
-  
+
   return results;
 }