@lincoln504/pi-research 1.0.10 → 1.0.12

package/src/infrastructure/research-run-semaphore.ts

@@ -0,0 +1,290 @@
+/**
+ * Research Run Semaphore
+ *
+ * Cross-process cap on the number of concurrent research runs, built on the
+ * verified {@link FileLockService}. This is the Phase 1 "run-cap" mitigation for
+ * the parallel-runs degradation documented in
+ * INVESTIGATION-2026-08-02-parallel-runs-sigabrt.md: when more than N research
+ * runs pile onto a single leader-elected browser/embedding pool, the shared
+ * priority queue saturates, runs churn through leader elections, and research
+ * degrades into ~60-error failures. Capping concurrency at the orchestration
+ * entry stops that at the source.
+ *
+ * ## Design
+ *
+ * A counting semaphore of N slots is realized as N well-known slot files
+ * (`research-run-slot-<i>.lock`) in the shared state directory, coordinated with
+ * {@link FileLockService}.
+ *
+ * - **Acquire** tries each slot once (round-robin from a rotating start index to
+ *   spread contention). Each per-slot attempt builds a **fresh** `FileLockService`
+ *   configured as a *try-take* (`lockTimeout: 0`): it either wins the slot (free,
+ *   or reclaimed from a *dead* owner) immediately or throws as soon as it sees a
+ *   *live* owner. If every slot is held by a live process, the loop waits
+ *   `pollDelayMs` and retries until `maxWaitMs` elapses, then rejects with
+ *   {@link ResearchRunCapacityError} (fail-fast with a clear message rather than
+ *   a 60-error degraded run).
+ * - **Release** unlinks the slot file via the captured slot service.
+ * - **Crash recovery** is automatic and PID-reuse-safe: a slot held by a process
+ *   that has died (killed, crashed, SIGKILLed by earlyoom) is reclaimed
+ *   immediately on the next acquire attempt, because `FileLockService` checks the
+ *   owner's PID+startTime liveness before treating a lock as stale.
+ *
+ * ### Why a *fresh* FileLockService per attempt
+ * `FileLockService.acquireLock()` serializes callers on a per-instance FIFO queue
+ * and only releases that queue turn in `releaseLock()` — i.e. after the run ends.
+ * Reusing a persistent per-slot instance would therefore **deadlock concurrent
+ * research starts within one process**: the first holder keeps slot[i]'s queue
+ * turn for minutes, blocking a second acquire on that slot forever (it would
+ * never reach the other slots). A throwaway instance per attempt has an empty
+ * queue, so concurrent attempts never block on stale queue state — while still
+ * reusing all of FileLockService's verified atomic-create / liveness / reclaim
+ * logic. Two concurrent attempts on the *same* slot simply race on `fs.open(wx)`:
+ * one wins, the other reads a live owner and fails fast to the next slot.
+ *
+ * ### Why `liveOwnerStaleThreshold` is effectively infinite
+ * A research run legitimately holds its slot for *minutes*. `FileLockService`
+ * would normally steal a lock from a *live* owner once it ages past
+ * `liveOwnerStaleThreshold` (to recover a process stalled mid-critical-section).
+ * For a run-cap that is exactly wrong: stealing a live run's slot would silently
+ * admit an (N+1)th concurrent run — reintroducing the saturation this guard
+ * exists to prevent. So slots are configured to *never* steal from a live owner;
+ * only provably-dead owners are reclaimed (immediately, regardless of age).
+ *
+ * ### Concurrency within one process
+ * `acquire()` returns an independent {@link RunSlotAcquisition} handle capturing
+ * its own slot service; it stores *no* shared "held slot" state. Two concurrent
+ * `runResearch` calls on the same singleton service each acquire a distinct slot
+ * safely (see "fresh instance per attempt" above), with no risk of one release
+ * clobbering the other's tracking.
+ */
+
+import * as pathmod from 'node:path';
+import { logger } from '../logger.ts';
+import { metrics } from '../utils/metrics.ts';
+import { FileLockService } from './file-lock-service.ts';
+import { ServiceLifecycle } from '../core/service-registry.ts';
+import { ServiceNames } from '../core/interfaces/service-names.ts';
+import type { IService } from '../core/service-registry.ts';
+import type { IProcessLifecycle } from '../core/interfaces/process-interfaces.ts';
+
+/** Default maximum number of concurrent research runs across all processes. */
+export const DEFAULT_MAX_CONCURRENT_RUNS = 2;
+/** Default per-acquire wait before failing fast when all slots are busy (0 = fail-fast). */
+export const DEFAULT_ACQUIRE_TIMEOUT_MS = 0;
+/** Poll interval between full slot sweeps while waiting for a slot. */
+const POLL_DELAY_MS = 250;
+
+/**
+ * Thrown when every run slot is held by a live process and the acquire timeout
+ * has elapsed. Distinct from internal/IO errors so callers can treat
+ * "capacity exhausted" (retry later) differently from "the semaphore is broken".
+ */
+export class ResearchRunCapacityError extends Error {
+  readonly slots: number;
+  constructor(slots: number, maxWaitMs: number) {
+    super(
+      `Maximum concurrent research runs (${slots}) reached${maxWaitMs > 0 ? ` after waiting ${maxWaitMs}ms` : ''}; ` +
+        `wait for an in-progress run to finish and retry. ` +
+        `(Override with PI_RESEARCH_MAX_CONCURRENT_RUNS.)`,
+    );
+    this.name = 'ResearchRunCapacityError';
+    this.slots = slots;
+  }
+}
+
+/** A held run slot. Call {@link RunSlotAcquisition.release} exactly once when the run ends. */
+export interface RunSlotAcquisition {
+  /** The 0-based slot index that was acquired (observability). */
+  readonly slotIndex: number;
+  /** Release the slot back to the pool. Safe to call once; subsequent calls are no-ops. */
+  release(): Promise<void>;
+}
+
+/**
+ * Counting semaphore over N slot files coordinated with {@link FileLockService}.
+ */
+export class ResearchRunSemaphore implements IService {
+  readonly name = ServiceNames.RESEARCH_RUN_SEMAPHORE;
+  lifecycle = ServiceLifecycle.UNINITIALIZED;
+
+  private readonly slotDir: string;
+  private readonly processLifecycle: IProcessLifecycle | null;
+  private readonly maxSlots: number;
+  private readonly defaultMaxWaitMs: number;
+  /** Rotating start index so contending processes don't all hammer slot 0 first. */
+  private nextStartIndex = 0;
+  private _initialized = false;
+
+  /**
+   * @param slotDir    Directory holding the `research-run-slot-<i>.lock` files.
+   * @param processLifecycle PID+startTime liveness source (PID-reuse-safe reclamation).
+   * @param maxSlots   N — maximum concurrent runs. Defaults from env then {@link DEFAULT_MAX_CONCURRENT_RUNS}.
+   * @param defaultMaxWaitMs Default acquire wait. Defaults from env then {@link DEFAULT_ACQUIRE_TIMEOUT_MS}.
+   */
+  constructor(slotDir: string, processLifecycle: IProcessLifecycle | null, maxSlots?: number, defaultMaxWaitMs?: number) {
+    this.slotDir = slotDir;
+    this.processLifecycle = processLifecycle;
+    this.maxSlots = Math.max(1, Math.floor(
+      maxSlots ?? numberFromEnv('PI_RESEARCH_MAX_CONCURRENT_RUNS') ?? DEFAULT_MAX_CONCURRENT_RUNS,
+    ));
+    this.defaultMaxWaitMs = Math.max(0, Math.floor(
+      defaultMaxWaitMs ?? numberFromEnv('PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS') ?? DEFAULT_ACQUIRE_TIMEOUT_MS,
+    ));
+  }
+
+  /** Number of slots (max concurrent runs). */
+  getMaxSlots(): number {
+    return this.maxSlots;
+  }
+
+  async initialize(): Promise<void> {
+    if (this._initialized) return;
+    this.lifecycle = ServiceLifecycle.INITIALIZING;
+    // Ensure the slot directory exists; slot files are created lazily on acquire.
+    try {
+      await import('node:fs/promises').then((fs) => fs.mkdir(this.slotDir, { recursive: true, mode: 0o700 }));
+    } catch (err) {
+      logger.warn(`[ResearchRunSemaphore] Could not create slot dir ${this.slotDir}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    this._initialized = true;
+    this.lifecycle = ServiceLifecycle.INITIALIZED;
+    logger.log(`[ResearchRunSemaphore] Initialized with ${this.maxSlots} run slot(s) (acquire timeout ${this.defaultMaxWaitMs}ms).`);
+  }
+
+  async dispose(): Promise<void> {
+    // Held slots are released by their RunSlotAcquisition handles (the runResearch
+    // finally block). A hard process exit leaves slot files behind, which the next
+    // acquire reclaims via PID+startTime liveness — so dispose has nothing to do.
+    this.lifecycle = ServiceLifecycle.DISPOSED;
+  }
+
+  /**
+   * Try to win one slot NON-BLOCKING. Builds a fresh {@link FileLockService}
+   * configured as a try-take (see class doc: "fresh instance per attempt").
+   * Returns an acquisition on success, or null if the slot is busy / lost a race.
+   */
+  private async trySlot(slotIndex: number): Promise<RunSlotAcquisition | null> {
+    const slot = new FileLockService({
+      lockFilePath: pathmod.join(this.slotDir, `research-run-slot-${slotIndex}.lock`),
+      lockTimeout: 0, // fail immediately on a live-contended slot (try-take)
+      lockRetries: 24, // enough iterations to complete a dead-owner reclaim + reacquire
+      lockRetryDelay: 50,
+      lockStaleThreshold: 15000, // legacy (no-pid) locks: reclaim once 15s old
+      // A live process holding a run slot for the entire (minutes-long) run is the
+      // NORMAL case, not a stall. Never steal from a live owner — doing so would
+      // admit an (N+1)th concurrent run, the exact condition this guard prevents.
+      // Dead owners are reclaimed immediately regardless of age.
+      liveOwnerStaleThreshold: Number.MAX_SAFE_INTEGER,
+      processLifecycle: this.processLifecycle ?? undefined,
+    });
+    try {
+      await slot.initialize(); // resolves own startTime (cached) + startup stale cleanup
+      await slot.acquireLock(); // try-take: wins, reclaims a dead owner, or throws fast
+    } catch {
+      return null; // busy (live owner) or lost a reclaim race — try the next slot
+    }
+    return {
+      slotIndex,
+      release: async () => {
+        try {
+          await slot.releaseLock();
+        } catch (err) {
+          logger.warn(`[ResearchRunSemaphore] Failed to release slot ${slotIndex}: ${err instanceof Error ? err.message : String(err)}`);
+        } finally {
+          // Clear the gauge even if the unlink failed: the slot is no longer held
+          // by THIS process either way, and a gauge that only ever ratchets to 1
+          // reports permanent saturation for the life of the process.
+          metrics.setGauge('research_run_slots_held', 0, { slot: String(slotIndex) });
+        }
+      },
+    };
+  }
+
+  /**
+   * Acquire one run slot, waiting up to `maxWaitMs` (defaults to the constructed
+   * value) when all slots are held by live processes. Resolves with a handle that
+   * must be released when the run ends.
+   *
+   * @param signal aborts the wait. Without it a run started under a long
+   *   `PI_RESEARCH_RUN_ACQUIRE_TIMEOUT_MS` would ignore Ctrl-C until the deadline
+   *   passed, because the wait happens before any orchestrator exists to cancel.
+   * @throws {ResearchRunCapacityError} if no slot frees within `maxWaitMs`.
+   * @throws {Error} named `AbortError` if `signal` aborts while waiting.
+   */
+  async acquire(maxWaitMs: number = this.defaultMaxWaitMs, signal?: AbortSignal): Promise<RunSlotAcquisition> {
+    if (!this._initialized) await this.initialize();
+    if (signal?.aborted) {
+      metrics.increment('research_run_acquire_total', 1, { status: 'aborted' });
+      throwIfAborted(signal);
+    }
+
+    const deadline = Date.now() + Math.max(0, maxWaitMs);
+    for (;;) {
+      // Round-robin from a rotating start so contending processes don't synchronize on slot 0.
+      const start = this.nextStartIndex;
+      this.nextStartIndex = (this.nextStartIndex + 1) % this.maxSlots;
+      for (let offset = 0; offset < this.maxSlots; offset++) {
+        const i = (start + offset) % this.maxSlots;
+        const got = await this.trySlot(i);
+        if (got) {
+          metrics.setGauge('research_run_slots_held', 1, { slot: String(i) });
+          metrics.increment('research_run_acquire_total', 1, { status: 'acquired' });
+          return got;
+        }
+      }
+      // Every slot is held by a live process.
+      if (Date.now() >= deadline) {
+        metrics.increment('research_run_acquire_total', 1, { status: 'exhausted' });
+        throw new ResearchRunCapacityError(this.maxSlots, maxWaitMs);
+      }
+      // Count the aborted outcome too, so acquired + exhausted + aborted accounts for
+      // every acquire attempt. A cancel that vanished from the counter would make the
+      // capacity metrics quietly under-report demand.
+      try {
+        await sleep(POLL_DELAY_MS, signal);
+      } catch (err) {
+        metrics.increment('research_run_acquire_total', 1, { status: 'aborted' });
+        throw err;
+      }
+    }
+  }
+}
+
+function throwIfAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw Object.assign(new Error('Research run slot acquisition aborted'), { name: 'AbortError' });
+  }
+}
+
+function numberFromEnv(key: string): number | undefined {
+  const raw = process.env[key];
+  if (raw === undefined || raw === '') return undefined;
+  const n = Number(raw);
+  return Number.isFinite(n) ? n : undefined;
+}
+
+/**
+ * Poll delay that resolves early (rejecting) on abort.
+ *
+ * The timer is deliberately NOT `unref`'d. Waiting for a run slot is foreground work: a process
+ * whose only pending handle is this timer must stay alive until the slot frees, and an unref'd
+ * timer would let Node drain the loop and exit mid-wait (observed as an early exit in the
+ * multi-process harness). The timer is always settled — it either fires or is cleared on abort —
+ * so keeping it referenced cannot outlive the acquire.
+ */
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      if (onAbort) signal?.removeEventListener('abort', onAbort);
+      resolve();
+    }, ms);
+    const onAbort = signal
+      ? () => {
+          clearTimeout(timer);
+          reject(Object.assign(new Error('Research run slot acquisition aborted'), { name: 'AbortError' }));
+        }
+      : undefined;
+    if (onAbort) signal?.addEventListener('abort', onAbort, { once: true });
+  });
+}

package/src/infrastructure/service-initialization.ts

@@ -22,6 +22,7 @@ import { StateValidator } from './state/state-validator.ts';
 import { WorkerPoolManager } from './browser/worker-pool-manager.ts';
 import { StatePathConfiguration } from './state/state-path-configuration.ts';
 import { FileLockService } from './file-lock-service.ts';
+import { ResearchRunSemaphore } from './research-run-semaphore.ts';
 import { StateBackupManager } from './state/state-backup-manager.ts';
 import { logger } from '../logger.ts';
 
@@ -73,6 +74,25 @@ export function registerInfrastructureServices(container: ServiceContainer = get
     container
   );
 
+  // Register Research Run Semaphore (Phase 1 cross-process run-cap).
+  // Depends on StatePathConfiguration (slot dir) and ProcessLifecycle (PID-reuse-safe
+  // reclamation). Lazy: only built on first research run. Acquired at runResearch()
+  // entry; released in its finally. A bug here fails OPEN (research proceeds) so it
+  // can never break research; genuine capacity exhaustion fails fast with a clear error.
+  registerService(
+    ServiceNames.RESEARCH_RUN_SEMAPHORE,
+    async () => {
+      const pathConfig = await getService<StatePathConfiguration>(ServiceNames.STATE_PATH_CONFIGURATION, undefined, container);
+      const processLifecycle = await getService<ProcessLifecycleService>(ServiceNames.PROCESS_LIFECYCLE, undefined, container);
+      return new ResearchRunSemaphore(pathConfig.getStateDir(), processLifecycle);
+    },
+    {
+      allowOverwrite: false,
+      enableLogging: true,
+    },
+    container
+  );
+
   // Register State Backup Manager Service
   // Depends on StatePathConfiguration
   registerService(

package/src/orchestration/deep-research-orchestrator.ts

@@ -555,6 +555,16 @@ export class DeepResearchOrchestrator {
         logger.debug('[DeepOrchestrator] onComplete observer threw:', obsErr);
       }
 
+      // Observability: log the FULL final synthesis body. Every other layer —
+      // researcher prompts, researcher final responses, searches, scrapes — is logged
+      // at DEBUG; the synthesis itself (the actual bytes returned to the caller) was
+      // the one conspicuous exception, which made post-hoc diagnosis of citation /
+      // output issues impossible (a reported "the engine returned placeholder X"
+      // could not be confirmed or refuted from the logs). Mirrors the DEBUG logging
+      // of researcher final responses. Truncation is deliberately NOT applied: the
+      // raw bytes are exactly what's needed to diagnose grounding failures.
+      logger.debug(`[DeepOrchestrator] Final synthesis for ${researchId} (${result.length} chars):\n${result}`);
+
       return result;
 
     } catch (error) {
@@ -589,6 +599,7 @@ export class DeepResearchOrchestrator {
           } catch (obsErr) {
             logger.debug('[DeepOrchestrator] fallback onComplete observer threw:', obsErr);
           }
+          logger.debug(`[DeepOrchestrator] Fallback synthesis for ${researchId} (${result.length} chars):\n${result}`);
           return result;
         }
       } catch (fallbackErr) {

package/src/orchestration/research-orchestration-service.ts

@@ -37,6 +37,7 @@ import type { Config } from '../config.ts';
 import { getConfig, DEFAULTS } from '../config.ts';
 import { getCachedScrapedContent, normalizeUrl, cleanupSharedLinks } from '../utils/shared-links.ts';
 import { runResearcher } from './researcher-executor.ts';
+import { ResearchRunSemaphore, ResearchRunCapacityError } from '../infrastructure/research-run-semaphore.ts';
 import { recordResearcherFailure, shouldStopResearch, createResearchStopError } from './session-state.ts';
 import type { ResearchSessionService } from './research-session-service.ts';
 import { QuickResearchOrchestrator } from './quick-research-orchestrator.ts';
@@ -107,6 +108,26 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
 
     const researchStart = Date.now();
 
+    // Cross-process run-cap (Phase 1): acquire one of N slots so concurrent
+    // research runs can't saturate the shared leader-elected browser/embedding
+    // pool and degrade into ~60-error failures. Fail-OPEN on internal/IO errors
+    // (a semaphore bug must never break research); only a genuine
+    // ResearchRunCapacityError (all slots held by live processes) propagates,
+    // failing the run fast with a clear message. The slot is released in finally.
+    let runSlot: { slotIndex: number; release(): Promise<void> } | null = null;
+    try {
+      const container = tryGetServiceContainerFromCtx(ctx);
+      const semaphore = await getService<ResearchRunSemaphore>(ServiceNames.RESEARCH_RUN_SEMAPHORE, ctx, container);
+      runSlot = await semaphore.acquire(undefined, signal);
+      logger.info(`[ResearchOrchestrationService] Acquired research run slot ${runSlot.slotIndex} (cap ${semaphore.getMaxSlots()}).`);
+    } catch (err) {
+      if (err instanceof ResearchRunCapacityError) throw err; // capacity exhausted → fail fast
+      // A cancel while queueing for a slot must stay cancelled. Fail-open here would
+      // start the very run the user just aborted.
+      if (err instanceof Error && err.name === 'AbortError') throw err;
+      logger.warn(`[ResearchOrchestrationService] Run-cap unavailable, proceeding without it: ${err instanceof Error ? err.message : String(err)}`);
+    }
+
     let result: string;
     try {
       if (depth === 0) {
@@ -149,6 +170,20 @@ export class ResearchOrchestrationService implements IResearchOrchestration {
       metrics.observe('research_manager_latency_ms', researchDuration, { depth: String(depth), status: 'error', source: 'extension' });
       metrics.increment('research_manager_requests_total', 1, { depth: String(depth), status: 'error', source: 'extension' });
       throw error;
+    } finally {
+      // Always release the run-cap slot — on success, error, or abort (the
+      // orchestrator throws on AbortSignal, hitting this finally before rethrow).
+      if (runSlot) {
+        // Never let a release failure mask the run's own outcome — but do report it.
+        // A slot that fails to release stays held for the lifetime of THIS process
+        // (liveness reclaim only fires once the owner is gone), so it silently costs
+        // one unit of capacity for every subsequent run in the session.
+        await runSlot.release().catch((relErr) => {
+          logger.warn(
+            `[ResearchOrchestrationService] Failed to release run slot ${runSlot?.slotIndex}; capacity is reduced until this process exits: ${relErr instanceof Error ? relErr.message : String(relErr)}`,
+          );
+        });
+      }
     }
   }
 

package/src/orchestration/research-synthesis-service.ts

@@ -13,6 +13,7 @@ import { parseCitations } from '../utils/text-utils.ts';
 import { normalizeCitations, formatCitedLinks, type GlobalCitation } from '../utils/citation-utils.ts';
 import { lastCitedLinksHeaderIndex } from '../utils/text-utils.ts';
 import { getScrapedLinks, isTranscribedLink, normalizeUrl } from '../utils/shared-links.ts';
+import { stripTrailingLlmPunctuation } from '../utils/url-utils.ts';
 import { logger } from '../logger.ts';
 import { ServiceLifecycle, type IService } from '../core/service-registry.ts';
 import { ServiceNames } from '../core/interfaces/service-names.ts';
@@ -26,6 +27,125 @@ import type { SteeringMessage } from './session-state.ts';
 const NO_SOURCES_NOTE =
   'CITED LINKS\n_No web sources were successfully retrieved for this report — searches or page fetches may have been blocked (for example by site bot-protection) or returned no usable results. Treat the findings above as unverified._';
 
+// Appended when the model wrote its own links but the run retrieved nothing we could
+// verify them against. The block is kept (it may still be a useful lead) but must not
+// read as though the engine checked it.
+const UNVERIFIED_LINKS_NOTE =
+  '_None of the links above could be verified: no page fetch succeeded this session and no citation could be matched to retrieved content. They may be inaccurate or invented — check each one before relying on it._';
+
+/**
+ * Largest `[N]` treated as a citation marker. Inline markers index a source list
+ * that is realistically well under this; the bound exists so an ordinary bracketed
+ * number in prose — a year (`[2023]`), an issue id, an array index — is left alone
+ * instead of being silently deleted as a "dangling citation".
+ */
+const MAX_PLAUSIBLE_CITATION_MARKER = 200;
+
+/**
+ * Rewrite inline `[N]` citation markers in a prose body: renumber the ones we can
+ * map, delete the ones known to dangle, leave everything else untouched.
+ *
+ * Deletion is whitespace-aware rather than followed by a global space collapse.
+ * A blanket `text.replace(/  +/g, ' ')` also eats *leading* indentation, which
+ * silently flattens nested list levels into one and turns 4-space indented code
+ * blocks into paragraphs — corrupting the structure of every report that contains
+ * them. So the pad preceding a deleted marker is consumed only when the marker is
+ * mid-line (i.e. real prose spacing); line-leading indentation is preserved.
+ *
+ * Code is skipped entirely. A report may quote code containing bracketed integers
+ * (`items[12]`), and on the no-CITED-LINKS path *every* out-of-range marker is treated
+ * as dangling — so without this guard a quoted array index would be silently deleted
+ * from the user's code sample. Fenced blocks are tracked across lines; single-backtick
+ * spans are detected by an odd backtick count earlier on the same line.
+ *
+ * @param remap  local marker number → verified global id
+ * @param isDangling  whether a marker not present in `remap` should be deleted
+ */
+function rewriteCitationMarkers(
+  text: string,
+  remap: Map<number, number>,
+  isDangling: (n: number) => boolean,
+): string {
+  const inFence = fencedCodeMask(text);
+  return text.replace(/([ \t]*)\[(\d+)\]/g, (match, pad: string, digits: string, offset: number) => {
+    const n = parseInt(digits, 10);
+    const mapped = remap.get(n);
+    if (mapped !== undefined) return `${pad}[${mapped}]`;
+    // Citation numbering is 1-based; `[0]` is always an index or a literal, never a marker.
+    if (n < 1 || n > MAX_PLAUSIBLE_CITATION_MARKER || !isDangling(n)) return match;
+    if (inFence(offset) || inInlineCodeSpan(text, offset)) return match;
+    // Preserve indentation when nothing but whitespace precedes on this line;
+    // otherwise drop the pad too so no double space is left behind.
+    const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
+    return text.slice(lineStart, offset).trim() === '' ? pad : '';
+  });
+}
+
+/**
+ * Build an `offset → is inside a ``` fenced block` predicate for `text`.
+ *
+ * Computed once per rewrite (a single line scan) rather than per marker, so marker
+ * rewriting stays linear in the body length.
+ */
+function fencedCodeMask(text: string): (offset: number) => boolean {
+  const ranges: Array<[number, number]> = [];
+  let open = -1;
+  let pos = 0;
+  for (const line of text.split('\n')) {
+    if (/^\s{0,3}(```|~~~)/.test(line)) {
+      if (open < 0) open = pos;
+      else {
+        ranges.push([open, pos + line.length]);
+        open = -1;
+      }
+    }
+    pos += line.length + 1; // +1 for the '\n' consumed by split
+  }
+  // An unterminated fence runs to the end of the body — treat the remainder as code.
+  if (open >= 0) ranges.push([open, text.length]);
+  return (offset: number) => ranges.some(([start, end]) => offset >= start && offset < end);
+}
+
+/** Whether `offset` sits inside a single-backtick code span on its own line. */
+function inInlineCodeSpan(text: string, offset: number): boolean {
+  const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
+  let ticks = 0;
+  for (let i = lineStart; i < offset; i++) if (text[i] === '`') ticks++;
+  return ticks % 2 === 1;
+}
+
+/**
+ * Redact inline `https?://` tokens in synthesis prose whose URL was NOT retrieved
+ * this session (absent from `trusted`, a set of normalized verified-source URLs).
+ *
+ * The synthesis body is otherwise passed through verbatim, so an unverified inline
+ * URL is the one channel by which a fabricated source can reach the user — and
+ * inline URLs violate the required format (links belong in CITED LINKS only).
+ * Legitimate inline references to real sources are preserved (they are present in
+ * `trusted`). Returns the (possibly) redacted text.
+ */
+function redactUnverifiedProseUrls(text: string, trusted: Set<string>): string {
+  if (trusted.size === 0) return text;
+  let removed = 0;
+  // Brackets are NOT excluded from the token: a bare `[^\s)\]>"']+` truncates
+  // `.../Python_(programming_language)` at the first `)`, so the truncated form
+  // never matches its own entry in `trusted` and a perfectly good Wikipedia link
+  // gets redacted. Instead take the greedy token and split off exactly the
+  // trailing punctuation `normalizeUrl` itself discards — so the compared string
+  // and the redacted span always agree — restoring that tail afterwards.
+  const result = text.replace(/https?:\/\/[^\s<>"']+/gi, (token) => {
+    const core = stripTrailingLlmPunctuation(token);
+    const tail = token.slice(core.length);
+    if (!core || trusted.has(normalizeUrl(core))) return token;
+    removed++;
+    return `[link removed: not found in retrieved sources]${tail}`;
+  });
+  if (removed > 0) {
+    logger.warn(`[ResearchSynthesisService] Redacted ${removed} unverified inline URL(s) from synthesis prose (not present in retrieved sources).`);
+  }
+  return result;
+}
+
 /**
  * Research Synthesis Service
  *
@@ -178,6 +298,18 @@ export class ResearchSynthesisService implements IService {
     let globalCitations: GlobalCitation[] =
       reports.size > 0 ? normalizeCitations(reports).globalCitations : [];
 
+    // NOTE: a "provenance gate" previously DROPPED citations tagged Scrape/Transcript
+    // whose URL was absent from the session's successful-scrape pool. It was REMOVED
+    // because it conflated a failed/blocked scrape (real URL, content unretrieved) with
+    // fabrication: under the common high scrape-failure rate (bot blocks on GitHub /
+    // Microsoft / etc.) it mass-dropped legitimate citations, and the remap below then
+    // neutralized their inline [N] markers into a broken '[removed]' token that reached
+    // users. We now KEEP every parsed citation (deduped by URL) and never emit
+    // '[removed]'; a dangling inline marker is removed cleanly instead. Fabrication
+    // defense rests on the URL dedup + the prose-URL sweep, which do not punish failed
+    // scrapes. (The successful-scrape pool is still reconciled below to FOLD IN any
+    // sources the researchers scraped but failed to cite.)
+
     // Robustness fallback: reconcile against ground-truth provenance — the URLs
     // ACTUALLY, successfully scraped this session (registerScrapedLinks only records
     // HTTP-success fetches). This runs even when SOME citations parsed: a
@@ -216,6 +348,7 @@ export class ResearchSynthesisService implements IService {
 
     if (globalCitations.length > 0) {
       const verifiedLinksSection = formatCitedLinks(globalCitations);
+      const trustedUrls = new Set(globalCitations.map((c) => normalizeUrl(c.url)));
       // Replace an existing CITED LINKS section with the verified version — but only
       // a line-leading, all-caps "CITED LINKS" header marks the section. An
       // unanchored /CITED LINKS[\s\S]*$/i would also fire on an incidental
@@ -225,31 +358,48 @@ export class ResearchSynthesisService implements IService {
       if (citedHeaderIdx >= 0) {
         logger.debug('[ResearchSynthesisService] Replacing existing CITED LINKS with verified version');
         // The verified list is renumbered (dedup by URL, implausible entries dropped),
-        // but the body's inline [N] markers were authored against the list being
-        // replaced. Remap them by URL identity — parse the synthesis's own list and
-        // map each written number to the global id of the same normalized URL. In
-        // deep mode the numbering already matches and this is a no-op; in quick mode
-        // (where the model's own list is what gets replaced) it prevents every
-        // marker after a dropped/deduped entry from pointing at the wrong source.
+        // but the body's inline [N] markers were authored against the list being replaced.
+        // Remap them by URL identity — parse the synthesis's own list and map each
+        // written number to the global id of the same normalized URL. In deep mode
+        // the numbering already matches and this is a no-op; in quick mode (where the
+        // model's own list is what gets replaced) it prevents every marker after a
+        // dropped/deduped entry from pointing at the wrong source.
         const ownCitations = parseCitations(synthesis);
         const urlToGlobalId = new Map(globalCitations.map((c) => [normalizeUrl(c.url), c.id]));
         const localToGlobal = new Map<number, number>();
+        // own-citation numbers whose URL is absent from the verified list. Their inline
+        // [N] markers are REMOVED cleanly (never '[removed]') so the prose reads
+        // naturally. A marker can dangle only if the lead cited a URL absent from the
+        // rebuilt list (e.g. it invented one not in any researcher report); removing it
+        // beats shipping a broken token or silently retargeting it at the wrong source.
+        const droppedNumbers = new Set<number>();
         ownCitations.forEach((cit, index) => {
+          const localKey = cit.number ?? index + 1;
           const globalId = urlToGlobalId.get(normalizeUrl(cit.url));
-          if (globalId !== undefined) localToGlobal.set(cit.number ?? index + 1, globalId);
+          if (globalId !== undefined) localToGlobal.set(localKey, globalId);
+          else droppedNumbers.add(localKey);
         });
         let body = synthesis.slice(0, citedHeaderIdx).trimEnd();
-        const needsRemap = [...localToGlobal].some(([local, global]) => local !== global);
-        if (needsRemap) {
-          body = body.replace(/\[(\d+)\]/g, (match, p1) => {
-            const globalId = localToGlobal.get(parseInt(p1, 10));
-            return globalId !== undefined ? `[${globalId}]` : match;
-          });
+        if (localToGlobal.size > 0 || droppedNumbers.size > 0) {
+          body = rewriteCitationMarkers(body, localToGlobal, (n) => droppedNumbers.has(n));
         }
+        // Defense-in-depth: redact unverified inline URLs in the prose body. The body
+        // is otherwise passed through verbatim, so this is the one channel by which a
+        // fabricated source can still reach the user; inline URLs also violate the
+        // required format. Verified inline references (in `trustedUrls`) are kept.
+        body = redactUnverifiedProseUrls(body, trustedUrls);
         return `${body}\n\n${verifiedLinksSection}`;
       }
       logger.warn('[ResearchSynthesisService] Synthesis missing CITED LINKS - appending verified version');
-      return `${synthesis.trim()}\n\n${verifiedLinksSection}`;
+      // No existing CITED LINKS header: the synthesis is all prose. The model's
+      // inline [N] markers were authored without a list we could remap against, so
+      // REMOVE any that fall outside the verified range (1..K) — they point at sources
+      // absent from the rebuilt list — before sweeping for unverified inline URLs and
+      // appending the verified section. Removed cleanly (never '[removed]').
+      const validIds = new Set(globalCitations.map((c) => c.id));
+      let prose = rewriteCitationMarkers(synthesis.trim(), new Map(), (n) => !validIds.has(n));
+      prose = redactUnverifiedProseUrls(prose, trustedUrls);
+      return `${prose}\n\n${verifiedLinksSection}`;
     }
 
     // Genuinely no sources were retrieved (e.g. every search/scrape was blocked by
@@ -257,7 +407,13 @@ export class ResearchSynthesisService implements IService {
     // uncited report that looks sourced — but also never destroy a CITED LINKS block
     // the model may have written that we simply could not parse (preserve as-is).
     if (/CITED LINKS/i.test(synthesis)) {
-      return synthesis;
+      // Preserved, but NOT silently: reaching here means no citation parsed AND no
+      // scrape succeeded, so every link in that block is unverified by construction.
+      // Returning it untouched shipped a fully sourced-looking report with no caveat —
+      // the worst case for a reader, since an unretrievable session is precisely when
+      // a model's links are most likely to be invented.
+      logger.warn('[ResearchSynthesisService] Preserving an unparseable CITED LINKS block from a run that retrieved no sources — flagging it as unverified.');
+      return `${synthesis.trimEnd()}\n\n${UNVERIFIED_LINKS_NOTE}`;
     }
     logger.warn('[ResearchSynthesisService] No citations and no scrape provenance — appending explicit no-sources note.');
     return `${synthesis.trim()}\n\n${NO_SOURCES_NOTE}`;