@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/knowledge/transformers-loader.ts

@@ -0,0 +1,31 @@
+/**
+ * Lazy loader for the @huggingface/transformers module.
+ *
+ * Same rationale as ./lancedb-loader.ts. The embedder/knowledge stack is reached
+ * only behind a dynamic import() (infrastructure/service-initialization.ts), but
+ * esbuild bundles the CLI into a single ESM file with `packages:'external'` and no
+ * splitting, and in that mode it HOISTS every *static* `import ... from
+ * '@huggingface/transformers'` to the top of the bundle. transformers' Node entry
+ * eagerly dlopens the native onnxruntime-node binding at module load, so a hoisted
+ * static import evaluates that native load before main() runs — and on any platform
+ * where the binding is absent (Intel macOS / darwin-x64, for which onnxruntime-node
+ * ships no binary; musl/Alpine; a partial install) the whole bundle fails to
+ * evaluate and even `pi-research --help` / `status` exit 1.
+ *
+ * Loading transformers through a runtime import() keeps esbuild from hoisting it, so
+ * the native binding is required only on first real embedder use — where
+ * KnowledgeStoreService.initialize() already catches a missing native stack
+ * (isNativeStackUnavailableError) and degrades the store to DISABLED. Help, version,
+ * status, and browser-only research then work with no binding present.
+ *
+ * Modules that need transformers *types* should keep `import type { … } from
+ * '@huggingface/transformers'` (type imports are erased and never hoisted); only
+ * runtime value access (pipeline, env, …) must go through getTransformers().
+ */
+
+let _transformers: typeof import('@huggingface/transformers') | undefined;
+
+/** Resolve the transformers module, memoizing after the first load. */
+export async function getTransformers(): Promise<typeof import('@huggingface/transformers')> {
+  return (_transformers ??= await import('@huggingface/transformers'));
+}

package/src/knowledge/webgpu-probe.mjs

@@ -0,0 +1,80 @@
+/**
+ * Out-of-process WebGPU viability probe (child process entry point).
+ *
+ * WHY THIS EXISTS: onnxruntime-node bundles Dawn as its WebGPU execution
+ * provider. On a host whose only Vulkan adapter is software/paravirtual
+ * (llvmpipe/lavapipe, virtio/venus, SwiftShader, "Microsoft Basic Render
+ * Driver", ...), Dawn happily *loads* the pipeline but then SIGSEGVs the moment
+ * it dispatches the first compute shader. A SIGSEGV is a process-level signal,
+ * not a JavaScript exception — no try/catch, no Promise rejection handler, and
+ * none of the embedder's WebGPU->CPU fallback logic can ever observe it. The
+ * V8 isolate simply dies.
+ *
+ * The only reliable way to learn "will WebGPU compute crash on this machine?"
+ * is to actually run one tiny embedding in a DISPOSABLE child process and read
+ * its exit code. A native crash here kills only this child; the parent reads a
+ * non-zero exit / fatal signal and chooses CPU. This converts an uncatchable
+ * native crash into an ordinary, catchable signal.
+ *
+ * Contract — the parent judges viability by the STDOUT SENTINEL, not the exit
+ * code. This is deliberate: onnxruntime-node's native teardown can itself abort
+ * (SIGABRT, "Attempt to use DefaultLogger but none has been registered") when the
+ * process exits with a live ORT session — even after a perfectly successful
+ * compute. So:
+ *   argv[2] = model id (e.g. onnx-community/granite-embedding-small-english-r2-ONNX)
+ *   argv[3] = HuggingFace cache directory (shared with the parent)
+ *   prints "PROBE_OK <dims>" to STDOUT (written synchronously) => compute worked
+ *   no "PROBE_OK" on stdout (crash during compute, load failure, ...) => not viable
+ * A SIGSEGV during the compute dispatch (the software/paravirtual-GPU case) kills
+ * the child BEFORE the sentinel is written, so its absence is the signal.
+ *
+ * This file is plain .mjs (no TypeScript, no bundling). It is shipped verbatim
+ * next to the bundled entry points (dist/) and resolves @huggingface/transformers
+ * from the installed package's node_modules, exactly like the other entry points.
+ */
+
+import process from 'node:process';
+import { writeSync } from 'node:fs';
+
+async function main() {
+  const model = process.argv[2];
+  const cacheDir = process.argv[3];
+  if (!model || !cacheDir) {
+    process.stderr.write('webgpu-probe: missing model/cacheDir args\n');
+    process.exit(2);
+  }
+
+  const { pipeline, env } = await import('@huggingface/transformers');
+  env.cacheDir = cacheDir;
+  // Keep the probe quiet — we only care about the exit code, not Dawn chatter.
+  try {
+    if (env.backends && env.backends.onnx) env.backends.onnx.logLevel = 'error';
+  } catch {
+    /* best-effort */
+  }
+
+  // Load on the WebGPU EP exactly as the real embedder would, then run ONE embed.
+  // The crash (if any) happens on this first compute dispatch, after load.
+  const pipe = await pipeline('feature-extraction', model, {
+    device: 'webgpu',
+    session_options: { intraOpNumThreads: 2, interOpNumThreads: 1 },
+  });
+
+  const out = await pipe('webgpu viability probe', { pooling: 'mean', normalize: true });
+
+  // Touch the result so the compute cannot be optimized away.
+  if (out && out.data && out.data.length > 0) {
+    // Write the sentinel SYNCHRONOUSLY to fd 1 so it is guaranteed on the pipe
+    // before process teardown — which may itself abort the ORT native runtime.
+    // The parent keys off this sentinel, so a teardown abort here is harmless.
+    writeSync(1, 'PROBE_OK ' + out.data.length + '\n');
+    process.exit(0);
+  }
+  process.stderr.write('webgpu-probe: empty embedding output\n');
+  process.exit(3);
+}
+
+main().catch((err) => {
+  process.stderr.write('webgpu-probe error: ' + (err && err.message ? err.message : String(err)) + '\n');
+  process.exit(1);
+});

package/src/knowledge/webgpu-viability.ts

@@ -0,0 +1,255 @@
+/**
+ * WebGPU viability resolution for the embedder.
+ *
+ * Resolves the configured embedding device ('auto' | 'webgpu' | 'cpu') to a
+ * concrete backend the embedder can hand to onnxruntime-node WITHOUT risking a
+ * native SIGSEGV in the main process:
+ *
+ *   - 'cpu'    -> 'cpu'    (no probe; always safe)
+ *   - 'webgpu' -> 'webgpu' (explicit force/override; advanced — no probe)
+ *   - 'auto'   -> run an out-of-process probe (see webgpu-probe.mjs) ONCE, cache
+ *                 the verdict per host signature, then return 'webgpu' if WebGPU
+ *                 compute actually works here, else 'cpu'.
+ *
+ * The probe runs in a disposable child process precisely because the failure
+ * mode on software/paravirtual GPUs is an uncatchable native crash, not a JS
+ * error. See webgpu-probe.mjs for the full rationale.
+ */
+
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { existsSync } from 'node:fs';
+import * as fsp from 'node:fs/promises';
+import * as path from 'node:path';
+
+import { logger } from '../logger.ts';
+import { getModelCacheDir } from './embedder-utils.ts';
+import { isModelCached } from './embedder-init.ts';
+import { WEBGPU_PROBE_TIMEOUT_MS } from '../constants.ts';
+
+export type ResolvedDevice = 'webgpu' | 'cpu';
+
+// Bump when the probe semantics change in a way that should invalidate cached
+// verdicts (e.g. a different onnxruntime/Dawn version that fixes or breaks a path).
+const VERDICT_SCHEMA = 1;
+
+interface Verdict {
+  schema: number;
+  signature: string;
+  viable: boolean;
+  ts: number;
+}
+
+/**
+ * Host signature used as the verdict cache key. A change in platform, arch, Node
+ * major, or model invalidates the cached verdict. Driver-only changes are NOT
+ * captured (no portable way to read driver version without launching a GPU
+ * process); set PI_RESEARCH_WEBGPU_REPROBE=1 to force a fresh probe after a
+ * driver/GPU change.
+ */
+function hostSignature(model: string): string {
+  const nodeMajor = process.versions.node.split('.')[0];
+  return `${process.platform}-${process.arch}-node${nodeMajor}-${model}`;
+}
+
+function verdictFilePath(): string {
+  // One level above the model cache dir: ~/.cache/pi-research/webgpu-viability.json
+  return path.join(path.dirname(getModelCacheDir()), 'webgpu-viability.json');
+}
+
+async function readVerdict(signature: string): Promise<boolean | null> {
+  if (process.env['PI_RESEARCH_WEBGPU_REPROBE'] === '1') return null;
+  try {
+    const raw = await fsp.readFile(verdictFilePath(), 'utf8');
+    const v = JSON.parse(raw) as Verdict;
+    if (v && v.schema === VERDICT_SCHEMA && v.signature === signature && typeof v.viable === 'boolean') {
+      return v.viable;
+    }
+  } catch {
+    /* missing/corrupt -> treat as no verdict */
+  }
+  return null;
+}
+
+async function writeVerdict(signature: string, viable: boolean): Promise<void> {
+  try {
+    const file = verdictFilePath();
+    await fsp.mkdir(path.dirname(file), { recursive: true });
+    const v: Verdict = { schema: VERDICT_SCHEMA, signature, viable, ts: Date.now() };
+    await fsp.writeFile(file, JSON.stringify(v), 'utf8');
+  } catch (err) {
+    logger.debug('[embedder] Could not persist WebGPU viability verdict:', err);
+  }
+}
+
+/**
+ * Locate the probe child script. In dev it sits next to this module
+ * (src/knowledge/webgpu-probe.mjs); in a built install it is copied next to the
+ * bundled entry point (dist/webgpu-probe.mjs) — in both cases it is a sibling of
+ * the running module file (import.meta.url), because esbuild flattens this module
+ * into the dist entry bundle. Returns null if it cannot be found (fail-safe: the
+ * caller then chooses CPU rather than risking a crash).
+ */
+function probeScriptPath(): string | null {
+  try {
+    const here = path.dirname(fileURLToPath(import.meta.url));
+    const candidate = path.join(here, 'webgpu-probe.mjs');
+    return existsSync(candidate) ? candidate : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Run the out-of-process probe. Viability is judged by the STDOUT SENTINEL
+ * ("PROBE_OK"), NOT the exit code: onnxruntime-node's native teardown can abort
+ * the child (SIGABRT) on exit even after a fully successful compute, so a
+ * non-zero exit does not imply failure. The sentinel is written synchronously
+ * before exit; a real compute crash (SIGSEGV on a software GPU) kills the child
+ * before the sentinel is ever produced. Timeout, spawn failure, or missing
+ * script => false (the safe CPU direction). A timeout is treated as a negative
+ * (cached CPU) on purpose: GPU behaviour is deterministic per host, so a hang
+ * would otherwise recur — and re-stall — on every start; the 30s cap is ~10x a
+ * real cached-model WebGPU init, so a false timeout on a capable host is unlikely
+ * (and PI_RESEARCH_WEBGPU_REPROBE=1 recovers it). NEVER throws.
+ */
+function runProbe(model: string, cacheDir: string, timeoutMs: number): Promise<boolean> {
+  return new Promise<boolean>((resolve) => {
+    const script = probeScriptPath();
+    if (!script) {
+      logger.warn('[embedder] WebGPU probe script not found — defaulting to CPU (safe).');
+      resolve(false);
+      return;
+    }
+
+    let settled = false;
+    const finish = (viable: boolean) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(viable);
+    };
+
+    let child: ReturnType<typeof spawn>;
+    try {
+      child = spawn(process.execPath, [script, model, cacheDir], {
+        stdio: ['ignore', 'pipe', 'pipe'],
+        // Inherit env so XDG_CACHE_HOME etc. stay consistent with the parent.
+        env: process.env,
+      });
+    } catch (err) {
+      logger.warn('[embedder] WebGPU probe failed to spawn — defaulting to CPU:', err);
+      finish(false);
+      return;
+    }
+
+    const timer = setTimeout(() => {
+      logger.warn(`[embedder] WebGPU probe timed out after ${timeoutMs}ms — defaulting to CPU.`);
+      try { child.kill('SIGKILL'); } catch { /* already gone */ }
+      finish(false);
+    }, timeoutMs);
+    if (typeof timer.unref === 'function') timer.unref();
+
+    let stdout = '';
+    let stderr = '';
+    child.stdout?.on('data', (d) => { stdout += String(d); });
+    child.stderr?.on('data', (d) => { stderr += String(d); });
+
+    child.on('error', (err) => {
+      logger.warn('[embedder] WebGPU probe process error — defaulting to CPU:', err);
+      finish(false);
+    });
+
+    child.on('close', (code, signal) => {
+      // Viability is the sentinel, not the exit code (teardown may abort post-success).
+      if (stdout.includes('PROBE_OK')) {
+        finish(true);
+        return;
+      }
+      const detail = stderr.trim().split('\n').pop() ?? '';
+      const how = signal ? `signal ${signal}` : `exit ${code}`;
+      logger.info(`[embedder] WebGPU probe produced no success sentinel (${how}) — not viable here, using CPU.${detail ? ' (' + detail + ')' : ''}`);
+      finish(false);
+    });
+  });
+}
+
+/**
+ * In-process memoization of the running probe, keyed by host signature. Mirrors
+ * the embedding-factory `_embeddingInitPromise` singleton idiom: the probe AND its
+ * `writeVerdict` persist run inside ONE promise held at module scope. If the caller
+ * that started the probe is torn down before it settles (e.g. a health check losing
+ * a `Promise.race` timeout, or a store dispose), the module-level reference keeps
+ * the promise alive so `writeVerdict` still runs on the continuation — so the
+ * expensive probe is paid at most once per process instead of repeating every init.
+ * Bypassed under PI_RESEARCH_WEBGPU_REPROBE=1 (same escape hatch as readVerdict).
+ */
+let _probeInFlight: { signature: string; promise: Promise<boolean> } | null = null;
+
+/**
+ * Resolve the configured device to a concrete backend. See module header.
+ *
+ * @param requested  config EMBEDDING_DEVICE value ('auto' | 'webgpu' | 'cpu')
+ * @param model      embedding model id (passed to the probe)
+ * @param timeoutMs  caller's upper bound; the probe is additionally capped at
+ *                   WEBGPU_PROBE_TIMEOUT_MS (it runs only against an already-cached
+ *                   model, so it needs no model-download budget).
+ */
+export async function resolveEmbeddingDevice(
+  requested: string,
+  model: string,
+  timeoutMs: number,
+): Promise<ResolvedDevice> {
+  if (requested === 'cpu') return 'cpu';
+  if (requested === 'webgpu') return 'webgpu';
+
+  // 'auto' (and any unexpected value) -> probe-gated.
+  const signature = hostSignature(model);
+
+  const cached = await readVerdict(signature);
+  if (cached !== null) {
+    logger.info(`[embedder] WebGPU viability (cached verdict): ${cached ? 'viable -> webgpu' : 'not viable -> cpu'}`);
+    return cached ? 'webgpu' : 'cpu';
+  }
+
+  // The probe loads the model on the WebGPU EP to test a real compute. If the model
+  // is not cached yet, downloading it INSIDE the probe would block init on a
+  // first-ever run and can hang for minutes on a GPU-less host. Defer instead: use
+  // CPU now WITHOUT caching a verdict, so the model downloads on the ordinary CPU
+  // path and the NEXT init (model present) runs a fast, download-free probe for the
+  // real verdict. Keeps the probe bounded and never on a model-download critical path.
+  if (!(await isModelCached(model))) {
+    logger.info('[embedder] Embedding model not cached yet — using CPU for now; WebGPU will be probed once the model is present.');
+    return 'cpu';
+  }
+
+  const reprobe = process.env['PI_RESEARCH_WEBGPU_REPROBE'] === '1';
+  if (reprobe || _probeInFlight?.signature !== signature) {
+    // Model is cached here, so the probe only does a WebGPU init + tiny compute:
+    // bound it tightly (WEBGPU_PROBE_TIMEOUT_MS) so it fails fast to CPU on a
+    // GPU-less/hanging host instead of running out the caller's full budget.
+    const probeTimeoutMs = Math.min(WEBGPU_PROBE_TIMEOUT_MS, timeoutMs);
+    logger.info('[embedder] Probing WebGPU viability in a disposable child process...');
+    const cacheDir = getModelCacheDir();
+    // runProbe never throws and writeVerdict swallows its own errors, so this
+    // promise never rejects — no poisoned-memo concern. The verdict is written
+    // INSIDE the memoized promise so it persists even if the awaiting caller is torn
+    // down before the probe settles — the expensive probe is paid at most once.
+    _probeInFlight = {
+      signature,
+      promise: (async () => {
+        const v = await runProbe(model, cacheDir, probeTimeoutMs);
+        await writeVerdict(signature, v);
+        return v;
+      })(),
+    };
+  }
+
+  const viable = await _probeInFlight.promise;
+  logger.info(
+    viable
+      ? '[embedder] WebGPU probe succeeded — using WebGPU.'
+      : '[embedder] WebGPU probe failed/crashed — using CPU (safe). This is expected on VMs, containers, CI, and software-Vulkan hosts.',
+  );
+  return viable ? 'webgpu' : 'cpu';
+}

package/src/knowledge/writer-queue.ts

@@ -0,0 +1,252 @@
+import { normalizeUrl, validateUrl } from '../utils/url-utils.ts';
+import { isEmbedderUnreachable } from './embedder-utils.ts';
+import { logger } from '../logger.ts';
+import { metrics } from '../utils/metrics.ts';
+import type { Chunker } from './chunker.ts';
+import { createHash } from 'node:crypto';
+import { ServiceLifecycle } from '../core/service-registry.ts';
+import { ServiceNames } from '../core/service-interfaces.ts';
+import type { 
+  IWriterQueue, 
+  IngestionItem,
+  StoreDocument 
+} from '../core/interfaces/knowledge-interfaces.ts';
+
+interface WriterQueueOptions {
+  store: {
+    isStoreClosed?: () => boolean;
+    findByUrl: (url: string) => Promise<StoreDocument[]>;
+    deleteByUrlAndType: (url: string, type: string) => Promise<void>;
+    addDocuments: (docs: StoreDocument[]) => Promise<void>;
+  };
+  chunker: Chunker | null;
+}
+
+/** Bounded window for draining the writer queue during dispose/shutdown. Kept well
+ *  under the global SHUTDOWN_TIMEOUT_MS (8s) so LanceDB close and the browser orphan
+ *  sweep still get their share of the teardown budget. */
+const DRAIN_DISPOSE_TIMEOUT_MS = 5_000;
+
+function isNoSpace(err: unknown): boolean {
+  const code = (err as NodeJS.ErrnoException)?.code;
+  const msg = err instanceof Error ? err.message : String(err);
+  return code === 'ENOSPC' || msg.includes('ENOSPC') || msg.toLowerCase().includes('no space left');
+}
+
+export class WriterQueue implements IWriterQueue {
+  readonly name = ServiceNames.WRITER_QUEUE;
+  lifecycle = ServiceLifecycle.UNINITIALIZED;
+
+  private queue: IngestionItem[] = [];
+  private processing = false;
+  private options: WriterQueueOptions;
+  private drainResolvers: (() => void)[] = [];
+  // FIX (#2): Per-URL lock map to prevent TOCTOU races when concurrent writers
+  // ingest the same URL. Key is the normalized URL; value is the in-flight promise.
+  private inflightByUrl = new Map<string, Promise<void>>();
+
+  constructor(options: WriterQueueOptions) {
+    this.options = options;
+  }
+
+  async initialize(): Promise<void> {
+    this.lifecycle = ServiceLifecycle.INITIALIZED;
+  }
+
+  async dispose(): Promise<void> {
+    // Bound the drain so a large burst enqueued right before shutdown (or a hung
+    // embed) can't consume the whole global SHUTDOWN_TIMEOUT_MS budget — which is
+    // shared with LanceDB close and the browser orphan sweep. On timeout we proceed
+    // and account for the un-persisted items rather than hanging teardown.
+    await this.drain(DRAIN_DISPOSE_TIMEOUT_MS);
+    this.lifecycle = ServiceLifecycle.DISPOSED;
+  }
+
+  enqueue(item: IngestionItem): void {
+    this.queue.push(item);
+    this.process().catch(err => {
+      logger.error('[writer-queue] Error in process():', err);
+    });
+  }
+
+  private async process(): Promise<void> {
+    if (this.processing) return;
+    this.processing = true;
+
+    while (this.queue.length > 0) {
+      const item = this.queue.shift()!;
+      try {
+        // FIX (#2): Chain per-URL to serialize concurrent ingests of the same URL.
+        // process() already runs sequentially, so we chain only for the race case
+        // where two enqueue() calls trigger concurrent process() calls.
+        const urlKey = normalizeUrl(item.url);
+        const prev = this.inflightByUrl.get(urlKey) ?? Promise.resolve();
+        const inflight = prev.then(() => this._ingestInner(item));
+        this.inflightByUrl.set(urlKey, inflight);
+        try {
+          await inflight;
+        } finally {
+          if (this.inflightByUrl.get(urlKey) === inflight) {
+            this.inflightByUrl.delete(urlKey);
+          }
+        }
+      } catch (err) {
+        if (isEmbedderUnreachable(err)) {
+          logger.warn(`[writer-queue] Embedder unreachable for ${item.url}, retrying once after 2s...`);
+          await new Promise(r => setTimeout(r, 2000));
+          try {
+            await this.ingest(item);
+          } catch (retryErr) {
+            logger.error(`[writer-queue] Retry failed for ${item.url}, dropping:`, retryErr);
+            metrics.increment('knowledge_ingest_dropped_total', 1, { reason: 'embedder_unreachable' });
+          }
+        } else if (isNoSpace(err)) {
+          logger.error(`[writer-queue] ENOSPC: disk full — dropping ${item.url}. Free up disk space to resume knowledge ingestion.`);
+          metrics.increment('knowledge_ingest_dropped_total', 1, { reason: 'enospc' });
+        } else {
+          logger.error(`[writer-queue] Failed to ingest ${item.url}:`, err);
+          metrics.increment('knowledge_ingest_dropped_total', 1, { reason: 'ingest_error' });
+        }
+      }
+    }
+
+    this.processing = false;
+    
+    // Notify all drain callers
+    const resolvers = [...this.drainResolvers];
+    this.drainResolvers = [];
+    for (const resolve of resolvers) {
+      resolve();
+    }
+  }
+
+  private async ingest(item: IngestionItem): Promise<void> {
+    const urlKey = normalizeUrl(item.url);
+    const prev = this.inflightByUrl.get(urlKey) ?? Promise.resolve();
+    const inflight = prev.then(() => this._ingestInner(item));
+    this.inflightByUrl.set(urlKey, inflight);
+    try {
+      await inflight;
+    } finally {
+      if (this.inflightByUrl.get(urlKey) === inflight) {
+        this.inflightByUrl.delete(urlKey);
+      }
+    }
+  }
+
+  private async _ingestInner(item: IngestionItem): Promise<void> {
+    const incomingType = (item.metadata?.['ingestionType'] as string | undefined) ?? 'synthesis-description';
+
+    if (!item.markdown) {
+      logger.warn(`[writer-queue] Skipping ingest for ${item.url} — markdown is empty`);
+      return;
+    }
+
+    // FIX (Issue 3/4): Validate URL format to prevent storing hallucinated/malformed URLs
+    if (!validateUrl(item.url)) {
+      logger.warn(`[writer-queue] Skipping ingest — URL failed validation: ${item.url}`);
+      return;
+    }
+
+    // Normalize URL for consistent deduplication and storage.
+    // The original URL is preserved in metadata for reference, but all
+    // dedup checks and store operations use the normalized form.
+    const normalizedUrl = normalizeUrl(item.url);
+
+    const hash = createHash('sha256').update(item.markdown).update(item.content ?? '').digest('hex');
+
+    if (this.options.store.isStoreClosed?.()) {
+      logger.warn(`[writer-queue] Skipping ingest for ${normalizedUrl} — store is closing`);
+      return;
+    }
+
+    // FIX (Issue 5): Add provenance metadata to distinguish verified vs unverified entries
+    const hasContent = !!item.content;
+    const provenanceCategory = hasContent ? 'scraped-verified' : 'description-unverified';
+    const provenanceMeta = {
+      provenance: provenanceCategory,
+      hasContent,
+      validatedAt: Date.now(),
+    };
+
+    // Dedup: check if we already have this exact URL+type with identical content.
+    // findByUrl applies scope filtering, so this only dedupes within the
+    // current project's visible scope (local workspace + global entries).
+    const existing = await this.options.store.findByUrl(normalizedUrl);
+    const sameType = existing.filter(c => c.metadata['ingestionType'] === incomingType);
+    if (sameType.length > 0 && sameType[0]!.metadata['contentHash'] === hash) {
+      logger.log(`[writer-queue] Skipping ${normalizedUrl} (${incomingType}) — content unchanged.`);
+      return;
+    }
+
+    if (sameType.length > 0) {
+      await this.options.store.deleteByUrlAndType(normalizedUrl, incomingType);
+    }
+
+    const rawChunks = this.options.chunker
+      ? this.options.chunker.chunk(item.markdown)
+      : [{ text: item.markdown, actual_overlap: 0 }];
+
+    if (rawChunks.length === 0) return;
+
+    const docs: StoreDocument[] = rawChunks.map((chunk, i) => ({
+      url: normalizedUrl,
+      text: chunk.text,
+      content: i === 0 ? (item.content ?? undefined) : undefined,
+      metadata: {
+        ...(item.metadata || {}),
+        // Preserve the original (pre-normalization) URL for debugging/display
+        originalUrl: item.url !== normalizedUrl ? item.url : undefined,
+        contentHash: hash,
+        ingestionType: incomingType,
+        ...provenanceMeta,
+        chunkIndex: i,
+        totalChunks: rawChunks.length,
+      },
+      timestamp: Date.now(),
+    }));
+
+    await this.options.store.addDocuments(docs);
+  }
+
+  async drain(timeoutMs?: number): Promise<void> {
+    // FIX (#7): Atomically check state AND register the resolver inside the
+    // Promise constructor to prevent the race where process() finishes between
+    // the check and new Promise().
+    return new Promise<void>((resolve) => {
+      if (!this.processing && this.queue.length === 0) {
+        resolve();
+        return;
+      }
+      // Optional bounded drain: if the queue doesn't empty within timeoutMs, resolve
+      // anyway so a caller (shutdown) isn't held hostage by a slow/hung embed. Items
+      // still queued at the deadline are counted as dropped (best-effort — a few may
+      // still land as process() finishes, so the metric is a conservative upper bound).
+      // These are re-scrapable cache entries, not primary user data.
+      let settled = false;
+      let timer: ReturnType<typeof setTimeout> | undefined;
+      const done = () => {
+        if (settled) return;
+        settled = true;
+        if (timer) clearTimeout(timer);
+        resolve();
+      };
+      this.drainResolvers.push(done);
+      if (timeoutMs && timeoutMs > 0) {
+        timer = setTimeout(() => {
+          const remaining = this.queue.length + (this.processing ? 1 : 0);
+          logger.warn(`[writer-queue] drain timed out after ${timeoutMs}ms with ~${remaining} item(s) not yet persisted; proceeding with shutdown.`);
+          if (remaining > 0) {
+            metrics.increment('knowledge_ingest_dropped_total', remaining, { reason: 'drain_timeout' });
+          }
+          // Detach our resolver so a later process() completion doesn't call it again
+          // (harmless via the settled guard, but keeps drainResolvers from leaking).
+          const idx = this.drainResolvers.indexOf(done);
+          if (idx !== -1) this.drainResolvers.splice(idx, 1);
+          done();
+        }, timeoutMs);
+        timer.unref?.();
+      }
+    });
+  }
+}