@lincoln504/pi-research 0.1.12 → 1.0.0

package/dist/webgpu-probe.mjs

@@ -0,0 +1,80 @@
+/**
+ * Out-of-process WebGPU viability probe (child process entry point).
+ *
+ * WHY THIS EXISTS: onnxruntime-node bundles Dawn as its WebGPU execution
+ * provider. On a host whose only Vulkan adapter is software/paravirtual
+ * (llvmpipe/lavapipe, virtio/venus, SwiftShader, "Microsoft Basic Render
+ * Driver", ...), Dawn happily *loads* the pipeline but then SIGSEGVs the moment
+ * it dispatches the first compute shader. A SIGSEGV is a process-level signal,
+ * not a JavaScript exception — no try/catch, no Promise rejection handler, and
+ * none of the embedder's WebGPU->CPU fallback logic can ever observe it. The
+ * V8 isolate simply dies.
+ *
+ * The only reliable way to learn "will WebGPU compute crash on this machine?"
+ * is to actually run one tiny embedding in a DISPOSABLE child process and read
+ * its exit code. A native crash here kills only this child; the parent reads a
+ * non-zero exit / fatal signal and chooses CPU. This converts an uncatchable
+ * native crash into an ordinary, catchable signal.
+ *
+ * Contract — the parent judges viability by the STDOUT SENTINEL, not the exit
+ * code. This is deliberate: onnxruntime-node's native teardown can itself abort
+ * (SIGABRT, "Attempt to use DefaultLogger but none has been registered") when the
+ * process exits with a live ORT session — even after a perfectly successful
+ * compute. So:
+ *   argv[2] = model id (e.g. onnx-community/granite-embedding-small-english-r2-ONNX)
+ *   argv[3] = HuggingFace cache directory (shared with the parent)
+ *   prints "PROBE_OK <dims>" to STDOUT (written synchronously) => compute worked
+ *   no "PROBE_OK" on stdout (crash during compute, load failure, ...) => not viable
+ * A SIGSEGV during the compute dispatch (the software/paravirtual-GPU case) kills
+ * the child BEFORE the sentinel is written, so its absence is the signal.
+ *
+ * This file is plain .mjs (no TypeScript, no bundling). It is shipped verbatim
+ * next to the bundled entry points (dist/) and resolves @huggingface/transformers
+ * from the installed package's node_modules, exactly like the other entry points.
+ */
+
+import process from 'node:process';
+import { writeSync } from 'node:fs';
+
+async function main() {
+  const model = process.argv[2];
+  const cacheDir = process.argv[3];
+  if (!model || !cacheDir) {
+    process.stderr.write('webgpu-probe: missing model/cacheDir args\n');
+    process.exit(2);
+  }
+
+  const { pipeline, env } = await import('@huggingface/transformers');
+  env.cacheDir = cacheDir;
+  // Keep the probe quiet — we only care about the exit code, not Dawn chatter.
+  try {
+    if (env.backends && env.backends.onnx) env.backends.onnx.logLevel = 'error';
+  } catch {
+    /* best-effort */
+  }
+
+  // Load on the WebGPU EP exactly as the real embedder would, then run ONE embed.
+  // The crash (if any) happens on this first compute dispatch, after load.
+  const pipe = await pipeline('feature-extraction', model, {
+    device: 'webgpu',
+    session_options: { intraOpNumThreads: 2, interOpNumThreads: 1 },
+  });
+
+  const out = await pipe('webgpu viability probe', { pooling: 'mean', normalize: true });
+
+  // Touch the result so the compute cannot be optimized away.
+  if (out && out.data && out.data.length > 0) {
+    // Write the sentinel SYNCHRONOUSLY to fd 1 so it is guaranteed on the pipe
+    // before process teardown — which may itself abort the ORT native runtime.
+    // The parent keys off this sentinel, so a teardown abort here is harmless.
+    writeSync(1, 'PROBE_OK ' + out.data.length + '\n');
+    process.exit(0);
+  }
+  process.stderr.write('webgpu-probe: empty embedding output\n');
+  process.exit(3);
+}
+
+main().catch((err) => {
+  process.stderr.write('webgpu-probe error: ' + (err && err.message ? err.message : String(err)) + '\n');
+  process.exit(1);
+});

package/docs/AGENT-SKILL.md

@@ -0,0 +1,102 @@
+## Agent Skill
+
+pi-research ships as a portable [Agent Skill](https://agentskills.io/specification)
+so any skills-compatible coding agent (Claude, OpenAI Codex CLI, OpenClaw, and
+others using the same `SKILL.md` directory model) can run web research through the
+pi-research software.
+
+![Installing the research skill into external agents](media/05-agent-skill.gif)
+
+### How it works
+
+```
+agent
+  │  shells out (Bash / exec)
+  ▼
+run.mjs  —  zero-dep launcher (skills/pi-research/scripts/)
+  │  locates the installed engine, or fails fast with guidance
+  ▼
+pi-research engine  —  the SDK (dist/cli.mjs)
+  │  init → run → shutdown
+  ▼
+cited Markdown report  →  stdout  →  back to the agent
+```
+
+The agent matches the `description` in `SKILL.md` and shells out to the launcher.
+`run.mjs` carries no dependencies; it locates the installed engine
+(`PI_RESEARCH_BIN` pointing at the engine's `dist/cli.mjs`, then
+`PI_RESEARCH_PATH` pointing at its package dir, then PATH / `node_modules` /
+`~/.pi/bin`) and exits with an actionable
+message — including config-file locations — if the package, a model, or an API key
+is missing. It exposes four subcommands: `research "<query>"` (live research),
+`knowledge "<query>"` (search past findings), `knowledge-config [set <mode>]`
+(show/set the per-directory knowledge-store mode), and `status` (inspect detection/config).
+
+### Installation flow
+
+The skill source lives at `skills/pi-research/` inside the package. Installing means
+linking that directory into each agent's skills folder.
+
+One-click (recommended). From the `pi` extension, run `/research-config` →
+Install in External Agents. The installer:
+
+1. Detects which target agents are present under `$HOME` — currently Claude
+   (`~/.claude/skills`), OpenAI Codex CLI (`~/.codex/skills`), and OpenClaw
+   (`~/.openclaw/skills`, OpenClaw's managed skill root, which accepts symlinked
+   skill folders).
+2. Symlinks `skills/pi-research/` into each present agent, never overwriting an
+   unrelated skill already in that slot.
+3. Records what it created in a manifest, so Remove from External Agents
+   removes only its own links. Stale links are also garbage-collected on startup and
+   on `npm uninstall`.
+
+**OpenClaw without `pi`**
+
+If you run OpenClaw but not the `pi` extension, register the skill with OpenClaw's
+own CLI after installing the engine — OpenClaw copies the `SKILL.md` folder into its
+managed skill root and reads it on the next session:
+
+```bash
+npm install -g @lincoln504/pi-research
+openclaw skills install "$(npm root -g)/@lincoln504/pi-research/skills/pi-research" --global
+```
+
+OpenClaw drives the skill through its `exec` tool, gated on `node` being on `PATH`
+(`metadata.openclaw.requires.bins`). A bare `openclaw skills install git:…` is *not*
+used here: OpenClaw expects `SKILL.md` at a repo root, whereas this package nests it
+under `skills/pi-research/` — install the engine from npm and point OpenClaw at that
+folder, as above.
+
+Cursor is intentionally excluded from the one-click flow — it has no global skills
+directory and reads only project-level `.cursor/skills/`. Link it per project:
+
+```bash
+ln -s "$(npm root -g)/@lincoln504/pi-research/skills/pi-research" .cursor/skills/pi-research
+```
+
+Manual. Symlink the directory into any agent's skills folder yourself:
+
+| Agent | Personal | Project |
+|-------|----------|---------|
+| Claude | `~/.claude/skills/pi-research/` | `<project>/.claude/skills/pi-research/` |
+| OpenAI Codex CLI | `~/.codex/skills/pi-research/` | `<project>/.codex/skills/pi-research/` |
+| OpenClaw | `~/.openclaw/skills/pi-research/` | `<workspace>/skills/pi-research/` |
+| Cursor | — (no global dir) | `<project>/.cursor/skills/pi-research/` |
+
+### Prerequisites
+
+- Node.js >= 22.19.0
+- `pi-research` installed where the launcher can find it, plus a model + API key.
+  See [Configuration](CONFIGURATION.md).
+
+```bash
+npm install -g @lincoln504/pi-research
+node "<skill_dir>/scripts/run.mjs" status   # verify the engine is detected
+```
+
+![One-command health and readiness check](media/06-health-check.gif)
+
+Once installed, ask the agent to research something — its skill system activates
+pi-research automatically. The in-package readme (`skills/pi-research/README.md`) and
+`skills/pi-research/references/configuration.md` carry the same detail for anyone
+browsing the skill directly.

package/docs/ARCHITECTURE.md

@@ -1,109 +1,335 @@
-# Architecture
+## Architecture
 
-## Orchestration Modes
+pi-research is a pi TUI extension for multi-agent web research. It runs inside the pi
+process, registers its tools and commands, and manages its own browser worker pool,
+service registry, and local knowledge store. One engine backs every front-end: besides
+the pi extension it is exposed as a standalone CLI, a portable agent skill — the same
+skill any skills-aware host runs, including OpenClaw — and a programmatic SDK
+(`src/sdk.ts`).
 
-Two primary modes: **Quick Mode** and **Deep Mode**.
+```
+pi CLI
+└── pi-research extension (src/index.ts)
+    ├── Registered Tools   research, health, research_knowledge_search (always registered; reports why when the store is disabled)
+    ├── Commands           /research, /research-config, /knowledge-store
+    ├── Events             input (mid-run steering), session_shutdown (cleanup)
+    └── Layers
+        ├── Orchestration   quick/deep research coordination
+        ├── Agent Tools     search, scrape, youtube_transcript, security_search, stackexchange, grep, read
+        ├── Infrastructure  browser pool, knowledge store, state manager
+        └── Core            service registry, scheduler, health checks
+```
 
-### Quick Mode
-Routes research tasks to a single **Agent Researcher** session. No coordination or evaluation overhead — suitable for focused queries.
+1. A query enters through `runResearch` — the single internal entry point — with a depth.
+   Callers phrase the request in natural language: when the `research` tool is invoked
+   in-session, the calling agent picks the depth (1–3) from the user's wording and the
+   task's complexity, guided by the tool's usage prompt (`src/prompts/research-tool-usage.md`).
+   CLI and SDK callers pass the depth explicitly.
+2. Depth 0 takes the quick path; depth 1–3 takes the deep path (below). The TUI and agent
+   skill are restricted to levels 1–3.
+3. On the deep path the coordinator plans the research tracks and runs one initial
+   search burst, then hands each researcher a set of result URLs to start from.
+4. Researchers scrape and read those pages through the scrape tooling and return cited
+   reports. They consider only what they scraped this session.
+5. The evaluator reviews the round and either runs another round or synthesizes the
+   final report.
+6. The result is returned as a single cited Markdown report; the cited URLs and their
+   summaries are queued into the knowledge store for future runs.
 
-### Deep Mode
-Uses a state machine to manage a three-status research lifecycle:
+### Orchestration
 
-1.  **Planning Status**: The AI coordinator analyzes the conversation context and generates a JSON-formatted research agenda—a list of high-level research tasks.
-2.  **Researching Status**: 
-    - **Parallel Execution**: **Agent Researchers** run in parallel (up to 3 concurrent). Each researcher handles one agenda task per round.
-    - **Report Injection**: Collective knowledge is managed by the orchestrator. When a researcher completes its task, its full report (up to 50k chars) is **injected** into the context of siblings starting later, or **steered** into already running ones. This ensures researchers don't just know *what* was scraped, but also *what was found*.
-    - **Lead Evaluation**: When a round is complete, a **Lead Evaluator** agent call is triggered. It reviews all accumulated findings against the original agenda.
-    - **Decision Outcome**: The evaluator makes a binary choice:
-        - **Synthesize**: Generate a final, exhaustive Markdown report (transition to **Completed**).
-        - **Delegate**: Formulate new, targeted queries for a subsequent round (remain in **Researching**).
-3.  **Completed Status**: The final synthesis is returned as the tool result.
+`runResearch` (`IResearchOrchestration`, implemented in
+`src/orchestration/research-orchestration-service.ts`) is the single internal entry
+point. It dispatches on depth.
 
-State transitions are managed by a pure reducer (`src/orchestration/deep-research-reducer.ts`), ensuring predictable behavior.
+Depth 0 — quick (`QuickResearchOrchestrator`): a single researcher runs directly with
+all tools; there is no coordinator, no planning phase, and no rounds. Depth 0 is
+reachable only through the SDK (`runQuickResearch`) or the CLI (`--depth 0`, which the
+agent skill can pass). The pi extension's `research` tool has a minimum depth of 1, so
+an in-session agent can never request quick mode.
 
----
+Depth 1–3 — deep (`DeepResearchOrchestrator`): the run proceeds in **rounds**. A round
+is one coordinate → research → evaluate cycle — the round's agenda is planned (by the
+coordinator in round 1, by the evaluator thereafter), a batch of **researchers** runs it
+in parallel, and the evaluator then decides whether to run another round or synthesize.
+Two limits apply independently: how many researchers run *within* a round, and how many
+rounds the run may take.
 
-## The Agent Researcher
+| Depth | Label  | Researchers per round (max) | Rounds (max) |
+|-------|--------|-----------------------------|--------------|
+| 1     | normal | 2                           | 2            |
+| 2     | deep   | 3                           | 3            |
+| 3     | ultra  | 5                           | 3            |
 
-An **Agent Researcher** is the fundamental building block of `pi-research`. Regardless of the orchestration mode, every researcher follows a structured three-phase internal lifecycle:
+These are ceilings, not targets: the coordinator and evaluator use as many researchers
+and rounds as the topic needs. A depth-2 run, for example, may spawn up to 3 researchers
+in each of up to 3 rounds. Queued steering messages (Alt+Enter) can unlock a few extra
+rounds past the cap (`MAX_EXTRA_ROUNDS_WITH_STEERING`).
 
-### Phase 1: Gathering
-The researcher uses broad tools to discover relevant information and identify high-quality URLs for further investigation.
-- **Budget**: 4 Gathering calls (shared across `search`, `security_search`, `stackexchange`, and `grep`).
-- **Goal**: Breadth-first exploration and URL discovery.
+The coordinator also runs the initial search burst and distributes its result URLs to
+round 1's researchers (`distributeSearchResults`), so in deep mode the researchers
+themselves do not call `search`.
 
-### Phase 2: Scraping
-The researcher deep-dives into the URLs identified during the Gathering phase to extract detailed content.
-- **Budget**: 4 Scrape calls (1 handshake + 3 execution batches).
-- **Goal**: Depth-first extraction of specific data and evidence.
-- **Deduplication**: Researchers check a shared URL pool to avoid re-scraping links already handled by siblings in the same operation.
+LLM-call conventions. Coordinator, evaluator, synthesis, JSON-repair, and
+knowledge-extraction calls go through `completeSimple` (`src/core/llm/pi-ai-completion.ts`)
+with `buildSafeOptions` (`src/core/llm/llm-utils.ts`); researcher sub-agents go through
+`createAgentSession`.
+Two conventions apply:
 
-### Phase 3: Reporting
-The researcher synthesizes its individual findings into a structured Markdown report.
-- **Structure**: Includes an executive summary, key findings, and a list of cited links.
-- **Signaling**: Once finished, the report is submitted to the orchestrator for injection into siblings (Deep Mode) or final return (Quick Mode).
+- Thinking is off by default. These calls emit structured JSON or cited reports, so a
+  chain-of-thought block only spends output-token budget (and can truncate the answer).
+  `PI_RESEARCH_LLM_THINKING_LEVEL` (default `off`) controls it, clamped per provider.
+- Output budgets are sized per role and clamped to the model's ceiling:
+  `PLANNING_MAX_TOKENS` for the plan/decision, `SYNTHESIS_MAX_TOKENS` for the final
+  report. A mid-round evaluation that cannot be parsed continues the existing agenda
+  rather than finalizing early, so a parse failure never truncates a run.
 
----
+### Research tools
 
-## Component Details
+Each researcher has a fixed tool set and a shared budget of 12 gathering calls per phase
+(`MAX_GATHERING_CALLS`).
 
-### Research Tools & Limits
-Each researcher session is constrained by a `ToolUsageTracker` to prevent infinite loops and manage token consumption:
-- **search**: Interfaces with the local SearXNG instance. Supports multiple queries per call.
-- **scrape**: Multi-batch protocol for page extraction.
-- **security_search**: Queries NVD, CISA KEV, GitHub Advisories, and OSV.
-- **stackexchange**: Queries the Stack Exchange network (Stack Overflow, etc.).
-- **grep**: Local codebase search using `ripgrep` (fallback to `grep`).
+| Tool | Quick | Deep | Backend |
+|------|-------|------|---------|
+| `search` | ✓ | — | DuckDuckGo Lite via the stealth browser |
+| `scrape` | ✓ | ✓ | Batch page fetch → Markdown via the stealth browser (up to 6 URLs per call) |
+| `youtube_transcript` | ✓ | ✓ | YouTube captions via youtubei.js + BotGuard PoToken (≤3 videos, one call per researcher) |
+| `security_search` | ✓ | ✓ | NVD, CISA KEV, GitHub Advisories, OSV |
+| `stackexchange` | ✓ | ✓ | Stack Exchange network |
+| `grep` | — | ✓ | Local ripgrep (from pi-coding-agent) |
+| `read` | ✓ | ✓ | Local file reads (from pi-coding-agent) |
 
-### Context-Aware Scraping Protocol
-To avoid exceeding LLM context windows, the `scrape` tool follows a four-step handshake. **Each batch step is gated by a 55% context check** — if the session token count already exceeds 55% of the model's context window, that batch is skipped entirely and the tool returns early.
+In deep research `search` is excluded — the coordinator runs the search burst and hands
+out URLs directly. In quick research `grep` is excluded — the single session is not
+expected to traverse a local codebase. Researchers cannot write files, run shell
+commands, or reach the network outside these tools.
 
-1. **Handshake**: The researcher identifies target URLs. The tool returns a list of URLs already scraped by siblings. No network activity; no context check.
-2. **Batch 1**: Up to 3 URLs — primary broad scraping.
-3. **Batch 2**: Up to 2 URLs — targeted follow-up, deduplicated against Batch 1.
-4. **Batch 3**: Up to 3 URLs — deep-dive scraping.
+### Browser infrastructure
 
-### Shared Link Deduplication
-Researchers share a URL pool per research operation, used by the `scrape` handshake to avoid redundant network requests. The pool tracks URLs only — findings are shared separately via Report Injection. It resets for each new query.
+All browser work (search, scrape, health checks) goes through a poolifier
+`FixedClusterPool` of worker processes — each a Node.js child process running its own
+camoufox (stealth Firefox) instance. Isolating the browser in workers means a crash in
+one worker cannot take down the orchestrator or other sessions.
 
-### SearXNG Lifecycle Management
-SearXNG runs as a Docker container (`searxng/searxng`, tag controlled by `SEARXNG_IMAGE_TAG`, default `latest`).
-- **Lazy Initialization**: The container starts only upon the first research request.
-- **Singleton Pattern**: A single container is shared across all pi processes on the machine. State is persisted to `~/.pi/state/searxng-singleton.json`; each pi process registers a session by PID and the container stays alive until all registered processes have exited.
-- **Configuration**: Runtime settings (proxies, API keys) are generated dynamically and volume-mounted into the container.
-- **Engine configuration**: Active search engines are defined in `config/searxng/default-settings.yml`. Engines can be enabled or disabled by editing that file before starting the container.
-- **Cross-Session Persistence**: Container state and locking are managed via `~/.pi/state/searxng-singleton.json` to handle multiple concurrent pi processes.
+```
+BrowserTaskScheduler
+└── FixedClusterPool (poolifier)
+    ├── Worker 1  →  camoufox instance
+    ├── Worker 2  →  camoufox instance
+    └── Worker N  →  camoufox instance
+```
 
-### Output File Location
+Key files:
+- `src/infrastructure/browser/browser-task-scheduler.ts` — dispatches tasks to the pool
+- `src/infrastructure/browser/thread-worker.ts` — worker entry point (bundled separately by esbuild)
+- `src/infrastructure/browser/thread-worker-messaging.ts` — IPC protocol
+- `src/infrastructure/browser/config.ts` — pool configuration, binary path detection
 
-When research completes, the report is written to a `.md` file named `pi-research-{sanitized-query}-{hash}.md` (e.g. `pi-research-rust-async-runtimes-a3.md`). The destination directory is resolved from the pi session's working directory (`cwd`) using a three-tier priority:
+### Knowledge store and data handling
 
-1. **cwd is home or a system directory** → `os.tmpdir()`. Prevents cluttering `~`, drive roots, or system paths like `/usr`.
-2. **cwd contains a recognised subdirectory** → that subdirectory. Probed in priority order: `research`, `docs`, `doc`, `ref`, `references`, `notes`. Matching is case-insensitive so it works on macOS APFS and Windows NTFS.
-3. **Otherwise** → cwd itself.
+The knowledge store is a local LanceDB vector table of past findings. It is optional
+(research works without it) and is driven entirely by the orchestrator — researchers
+never call it directly:
 
-The two-character hash suffix (`{letter}{digit}`, e.g. `b7`) is a collision guard — the file is opened with an exclusive-create flag and the hash is regenerated on `EEXIST`. The full saved path is appended as a footer in the report and returned to the pi UI.
+- Before each researcher starts, the orchestrator searches the store for that
+  researcher's goal and injects any matching historical URLs (with summaries) into its
+  prompt as starting points.
+- After a run, the cited URLs and their descriptions are enqueued into the async writer
+  queue and stored in the background — writes never block a run.
 
-### TUI Progress Percentage
+On ingest, each source's summary and full scraped Markdown are split into chunks and
+embedded into vectors. A SHA-256 content hash of the page dedupes re-ingested URLs:
+an unchanged page is skipped, a changed page replaces its old rows. Each row carries the
+vector, the normalized URL, the text and full content, a timestamp, and scope flags
+(project vs. global) that are filtered at query time.
 
-The header percentage (`Research: 70%`) tracks tool-call units completed against a budget established after planning. The budget is set from the number of agenda items the coordinator produces — which scales with query complexity (complexity 1 → 2 initial researchers, 2 → 3, 3 → 5), each researcher allocated 10 units. The budget expands dynamically when the evaluator delegates additional rounds. If research goes beyond the planned rounds the display switches to `exploring` instead of a percentage.
+```
+WriterQueue (async, non-blocking)
+└── KnowledgeStore
+    ├── Embedder  (onnx-community/granite-embedding-small-english-r2-ONNX via @huggingface/transformers)
+    │   └── backend: auto (out-of-process WebGPU probe → webgpu or cpu) / webgpu / cpu
+    └── LanceDB   (knowledge_db/ directory, Arrow-backed vector table)
+```
 
-### Evaluator Decision Framework
-The Lead Evaluator follows a strict priority list:
-1.  **Error Check**: If all researchers in a round failed, it reports a critical failure and stops.
-2.  **Coverage Check**: It compares cumulative findings against the initial agenda items.
-3.  **Round Budget**: It checks the round budget (`targetRounds + MAX_EXTRA_ROUNDS`). The hard limit is `targetRounds + 2` bonus rounds.
-4.  **Action Selection**: Produces either a Markdown synthesis (final result) or a JSON delegation object (`{"action": "delegate", "queries": [...]}`). Any non-JSON response is treated as a final synthesis to prevent the research from getting stuck.
+Key files: `src/knowledge/store.ts` (LanceDB operations), `embedder.ts` (model load +
+batched inference), `writer-queue.ts` (async writes + content-hash dedup), `chunker.ts`
+(chunking), `webgpu-viability.ts` (out-of-process GPU probe + cached verdict),
+`migration.ts` (model-change migration: drop / backup / re-embed).
 
----
+The store needs native ONNX-runtime and LanceDB bindings. On platforms with no prebuilt
+binary — notably Intel macOS (`darwin-x64`) — it is absent: the health check reports it
+disabled-but-healthy and research runs without the cache. See
+[KNOWLEDGE-STORE.md](KNOWLEDGE-STORE.md) for the full subsystem and platform matrix.
 
-## Project Structure
+### Services and lifecycle
 
-- `src/orchestration/`: Core state machine, coordinator, and evaluator logic.
-- `src/infrastructure/`: Docker container management and SearXNG lifecycle.
-- `src/tools/`: Implementations for search, scraping, and specialized database queries.
-- `src/tui/`: Real-time terminal UI components using `pi`'s display capabilities.
-- `src/utils/`: Shared utilities for token tracking, JSON extraction, and link deduplication.
-- `src/prompts/`: Markdown-based system instructions for the Coordinator, Researcher, and Evaluator.
-- `config/`: Default SearXNG engine and rate-limiter configurations.
+Services are registered with async factory functions and resolved through a registry
+(`getService()`), initialized lazily or eagerly with dependencies wired at init time.
+
+```typescript
+registerService(ServiceNames.FOO, async () => {
+  const dep = await getService<IBar>(ServiceNames.BAR);
+  return new FooService(dep);
+}, { lazyInitialization: true });
+
+const foo = await getService<IFoo>(ServiceNames.FOO);
+```
+
+Services that hold resources implement `dispose()`; the registry disposes them in
+reverse dependency order. Resolving through the registry (rather than direct imports)
+enforces lifecycle discipline (init → use → dispose) and lets tests swap in mocks.
+
+- Core (`src/core/`): `PlanningService`, `SchedulerService`, `HealthCheckService`
+- Infrastructure (`src/infrastructure/`): `StateManagerService`, `KnowledgeStoreService`, `MetricsService`, `WorkerPoolManager`, `FileLockService`, `GPUResourceService` (plus `WriterQueue`, defined in `src/knowledge/` and registered here)
+- Orchestration (`src/orchestration/`): `ResearchOrchestrationService`, `ResearchSessionService`, `ResearchSynthesisService`
+
+Cross-session, cross-process state (active sessions, browser status, metrics) lives in
+`StateManagerService` (`src/infrastructure/state/`), which serializes concurrent writes
+with file-based locking (`FileLockService`).
+
+### TUI
+
+The live progress panel uses `@earendil-works/pi-tui`, which handles terminal state
+(keyboard protocol, mouse tracking, bracketed paste). stdio capture (to keep stray
+output from corrupting the panel and to guarantee a clean exit) lives in
+`src/utils/stdio-capture.ts`.
+
+### Project structure
+
+```
+src/
+├── index.ts              extension entry point (tools, commands, events, lifecycle)
+├── cli.ts                standalone CLI entry point
+├── sdk.ts                programmatic SDK (non-extension use)
+├── config.ts             env-var parsing, validation, singleton
+├── constants.ts          team sizes, round caps, tool budgets, batch limits
+├── logger.ts             structured logger (JSONL, TUI-safe)
+├── tool.ts               re-export barrel for the research + health tool definitions
+├── research-config.ts    /research-config TUI
+├── core/
+│   ├── llm/              prompts, model resolution, agentic JSON repair, inject-date
+│   ├── interfaces/       abstraction contracts (observer, planning, orchestration)
+│   ├── planning-service.ts, scheduler-service.ts
+│   ├── service-registry.ts, service-interfaces.ts, service-initialization.ts
+│   └── planning-utils.ts
+├── infrastructure/
+│   ├── browser/          worker pool, task scheduler, IPC, camoufox config
+│   ├── state/            state manager, session tracking, metrics collector
+│   ├── embedding/        local embedding server management
+│   ├── knowledge-store-service.ts, metrics-service.ts, file-lock-service.ts
+│   └── process-lifecycle-service.ts
+├── orchestration/
+│   ├── deep-research-orchestrator.ts, quick-research-orchestrator.ts
+│   ├── research-orchestration-service.ts, research-synthesis-service.ts
+│   ├── research-session-service.ts, session-state.ts, session-context.ts
+│   ├── researcher-executor.ts, researcher.ts, headless-observer.ts
+├── prompts/              Markdown prompt templates for all agents
+├── tools/                search, scrape, youtube_transcript, security, stackexchange, grep, read, knowledge-search
+├── knowledge/            embedder, store, writer queue, chunker, migration, webgpu probe
+├── web-research/         DuckDuckGo search, scraper, retry logic
+├── security/             NVD, CISA KEV, OSV, GitHub Advisory clients
+├── stackexchange/        Stack Exchange API client
+├── youtube/              YouTube transcript client (InnerTube + BotGuard PoToken)
+├── skill-install/        research-skill installer for coding-agent harnesses
+├── tui/                  panels, layout, controller, wave animation, terminal utils
+├── healthcheck/          health-check registry and checks
+├── cleanup/              research-result cleanup
+├── observers/            research-observer implementation
+├── types/                shared index and TUI types
+└── utils/                circuit breaker, text utils, shared-links, metrics, error tracking
+```
+
+### Key design decisions
+
+Sandboxed researchers — researcher agents are limited to the tool set above. They cannot
+write files, spawn processes, or make arbitrary network calls.
+
+Worker pool over direct browser — browser processes are isolated in workers so a crash
+in one cannot affect the orchestrator or other sessions.
+
+Pinned browser stack — `playwright-core` and `impit` are pinned to exact versions and
+`camoufox-js` to its `0.10.x` line (`^0.10.2`, which excludes the `0.11.x` alpha); the
+three are coupled and upgraded together, because each floating range broke fresh consumer
+installs that our lockfile masked. camoufox-js `0.10.2` fetches Firefox
+135/beta.24, the newest camoufox with binaries for every supported OS (later builds
+dropped Windows). playwright-core `1.60.0` is the newest the FF135 Juggler protocol
+accepts (1.61 rejects it and fails every launch). impit `0.13.0` avoids the
+`only-allow pnpm` preinstall guard that impit 0.13.1/0.14.0 shipped, which fails
+`npm install -g`; a direct-dependency pin is required because npm `overrides` do not
+propagate to consumers. Will upgrade all three together when camoufox ships stable
+cross-platform binaries. Rationale in full:
+`src/infrastructure/browser/thread-worker-browser.ts`.
+
+Pinned data stack — `apache-arrow` is a direct dependency at `21.1.0`, and `overrides`
+forces the whole tree to that single version so LanceDB and Arrow share one Arrow instance
+(mismatched Arrow copies do not interoperate — arrays built by one are rejected by the
+other). This sits above `@lancedb/lancedb` 0.29's declared Arrow peer ceiling
+(`>=15.0.0 <=18.1.0`); it is verified working, but the override should be re-validated
+whenever `@lancedb/lancedb` is upgraded.
+
+Pinned validation library — `typebox` is pinned to the exact version the pi host packages
+depend on (`@earendil-works/pi-ai` / `@earendil-works/pi-coding-agent` pin `1.1.38`). Every
+tool's parameter schema is built with TypeBox here and handed across the boundary to pi's
+tool system, so the two must agree on `Value.Check`/`Convert` semantics. A floating `^1.1.38`
+range let a fresh consumer install resolve pi-research to a newer TypeBox than pi's, shipping
+an untested cross-version pairing; the exact pin keeps pi-research on the same version pi
+validates with. Bump it in lockstep with the pi host, not independently. (`undici`, by
+contrast, tracks the host's major — the host is on undici 8, and pi-research only uses the
+stable `Agent` connector API, so it follows `^8`.)
+
+Transient-failure resilience — every LLM call is a potential single point of failure on a
+streaming endpoint that can drop mid-response (undici surfaces this as `terminated`). The
+coordinator and evaluator calls retry fast transient transport failures (socket aborts, 5xx,
+429, provider overload) with bounded exponential backoff — mirroring the per-researcher retry
+(`PI_RESEARCH_MAX_RETRIES`) — and, if still failing, degrade to a deterministic fallback plan
+rather than aborting the run. An app-level LLM timeout is not retried (it already spent the
+full budget); it degrades directly. Retry counts are internal constants, not configuration.
+
+Registry over direct imports — services are registered and resolved through the registry
+to support testing (mock replacement) and enforce init → use → dispose lifecycle.
+
+Pure ESM — the codebase is ES Modules (`"type": "module"`). Worker bundles are built with
+esbuild (`npm run build:worker`) before integration tests or publishing.
+
+Enforced boundaries — `docs/deps.svg` is regenerated on every push (madge), and
+architectural rules are enforced by dependency-cruiser
+(`config/tooling/dependency-cruiser.cjs`).
+
+### Built with
+
+Browser & scraping
+
+- [Camoufox](https://camoufox.com) — stealth Firefox (driven via [Playwright](https://playwright.dev)) for undetected search and scraping
+- [poolifier](https://github.com/poolifier/poolifier) — the worker-process pool behind the browser workers
+- [html-to-markdown](https://github.com/Goldziher/html-to-markdown) & [node-html-markdown](https://github.com/crosstype/node-html-markdown) — convert scraped HTML to Markdown
+- `pdf-oxide-wasm` — PDF text extraction (Rust/WASM)
+
+Knowledge store & embeddings
+
+- [Transformers.js](https://github.com/huggingface/transformers.js) — local embedding inference (model execution via ONNX Runtime)
+- Google [Dawn](https://dawn.googlesource.com/dawn) — the WebGPU backend, accessed through the `webgpu` Node binding
+- [LanceDB](https://lancedb.com) — on-disk vector database
+- [Apache Arrow](https://arrow.apache.org) — the columnar schema the vector table is built on
+
+YouTube transcripts
+
+- [youtubei.js](https://github.com/LuanRT/YouTube.js) — YouTube internal-API client
+- [BgUtils](https://github.com/LuanRT/BgUtils) — BotGuard PoToken generation
+- [jsdom](https://github.com/jsdom/jsdom) — DOM environment for minting the PoToken
+
+Host & runtime
+
+- [pi](https://github.com/earendil-works/pi) — the host runtime, agent SDK, and TUI toolkit
+- [TypeBox](https://github.com/sinclairzx81/typebox) — runtime config schema and validation
+
+### Development
+
+```bash
+npm run test:unit         # unit tests, no browser required
+npm run test:integration  # requires camoufox (Xvfb only for the opt-in virtual-display tests)
+npm run type-check        # TypeScript strict mode (src)
+npm run type-check:tests  # TypeScript strict mode (tests)
+npm run lint              # ESLint
+npm run deps:check        # architectural rule enforcement
+npm run build:worker      # bundle the browser worker (required before integration tests / publish)
+```