@apmantza/greedysearch-pi 2.1.3 → 2.1.5

package/docs/banner.svg

@@ -0,0 +1,86 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 480" width="1200" height="480">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0d1117"/>
+      <stop offset="50%" style="stop-color:#161b22"/>
+      <stop offset="100%" style="stop-color:#0d1117"/>
+    </linearGradient>
+
+    <linearGradient id="title" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" style="stop-color:#58a6ff"/>
+      <stop offset="50%" style="stop-color:#bc8cff"/>
+      <stop offset="100%" style="stop-color:#3fb950"/>
+    </linearGradient>
+
+    <linearGradient id="piGrad" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#8b5cf6"/>
+      <stop offset="100%" style="stop-color:#a78bfa"/>
+    </linearGradient>
+
+    <filter id="glow" x="-20%" y="-20%" width="140%" height="140%">
+      <feGaussianBlur stdDeviation="6" result="blur"/>
+      <feComposite in="SourceGraphic" in2="blur" operator="over"/>
+    </filter>
+
+    <pattern id="dots" width="20" height="20" patternUnits="userSpaceOnUse">
+      <circle cx="2" cy="2" r="0.8" fill="#30363d" opacity="0.4"/>
+    </pattern>
+  </defs>
+
+  <!-- Background -->
+  <rect width="1200" height="480" fill="url(#bg)"/>
+  <rect width="1200" height="480" fill="url(#dots)"/>
+
+  <circle cx="1050" cy="100" r="300" fill="#bc8cff" opacity="0.03"/>
+  <circle cx="1150" cy="400" r="200" fill="#3fb950" opacity="0.03"/>
+
+  <!-- ===== PI LOGO (left) ===== -->
+  <g transform="translate(60, 140)">
+    <circle cx="60" cy="60" r="72" fill="none" stroke="#8b5cf6" stroke-width="2" opacity="0.3"/>
+    <circle cx="60" cy="60" r="62" fill="none" stroke="#8b5cf6" stroke-width="1" opacity="0.1"/>
+    <g transform="translate(18, 18) scale(2.6)">
+      <path fill="#8b5cf6" fill-rule="evenodd" d="M0 0H15V10H10V15H5V20H0ZM5 5V10H10V5Z"/>
+      <rect x="15" y="10" width="5" height="10" fill="#8b5cf6"/>
+    </g>
+  </g>
+
+  <!-- ===== TITLE ===== -->
+  <text x="240" y="195" font-family="'Segoe UI', -apple-system, system-ui, sans-serif" font-size="72" font-weight="700" letter-spacing="-1.5" fill="url(#title)" filter="url(#glow)">
+    GreedySearch
+  </text>
+
+  <!-- ===== "No API keys" badge ===== -->
+  <g transform="translate(240, 218)">
+    <rect x="0" y="0" width="155" height="28" rx="14" fill="#1f6feb" opacity="0.15"/>
+    <rect x="0" y="0" width="155" height="28" rx="14" fill="none" stroke="#1f6feb" stroke-width="1" opacity="0.3"/>
+    <text x="77" y="19" font-family="'Segoe UI', -apple-system, system-ui, sans-serif" font-size="13" font-weight="600" fill="#58a6ff" text-anchor="middle">🔑 No API keys</text>
+  </g>
+
+  <!-- ===== TAGLINE ===== -->
+  <text x="240" y="280" font-family="'Segoe UI', -apple-system, system-ui, sans-serif" font-size="20" font-weight="400" fill="#8b949e" letter-spacing="0.3">
+    Multi-engine AI search via browser automation.
+  </text>
+
+  <!-- ===== ENGINE PILLS ===== -->
+  <g transform="translate(240, 310)">
+    <!-- Perplexity -->
+    <rect x="0" y="0" width="115" height="30" rx="15" fill="#00c8b4" opacity="0.12"/>
+    <rect x="0" y="0" width="115" height="30" rx="15" fill="none" stroke="#00c8b4" stroke-width="1" opacity="0.3"/>
+    <text x="57" y="20" font-family="'Segoe UI', -apple-system, system-ui, sans-serif" font-size="13" font-weight="600" fill="#00c8b4" text-anchor="middle">Perplexity</text>
+
+    <!-- Bing -->
+    <rect x="130" y="0" width="115" height="30" rx="15" fill="#00a4ef" opacity="0.12"/>
+    <rect x="130" y="0" width="115" height="30" rx="15" fill="none" stroke="#00a4ef" stroke-width="1" opacity="0.3"/>
+    <text x="187" y="20" font-family="'Segoe UI', -apple-system, system-ui, sans-serif" font-size="13" font-weight="600" fill="#00a4ef" text-anchor="middle">Bing Copilot</text>
+
+    <!-- Google AI -->
+    <rect x="260" y="0" width="115" height="30" rx="15" fill="#fbbc04" opacity="0.12"/>
+    <rect x="260" y="0" width="115" height="30" rx="15" fill="none" stroke="#fbbc04" stroke-width="1" opacity="0.3"/>
+    <text x="317" y="20" font-family="'Segoe UI', -apple-system, system-ui, sans-serif" font-size="13" font-weight="600" fill="#fbbc04" text-anchor="middle">Google AI</text>
+
+    <!-- Gemini -->
+    <rect x="390" y="0" width="100" height="30" rx="15" fill="#8b5cf6" opacity="0.12"/>
+    <rect x="390" y="0" width="100" height="30" rx="15" fill="none" stroke="#8b5cf6" stroke-width="1" opacity="0.3"/>
+    <text x="440" y="20" font-family="'Segoe UI', -apple-system, system-ui, sans-serif" font-size="13" font-weight="600" fill="#a78bfa" text-anchor="middle">Gemini</text>
+  </g>
+</svg>

package/docs/development.md

@@ -0,0 +1,63 @@
+# Development
+
+## Project Layout
+
+- `index.ts` — Pi extension entrypoint.
+- `src/tools/greedy-search-handler.ts` — Pi tool registration/handler.
+- `bin/search.mjs` — CLI orchestration.
+- `extractors/` — engine-specific browser automation.
+- `src/search/` — search pipeline, Chrome lifecycle, recovery, synthesis,
+  source ranking, and research orchestration.
+- `test.mjs`, `test-suite/`, `test/` — unit and smoke tests.
+
+Pi loads this extension through `jiti`; TypeScript does not need to be
+precompiled for Pi runtime.
+
+## Checks
+
+```bash
+npm run check:lockfile
+npm run lint
+node test.mjs unit
+npm pack --dry-run --json
+```
+
+Jiti load check:
+
+```bash
+node - <<'NODE'
+import { createJiti } from 'file:///C:/Users/R3LiC/AppData/Roaming/npm/node_modules/@earendil-works/pi-coding-agent/node_modules/jiti/lib/jiti.mjs';
+const jiti = createJiti(import.meta.url, { interopDefault: true });
+const mod = await jiti.import('./index.ts');
+console.log('jiti ok', typeof mod.default);
+NODE
+```
+
+## Useful Headless Smoke Checks
+
+```bash
+node bin/launch.mjs --kill || node bin/kill-visible.mjs
+node bin/search.mjs all --inline --stdin --full <<'EOF'
+TypeScript 5.8 5.9 Node.js ESM module-resolution changes
+EOF
+
+node bin/search.mjs all --inline --stdin --depth research \
+  --breadth 1 --iterations 1 --max-sources 3 <<'EOF'
+Node.js native TypeScript type stripping for CLI authors
+EOF
+```
+
+## Extractor Notes
+
+When adding or changing an extractor:
+
+1. Reuse `extractors/common.mjs` utilities.
+2. Prefer single in-browser polling evals over Node-side CDP polling loops.
+3. Use language-agnostic selectors and data attributes where possible.
+4. Avoid matching English UI strings except as a last resort.
+5. Register new engines in both `src/search/constants.mjs` and `bin/search.mjs`
+   when they need all-mode pre-seeding.
+6. Update `README.md`, `docs/`, `src/tools/greedy-search-handler.ts`, and
+   `CHANGELOG.md`.
+
+See [`AGENTS.md`](../AGENTS.md) for the full extractor and recovery guide.

package/docs/inspiration2.md

@@ -0,0 +1,190 @@
+# Inspiration: Browser Automation Patterns from the Ecosystem
+
+This document captures patterns and ideas from two open-source browser automation
+projects that could inform future improvements to GreedySearch-pi. Kameleo is
+omitted — its anti-detection engine is commercial and C++-level, not applicable
+to our JS-level CDP approach.
+
+---
+
+## 1. [chrome-devtools-axi](https://github.com/kunchenguid/chrome-devtools-axi)
+
+A wrapper around `chrome-devtools-mcp` (Google's official MCP server) with TOON
+encoding, accessibility tree snapshots, contextual suggestions, and a persistent
+bridge architecture. Designed as an Agent Skill (AXI-compliant).
+
+### Notable patterns
+
+#### Generation-based stale ref detection
+
+Every accessibility tree snapshot carries a `g<N>:` generation prefix that bumps
+on each capture. When an agent passes a stale ref back (e.g. `@g1:5` after the
+page re-rendered), the action fails loudly with `STALE_REF` instead of silently
+no-op'ing. The agent then re-snapshots and retries.
+
+**Applicability to GreedySearch-pi:** We don't use accessibility refs, but the
+generation-counter concept could help with the ChatGPT copy-button race
+condition. Currently, `document.querySelectorAll('[data-testid="copy-turn-action-button"]')`
+can pick the wrong button when the assistant response hasn't rendered its own
+copy button yet. A generation counter on the assistant message element would let
+us verify we're clicking the right generation's button.
+
+#### Named session isolation (`CHROME_DEVTOOLS_AXI_SESSION`)
+
+Each session name gets its own bridge process, port (auto-derived from the name),
+and on-disk state. In isolated mode each bridge also launches its own Chrome,
+so concurrent sessions share neither browser state nor stale-ref tracking.
+
+**Applicability to GreedySearch-pi:** Could support concurrent GreedySearch
+users on a shared machine without port conflicts. Currently we use a single
+Chrome profile at `<tmp>/greedysearch-chrome-profile` on port 9222. Named
+sessions would let multiple Pi agents run searches simultaneously.
+
+#### Persistent bridge server (one per user session)
+
+A detached process keeps the MCP session alive across commands, so Chrome
+doesn't restart every invocation. Writes a PID file to `~/.chrome-devtools-axi/bridge.pid`,
+recycles stale CDP targets after a deep health check, and reaps child processes
+on stop.
+
+**Applicability to GreedySearch-pi:** We already have per-tab daemons in
+`bin/cdp.mjs` that hold CDP sessions open. The difference is per-tab vs
+per-user-session. A per-user bridge could reduce Chrome launch overhead when
+switching between engines, but our current model (one Chrome instance, multiple
+tabs) already achieves this.
+
+#### Multi-step script execution (`run` command)
+
+Accepts multi-step scripts from stdin, executing them sequentially against the
+same browser session. Each step is a CLI command (open, click, fill, eval, etc.).
+
+**Applicability to GreedySearch-pi:** Our research mode already orchestrates
+multi-step workflows (plan → search → fetch → synthesize). The `run` pattern
+is cleaner for ad-hoc multi-step tasks but our orchestration is purpose-built
+for research.
+
+#### `CHROME_DEVTOOLS_AXI_BROWSER_URL` for external Chrome
+
+Connects to an existing Chrome instance instead of launching one. Accepts both
+`http://` URLs (fetches `/json/version` to discover WebSocket URL) and `ws://`
+endpoints directly. Supports authenticated WebSocket endpoints via
+`CHROME_DEVTOOLS_AXI_WS_HEADERS`.
+
+**Applicability to GreedySearch-pi:** We already support `CDP_PROFILE_DIR` for
+targeting our own Chrome profile. The WebSocket auth header support could be
+useful for connecting to remote browser instances (BrowserBase, Browserless,
+etc.) in a future cloud mode.
+
+#### `CHROME_DEVTOOLS_AXI_CHANNEL` for release channel selection
+
+Picks which installed Chrome release channel to target — `stable` (default),
+`beta`, `canary`, or `dev`. Used by `--autoConnect` and launch modes.
+
+**Applicability to GreedySearch-pi:** Could let users test against Chrome Beta
+or Canary to catch anti-bot changes before they hit stable.
+
+### What's not applicable
+
+- **TOON encoding** — compact structured output format. We use JSON for
+  structured data and plain text for answers. Not needed.
+- **Lighthouse / performance / heap snapshots** — performance auditing is
+  outside GreedySearch's scope.
+- **chrome-devtools-mcp dependency** — wrapping MCP would mean losing control
+  over our extraction pipelines, stealth patches, and multi-engine
+  parallelization. Not worth it.
+
+---
+
+## 2. [faster-chrome-devtools-skill](https://github.com/zeke/faster-chrome-devtools-skill)
+
+A Pi-compatible Agent Skill wrapping CDP in a zero-dependency Node.js script.
+Shares the same ancestor (`chrome-cdp-skill` by pasky) as our `bin/cdp.mjs`.
+WebSocket client from scratch (RFC 6455), background daemon, accessibility
+snapshots, Cloudflare Browser Run support.
+
+### Notable patterns
+
+#### Daemon-level Runtime.enable workaround (`captureMainContext`)
+
+**Already implemented in GreedySearch-pi.** Our `bin/cdp.mjs` has the exact
+pattern: briefly enable `Runtime` at daemon start (50-100ms), capture the main
+execution context ID, immediately disable, then pass `contextId` explicitly on
+every `Runtime.evaluate` call. This avoids the persistent `Runtime.enable`
+detection vector that Cloudflare/DataDome watch for (the browser auto-serializes
+console objects when Runtime is active, triggering proxy traps in anti-bot
+scripts).
+
+Our implementation is in `bin/cdp.mjs` — `captureMainContext()` function with
+detailed comments citing rebrowser.net and the detection mechanism.
+
+#### `loadall` command (repeated click-until-gone)
+
+Repeatedly clicks a CSS selector (e.g. "Load more") at a configurable interval
+until the element disappears from the DOM. Has a 5-minute hard deadline and
+interval clamping (100ms–30s).
+
+**Applicability to GreedySearch-pi:** Could be useful for infinite-scroll result
+pages in research mode. Some search engines (Bing, Google) paginate results
+behind "Show more" buttons. A `loadall`-style helper in `extractors/common.mjs`
+would let extractors scroll through all results before extracting.
+
+#### Compact accessibility tree mode
+
+`shouldShowAxNode()` filters out `InlineTextBox` nodes and empty/trivial nodes
+from snapshot output, producing a much more compact tree.
+
+**Applicability to GreedySearch-pi:** Minor UX improvement for our `snap`
+command output. Not a priority.
+
+#### `browserraw` / `browse` level commands
+
+Sends browser-level CDP commands (no sessionId) for things like
+`Browser.setDownloadBehavior`, `Browser.grantPermissions`, etc.
+
+**Applicability to GreedySearch-pi:** Could be useful for PDF download in
+research mode when fetching academic papers. Currently we fetch via HTTP; a
+browser-level download handler would let us capture PDFs that require
+JavaScript rendering.
+
+#### Cloudflare Browser Run support
+
+Connects to Cloudflare's remote browser service via `wss://` endpoints with
+authentication headers. Uses `lab=true` by default for Chrome beta features.
+
+**Applicability to GreedySearch-pi:** Future direction — running GreedySearch
+in the cloud without a local Chrome instance. The WebSocket auth header pattern
+(`CHROME_DEVTOOLS_AXI_WS_HEADERS`) from chrome-devtools-axi complements this.
+
+#### `--ws-endpoint` / `--http-endpoint` env vars
+
+Cleaner than our `CDP_PROFILE_DIR` approach for specifying which Chrome to
+connect to. `CDP_WS_ENDPOINT` and `CDP_HTTP_ENDPOINT` env vars with explicit
+`--ws-endpoint` / `--http-endpoint` CLI flags.
+
+**Applicability to GreedySearch-pi:** Could simplify our connection logic.
+Currently we derive the WebSocket URL from `DevToolsActivePort` in the profile
+directory. Accepting an explicit endpoint would be more flexible.
+
+### What's not applicable
+
+- **Hand-rolled RFC 6455 WebSocket client** — unnecessary since Node.js 22+
+  has built-in `WebSocket`. We use the native one.
+- **General-purpose browser skill** — it's designed for ad-hoc browsing, not
+  multi-engine search. No clipboard interception, no stream detection, no
+  source extraction.
+- **Per-tab daemon model** — same as ours. No improvement to adopt.
+
+---
+
+## Summary of actionable ideas
+
+| Idea | Source | Priority | Effort |
+|------|--------|----------|--------|
+| Generation counters for stale-ref detection | chrome-devtools-axi | Medium | Small |
+| Named session isolation | chrome-devtools-axi | Low | Medium |
+| `loadall` helper for infinite-scroll pages | faster-chrome-devtools-skill | Medium | Small |
+| Browser-level CDP for PDF download | faster-chrome-devtools-skill | Low | Small |
+| Explicit WS endpoint env vars | faster-chrome-devtools-skill | Low | Small |
+| Remote browser auth headers | both | Low | Medium |
+| Release channel selection | chrome-devtools-axi | Low | Small |
+| Runtime.enable workaround | faster-chrome-devtools-skill | ✅ Already done | — |

package/docs/releases.md

@@ -0,0 +1,88 @@
+# Release Workflow
+
+Releases are automated from `package.json` versions and `CHANGELOG.md` sections.
+The curated changelog entry is the source of truth for GitHub release notes.
+
+## Cut a Release
+
+1. Bump `package.json`.
+2. Add entries under `## [Unreleased]`.
+3. Promote the changelog section:
+
+   ```bash
+   npm run changelog:release
+   ```
+
+   Or with explicit values:
+
+   ```bash
+   node scripts/changelog-release.mjs 2.2.0 --date 2026-07-03
+   ```
+
+4. Run checks:
+
+   ```bash
+   npm run check:lockfile
+   npm run lint
+   node test.mjs unit
+   ```
+
+5. Commit and push to `master`.
+
+## Changelog Scripts
+
+- `npm run changelog:check` — verifies `## [Unreleased]` has releaseable
+  entries.
+- `npm run changelog:release` — moves `Unreleased` into the current
+  `package.json` version and opens a fresh empty `Unreleased` section.
+- `npm run changelog:extract -- <version>` — prints release notes for a
+  version.
+- `npm run release:backfill-notes` — dry-run GitHub release-body backfill.
+
+The parser supports both current headings like:
+
+```markdown
+## [2.1.3] — 2026-06-21
+```
+
+and older headings like:
+
+```markdown
+## v1.8.5 (2026-04-29)
+```
+
+so historical releases can be backfilled from the same changelog.
+
+## GitHub Release Notes
+
+`.github/workflows/release.yml` extracts release notes with:
+
+```bash
+node scripts/changelog-extract.mjs "$VERSION" --summary -o RELEASE_NOTES.md
+```
+
+and passes `RELEASE_NOTES.md` to `softprops/action-gh-release`. GitHub release
+bodies therefore match the curated changelog summary instead of generated PR
+or commit-title notes.
+
+## Backfill Existing Releases
+
+Preview the update plan:
+
+```bash
+npm run release:backfill-notes
+```
+
+Apply it with the GitHub CLI authenticated:
+
+```bash
+node scripts/backfill-github-releases.mjs --apply
+```
+
+Limit to specific tags:
+
+```bash
+node scripts/backfill-github-releases.mjs --apply --only v2.1.3,v2.1.2
+```
+
+Use `--full` to write the full changelog prose instead of the summarized body.

package/docs/research.md

@@ -0,0 +1,82 @@
+# Research Mode
+
+Set `depth: "research"` to run GreedySearch's iterative research workflow.
+
+```js
+greedy_search({
+  query: "Evaluate browser automation options for AI agents",
+  depth: "research",
+  breadth: 3,
+  iterations: 2,
+  maxSources: 8,
+});
+```
+
+## Workflow
+
+Research mode performs:
+
+1. Query complexity classification when breadth/iterations are not explicit.
+2. Action planning with Gemini.
+3. Fast all-engine child searches using the configured engine list.
+4. Direct URL fetches when useful.
+5. Source ranking, dedupe, and source-content fetching.
+6. Evidence and learning extraction.
+7. Final cited synthesis.
+8. Citation audit, URL reachability checks, and deterministic floor checks.
+
+Simple questions may take a single-pass path. Explicit `breadth` or
+`iterations` values always override classifier suggestions.
+
+## Bundle Layout
+
+Research bundles are written by default under
+`.pi/greedysearch-research/<timestamp>_<query>/`.
+
+```text
+STATUS.md              # floor status, question ledger, and gaps
+OUTLINE.md             # bundle table of contents
+provenance.md          # run metadata and verification summary
+reports/SUMMARY.md     # final cited report
+reports/CLAIMS.md      # claims mapped to source IDs
+reports/EVIDENCE.md    # extracted source evidence
+reports/GAPS.md        # caveats and remaining uncertainties
+sources/               # fetched source markdown files
+data/manifest.json     # metadata, stop reason, floor checks, citation audit
+data/rounds.json       # per-round actions/learnings/gaps
+data/sources.json      # ranked source registry
+data/questions.json    # open/closed question ledger
+data/evidence.json     # structured evidence per useful source
+```
+
+## CLI
+
+```bash
+node bin/search.mjs all --inline --stdin --depth research \
+  --breadth 3 --iterations 2 --max-sources 8 <<'EOF'
+Evaluate browser automation options for AI agents
+EOF
+
+node bin/search.mjs all --inline --stdin --depth research \
+  --research-out-dir ./research-topic <<'EOF'
+Topic
+EOF
+
+node bin/search.mjs all --inline --stdin --depth research \
+  --no-research-bundle <<'EOF'
+Topic
+EOF
+```
+
+## Verification
+
+The provenance sidecar records:
+
+- sources consulted, fetched, and cited;
+- primary/official source counts;
+- citation audit status;
+- citation URL reachability;
+- floor checks and overall status.
+
+Bot-protected or HEAD-disallowing hosts are skipped in URL reachability rather
+than marked dead when they return common anti-bot statuses such as 403.

package/docs/runtime.md

@@ -0,0 +1,65 @@
+# Runtime and Chrome
+
+GreedySearch uses a dedicated Chrome profile and debug port. It must not attach
+to your normal Chrome profile.
+
+- Profile: OS temp directory `greedysearch-chrome-profile`
+- Port: `9222`
+- Default mode: headless
+
+## Pi Commands
+
+```text
+/greedy-visible      # launch visible Chrome for captcha/login/cookie setup
+/greedy-status       # show GreedySearch Chrome status
+/greedy-kill         # stop GreedySearch Chrome
+/set-greedy-locale   # set default result language
+```
+
+## Environment Variables
+
+- `GREEDY_SEARCH_VISIBLE` — set `1` to show Chrome instead of headless.
+- `GREEDY_SEARCH_ALWAYS_VISIBLE` — set `1` to force visible mode for all runs.
+- `GREEDY_SEARCH_IDLE_TIMEOUT_MINUTES` — headless idle cleanup timeout;
+  default `5`.
+- `GREEDY_SEARCH_LOCALE` — default result language; default `en`.
+- `CHROME_PATH` — Chrome/Chromium executable path; auto-detected by default.
+
+## Runtime Helpers
+
+Git install path:
+
+```bash
+GS=~/.pi/agent/git/github.com/apmantza/GreedySearch-pi
+node "$GS/bin/launch.mjs" --status
+node "$GS/bin/visible.mjs"
+node "$GS/bin/visible.mjs" --kill
+node "$GS/bin/kill-visible.mjs"
+node "$GS/bin/cdp-visible.mjs" list
+node "$GS/bin/cdp-headless.mjs" list
+node "$GS/bin/cdp-greedy.mjs" list
+```
+
+npm global install path:
+
+```bash
+GS="$(npm root -g)/@apmantza/greedysearch-pi"
+node "$GS/bin/launch.mjs" --status
+node "$GS/bin/visible.mjs"
+node "$GS/bin/visible.mjs" --kill
+node "$GS/bin/kill-visible.mjs"
+node "$GS/bin/cdp-visible.mjs" list
+node "$GS/bin/cdp-headless.mjs" list
+node "$GS/bin/cdp-greedy.mjs" list
+```
+
+## CDP Safety
+
+Use only the safe wrappers for manual debugging:
+
+- `bin/cdp-visible.mjs` — refuses unless GreedySearch Chrome is visible.
+- `bin/cdp-headless.mjs` — refuses unless GreedySearch Chrome is headless.
+- `bin/cdp-greedy.mjs` — attaches only to the GreedySearch Chrome profile.
+
+Avoid raw `bin/cdp.mjs` unless `CDP_PROFILE_DIR` explicitly points at the
+GreedySearch profile.

package/docs/source-fetching.md

@@ -0,0 +1,33 @@
+# Source Fetching
+
+For `engine: "all"`, GreedySearch ranks discovered sources and fetches top source
+content by default. `synthesize: true` then asks the configured synthesizer to
+combine engine answers with fetched evidence.
+
+## Supported Sources
+
+- **PDFs** — direct PDF links are parsed into markdown text.
+- **Semantic Scholar** — academic paper URLs and direct PDFs are preferred when
+  the engine is used directly or opted into the fan-out.
+- **Reddit** — public `.json` API for posts and comments.
+- **GitHub** — REST API for repos, READMEs, file trees, and raw file content.
+- **General web** — Readability extraction with browser fallback.
+
+## Metadata
+
+Fetched sources include the best available:
+
+- title;
+- final URL;
+- status/content type;
+- byline and site name;
+- publish or modified date;
+- language;
+- excerpt/snippet;
+- trimmed markdown/plain text content.
+
+## Security
+
+Fetchers reject private/internal URLs and re-check the final redirected URL to
+avoid SSRF bypasses. GitHub and Reddit URLs use dedicated fetchers where
+possible to avoid fragile page scraping.