heyspex 0.1.0 → 0.2.0

package/vendor/heyspex-mcp/plan-ground.mjs

@@ -1,11 +1,24 @@
 /**
  * Pre-plan grounding for HeySpex MCP (/hs-ground).
- * Holistic vault scan, north-star/pillar hooks, architecture placement,
- * stress-test prompts, enhancement seeds — read-only.
+ * North-star/pillar hooks, architecture placement, stress-test prompts,
+ * enhancement seeds — read-only.
+ *
+ * RETRIEVAL IS BOUNDED AND REMOTE ([[HS-YIR1-8KK5-TASK-planning-grounding-retrieves-through]]
+ * of [[HS-FI33-8KK5-EPIC-lexical-retrieval-in-postgres-generated]]). This module
+ * used to walk `specs/**` from disk on every call, parse every markdown file and
+ * build an in-process BM25 corpus + backlink graph — work proportional to vault
+ * size on a request path. It now issues a fixed, small number of bounded
+ * `consoleFetch` calls to `GET /api/search` (top-k per angle, k <=
+ * {@link SEARCH_ANGLE_LIMIT}, paged with the endpoint's `next_cursor`), so
+ * grounding a plan costs the same at six thousand specs and at a million.
+ *
+ * NOTHING in this file may read the filesystem — `plan-ground.guard.test.mjs`
+ * greps the source for `readFileSync` / `readdirSync` / `walkMdFiles` /
+ * `fs.glob` / `globSync` to keep that true.
  */
 
-import { readFileSync, existsSync, readdirSync } from 'node:fs';
-import { join, relative, basename } from 'node:path';
+import { basename } from 'node:path';
+import { consoleFetch as defaultConsoleFetch } from './console-transport.mjs';
 import {
   analyzeGroundingConflicts,
   computeSafeContextSpecs,
@@ -27,27 +40,89 @@ const ANCHOR_SPECS = [
   // WIP ledger (.heyspex/wip-state.json, surfaced in the /hs-next brief), not a markdown anchor.
 ];
 
-const SCAN_DIRS = [
-  { rel: 'specs/epics', bucket: 'epics' },
-  { rel: 'specs/tasks', bucket: 'tasks' },
-  { rel: 'specs/protocols', bucket: 'protocols' },
-  { rel: 'specs/adr', bucket: 'adrs' },
-  { rel: 'specs/use-cases', bucket: 'use_cases' },
-  { rel: 'specs/memory', bucket: 'memories' },
-  { rel: 'specs/research', bucket: 'research' },
-  // Captures hold the owner's not-yet-specced design vocabulary (e.g. named
-  // operating modes). Excluding them was the [[HS-RES-2QX6]] root cause: a
-  // pre-plan ground that could not surface [[HS-CAP-UIBF]] let an agent reinvent
-  // an already-designed concept. Milestones + diagnoses round out the in-flight
-  // surfaces a plan can collide with (mirrors plan-review.mjs OVERLAP_DIRS).
-  { rel: 'specs/captures', bucket: 'captures' },
-  { rel: 'specs/milestones', bucket: 'milestones' },
-  { rel: 'specs/diagnoses', bucket: 'diagnoses' },
-  { rel: 'specs/future-considerations', bucket: 'future_considerations' },
-];
+/**
+ * Frontmatter `type:` → discovery bucket. Replaces the old `SCAN_DIRS` folder
+ * walk: the search endpoint returns a row's `type`, so the bucket a candidate
+ * lands in is read off the row instead of off the directory it was walked from.
+ * Captures stay in the corpus — excluding them was the [[HS-RES-2QX6]] root
+ * cause (an agent reinvented an already-designed concept because /hs-ground
+ * could not see [[HS-CAP-UIBF]]); milestones + diagnoses round out the in-flight
+ * surfaces a plan can collide with.
+ */
+const TYPE_BUCKETS = {
+  epic: 'epics',
+  task: 'tasks',
+  protocol: 'protocols',
+  adr: 'adrs',
+  'use-case': 'use_cases',
+  use_case: 'use_cases',
+  'memory-lesson': 'memories',
+  'memory-decision': 'memories',
+  'memory-reference': 'memories',
+  'memory-ui-pattern': 'memories',
+  memory: 'memories',
+  research: 'research',
+  capture: 'captures',
+  milestone: 'milestones',
+  diagnose: 'diagnoses',
+  diagnosis: 'diagnoses',
+  'future-consideration': 'future_considerations',
+};
+
+/** The `type=` filter values for the bucket-targeted recall angles (CSV, pushed into SQL). */
+const PROTOCOL_TYPE_FILTER = 'protocol';
+const MEMORY_TYPE_FILTER = 'memory-decision,memory-lesson,memory-reference,memory-ui-pattern';
+const WORK_TYPE_FILTER = 'epic,task,milestone';
+
+/**
+ * The only two numbers that bound retrieval cost. `SEARCH_ANGLE_LIMIT` is the
+ * per-call `?limit=` (the task contract: k <= 20); a wider recall window is
+ * assembled by following the endpoint's `next_cursor` for at most
+ * `MAX_PAGES_PER_ANGLE` pages, never by asking for the whole vault.
+ */
+const SEARCH_ANGLE_LIMIT = 20;
+const MAX_PAGES_PER_ANGLE = 3;
+
+/**
+ * How many of the top-ranked candidates get a bounded metadata read
+ * (`GET /api/specs/<id>`) for their basename, tags, authored summary and
+ * `updated` stamp. Constant, so enrichment cost never tracks vault size. Rows
+ * outside the cap keep the endpoint's `id`/`name`/`type`/`status`/`rank`/
+ * `snippet` and score on rank + status alone.
+ */
+const ENRICH_CAP = 12;
+
+/** Known-tags page size for the focus resolver's strict-tag vocabulary lookup. */
+const KNOWN_TAGS_PAGE = 200;
+
+/**
+ * How many of the per-row calls may be in flight at once. Bounded work is not
+ * the same as polite work: firing the whole enrichment batch plus the anchor
+ * resolves at a single-process dev console at once made every one of them time
+ * out (measured on the disk read tier, where each `/api/specs/:id` is a vault
+ * read). Small and fixed — the wall-clock cost of the extra rounds is noise
+ * against one timeout.
+ */
+const FETCH_CONCURRENCY = 4;
+
+/** `Promise.all`-with-a-lid: run `fn` over `items`, at most {@link FETCH_CONCURRENCY} at a time. */
+async function mapWithConcurrency(items, fn, concurrency = FETCH_CONCURRENCY) {
+  const results = new Array(items.length);
+  let cursor = 0;
+  const worker = async () => {
+    for (;;) {
+      const index = cursor++;
+      if (index >= items.length) return;
+      // eslint-disable-next-line no-await-in-loop -- that is the point: one at a time per worker.
+      results[index] = await fn(items[index], index);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
+  return results;
+}
 
 const AREA_BOOSTS = {
-  console: { paths: ['spex-console/'], tags: ['console', 'ui', 'cockpit'], surfaces: ['console'] }, // legacy-cockpit tag accepted
+  console: { paths: ['spex-console/'], tags: ['console', 'ui'], surfaces: ['console'] },
   companion: { paths: ['companion/'], tags: ['companion', 'windows'], surfaces: ['companion'] },
   mcp: { paths: ['vendor/heyspex-mcp/'], tags: ['mcp', 'slash'], surfaces: ['mcp'] },
   vault: { paths: ['specs/'], tags: ['vault', 'specs'], surfaces: ['vault'] },
@@ -71,24 +146,22 @@ const STRESS_TEST_PROMPTS = [
   'Is this in-scope for current milestones, or should it be deferred to horizon?',
 ];
 
-const SKIP_DIRS = new Set([
-  'node_modules', '.git', '.next', 'dist', 'build', 'coverage', '.heyspex',
-]);
-
 // ─────────────────────────────────────────────────────────────────────────────
-// TF-IDF Search Index (BM25 + Backlinks + Recency)
+// Reranking over a bounded retrieval window (Postgres lexical recall + signals)
 // ─────────────────────────────────────────────────────────────────────────────
 //
-// ## Algorithm Overview (HS-EPIC-74YP)
+// ## Algorithm Overview (HS-EPIC-74YP, retrieval retired to Postgres by HS-YIR1)
 //
-// This module implements a Google-like relevance ranking system for the HeySpex
-// spec vault, combining multiple signals into a weighted score:
+// Retrieval is `GET /api/search` — `search_vector @@ websearch_to_tsquery`,
+// ordered by `ts_rank_cd`, GIN-index-backed, keyset-paged. This module no longer
+// computes BM25 itself; it RERANKS the bounded window the endpoint returns,
+// combining the endpoint's rank with the signals that live outside the tsvector:
 //
-// ### Signal 1: BM25 TF-IDF (primary signal, weight: 3.0)
-// - Industry-standard document retrieval algorithm (Okapi BM25)
-// - Rare terms score higher than common terms via IDF (inverse document frequency)
-// - Term frequency saturates to prevent keyword stuffing
-// - Document length normalization prevents long docs from dominating
+// ### Signal 1: Retrieval recall (primary signal, weight: 3.0)
+// - `ts_rank_cd` from the endpoint (title weight A over body weight B)
+// - plus a positional term, so the disk read tier (which returns rank 0 for
+//   every row) still ranks by the order the endpoint handed rows back
+// - plus a small bonus for a candidate that surfaced on more than one angle
 //
 // ### Signal 2: Title Match Boost (weight: 2.0)
 // - Full phrase match in title: 10 points
@@ -109,204 +182,317 @@ const SKIP_DIRS = new Set([
 // - in_review: 1.0 (pending completion)
 // - closed: 0 (completed, less relevant for planning)
 //
-// ### Signal 6: Backlink Importance (weight: 1.2)
-// - Specs with more inbound wikilinks are more "authoritative"
-// - Uses log scale: log1p(count), capped at 3
-// - Prevents heavily-linked specs from dominating while still boosting them
-//
-// ### Signal 7: Recency (weight: 0.6)
+// ### Signal 6: Recency (weight: 0.6)
 // - Recently updated specs are more relevant for planning
 // - Decay tiers: <7d=2, <30d=1.5, <90d=1, <365d=0.5, older=0
+// - Applies to candidates whose bounded metadata read returned an `updated`
+//   stamp; a row outside ENRICH_CAP simply scores 0 here (never a file read).
 //
 // ### Formula
-//   Score = (BM25 × 3.0) + (title × 2.0) + (tag × 1.5) + (area × 1.0)
-//         + (status × 0.8) + (backlink × 1.2) + (recency × 0.6)
+//   Score = (recall × 3.0) + (title × 2.0) + (tag × 1.5) + (area × 1.0)
+//         + (status × 0.8) + (recency × 0.6)
+//
+// ### Retired signal: backlink importance (was weight 1.2)
+// The inbound-wikilink boost needed a whole-vault walk to build the backlink
+// graph — exactly the cost this task exists to remove — so it is GONE, not
+// approximated. A per-spec backlink count can return later through
+// `/api/links/:id` / `/api/backlinks/:id`, which reads `spec_links` in one
+// indexed query. ([[HS-YIR1-8KK5-TASK-planning-grounding-retrieves-through]])
 //
 // ### Tuning
 // - Weights chosen empirically to balance precision vs recall
-// - tfidf is primary signal (3.0) because it captures semantic relevance
+// - recall is the primary signal (3.0): it is the endpoint's ranked relevance
 // - title gets 2.0 because explicit matches are high confidence
 // - status/recency are lower (0.6-0.8) as tiebreakers, not primary filters
 //
 // See: [[hs-ground-search-algorithm]] for design decision documentation
-// Tests: vendor/heyspex-mcp/plan-ground.test.mjs (64 unit tests)
+// Tests: vendor/heyspex-mcp/plan-ground.test.mjs
 // ─────────────────────────────────────────────────────────────────────────────
 
-/** Lazy index singleton with cache invalidation */
-let cachedIndex = null;
-let cacheTimestamp = 0;
-let cachedVaultRoot = null;
-const CACHE_TTL_MS = 30_000; // 30 seconds
-
 /**
  * Scoring weights for combining relevance signals.
  *
  * These weights were tuned empirically to balance:
- * - tfidf (3.0): Primary signal for semantic relevance
+ * - rank (3.0): Primary signal — the search endpoint's ranked recall
  * - title (2.0): High-confidence explicit matches
  * - tag (1.5): User-defined categorization
  * - area (1.0): Code area relevance
  * - status (0.8): Prioritize actionable specs
- * - backlink (1.2): Graph-based importance
  * - recency (0.6): Time decay as tiebreaker
  *
  * @see [[hs-ground-search-algorithm]] for rationale
  */
 const SCORE_WEIGHTS = {
-  tfidf: 3.0,      // BM25 contribution (primary signal)
+  rank: 3.0,       // Retrieval recall from /api/search (primary signal)
   title: 2.0,      // Title match boost
   tag: 1.5,        // Tag match
   area: 1.0,       // Area/surface match
   status: 0.8,     // Status boost
-  backlink: 1.2,   // Graph importance
   recency: 0.6,    // Time decay
 };
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Bounded console retrieval — every read below is one HTTP call with an
+// explicit `limit`, optionally continued with the endpoint's opaque cursor.
+// ─────────────────────────────────────────────────────────────────────────────
+
 /**
- * BM25 algorithm parameters.
- *
- * k1 (1.5): Term frequency saturation parameter
- * - Controls how quickly TF contribution saturates
- * - Range: 1.2-2.0 typical; 1.5 is a balanced default
- * - Higher = TF matters more; lower = saturates faster
- *
- * b (0.75): Document length normalization parameter
- * - Controls how much long documents are penalized
- * - Range: 0.0-1.0; 0.75 is the standard default
- * - Higher = more normalization; 0 = no length penalty
- *
- * These are the Robertson/Walker defaults from the original BM25 paper.
+ * Raised when the grounding path cannot retrieve at all (console unreachable,
+ * not signed in, 5xx). Callers turn this into a typed degraded/error result —
+ * grounding NEVER falls back to reading the vault off disk.
  */
-const BM25_K1 = 1.5;  // Term frequency saturation
-const BM25_B = 0.75;  // Document length normalization
+class ConsoleRetrievalError extends Error {
+  constructor(message, { code = 'CONSOLE_UNREACHABLE', status = 0, url = null } = {}) {
+    super(message);
+    this.name = 'ConsoleRetrievalError';
+    this.code = code;
+    this.status = status;
+    this.url = url;
+  }
+}
+
+function clampInt(value, min, max, fallback) {
+  const n = Number(value);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.min(Math.max(Math.trunc(n), min), max);
+}
+
+/** `<mark>` markers from `ts_headline` are for the console UI, not for an agent. */
+function stripHeadlineMarks(snippet) {
+  return String(snippet || '')
+    .replace(/<\/?mark>/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
 
 /**
- * Build the in-memory search index over all specs.
- * Computes: corpus stats, term frequencies, backlink graph, recency data.
+ * Normalize one search row. The Neon tier answers `{ rows: [...] }` with a
+ * `ts_rank_cd` rank and a `ts_headline` snippet; the disk tier answers the same
+ * `rows` shape with `rank: 0` (and a legacy `results` alias). Read either.
  */
-function buildVaultIndex(vaultRoot) {
-  const startMs = Date.now();
-  const specs = new Map();
-  const corpus = {
-    totalDocs: 0,
-    documentFrequency: new Map(),
-    avgDocLength: 0,
+function normalizeSearchRow(row) {
+  if (!row || typeof row !== 'object') return null;
+  const id = String(row.id ?? '').trim();
+  if (!id) return null;
+  const rank = Number(row.rank);
+  return {
+    id,
+    name: String(row.name ?? id),
+    type: String(row.type ?? 'unknown'),
+    status: String(row.status ?? 'open').toLowerCase(),
+    rank: Number.isFinite(rank) && rank > 0 ? rank : 0,
+    snippet: stripHeadlineMarks(row.snippet),
   };
-  const backlinkGraph = new Map(); // target -> Set<source>
+}
 
-  // Phase 1: Parse all specs, extract tokens and links
-  const specsRoot = join(vaultRoot, 'specs');
-  const allFiles = walkMdFiles(specsRoot);
-  let totalLength = 0;
+/**
+ * ONE bounded page of `GET /api/search`. `limit` is hard-capped at
+ * {@link SEARCH_ANGLE_LIMIT} here, not by the caller — no code path in this
+ * module can ask the endpoint for an unbounded page.
+ */
+async function fetchSearchPage(fetchImpl, { q, limit, type, status, after }) {
+  const params = new URLSearchParams();
+  params.set('q', q);
+  params.set('limit', String(clampInt(limit, 1, SEARCH_ANGLE_LIMIT, SEARCH_ANGLE_LIMIT)));
+  if (type) params.set('type', type);
+  if (status) params.set('status', status);
+  if (after) params.set('after', after);
+  const res = await fetchImpl(`/api/search?${params.toString()}`);
+  if (!res || !res.ok) {
+    const status = res?.status ?? 0;
+    // Split "the console is not there" from "the console answered badly": the
+    // first sends an agent to heyspex_dev_server_status, the second is a bug or
+    // a sign-in problem and no amount of restarting fixes it.
+    const code = status === 0
+      ? 'CONSOLE_UNREACHABLE'
+      : status === 401
+        ? 'CONSOLE_NOT_SIGNED_IN'
+        : 'SEARCH_FAILED';
+    throw new ConsoleRetrievalError(
+      `GET /api/search returned ${status || 'no response'}`,
+      { code, status, url: res?.url ?? null },
+    );
+  }
+  const body = res.body && typeof res.body === 'object' ? res.body : {};
+  const raw = Array.isArray(body.rows)
+    ? body.rows
+    : Array.isArray(body.results)
+      ? body.results
+      : [];
+  return {
+    rows: raw.map(normalizeSearchRow).filter(Boolean),
+    next_cursor: typeof body.next_cursor === 'string' && body.next_cursor ? body.next_cursor : null,
+    mode: typeof body.mode === 'string' ? body.mode : null,
+  };
+}
 
-  for (const filePath of allFiles) {
-    try {
-      const content = readFileSync(filePath, 'utf8');
-      const { frontmatter, body } = parseFrontmatter(content);
-      const fileBase = basename(filePath, '.md');
-      const key = fileBase.toLowerCase();
-
-      const titleTokens = tokenize(frontmatter.name || fileBase);
-      const bodyTokens = tokenize(body);
-      const allTokens = [...titleTokens, ...bodyTokens];
-
-      // Build term frequency map
-      const termFrequency = new Map();
-      for (const token of allTokens) {
-        termFrequency.set(token, (termFrequency.get(token) || 0) + 1);
-      }
+/**
+ * One recall ANGLE: up to `want` rows assembled from at most
+ * {@link MAX_PAGES_PER_ANGLE} bounded pages, following the endpoint's keyset
+ * cursor. Never `OFFSET`, never a total count, never a second query per row.
+ */
+async function fetchSearchAngle(fetchImpl, { label, q, want, type, status }) {
+  const target = clampInt(want, 1, SEARCH_ANGLE_LIMIT * MAX_PAGES_PER_ANGLE, SEARCH_ANGLE_LIMIT);
+  const rows = [];
+  let after = null;
+  let mode = null;
+  for (let page = 0; page < MAX_PAGES_PER_ANGLE && rows.length < target; page++) {
+    const remaining = target - rows.length;
+    // eslint-disable-next-line no-await-in-loop -- keyset paging is inherently sequential.
+    const res = await fetchSearchPage(fetchImpl, {
+      q,
+      limit: Math.min(remaining, SEARCH_ANGLE_LIMIT),
+      type,
+      status,
+      after,
+    });
+    mode = mode ?? res.mode;
+    rows.push(...res.rows);
+    if (!res.next_cursor || res.rows.length === 0) break;
+    after = res.next_cursor;
+  }
+  return { label, mode, rows: rows.slice(0, target) };
+}
 
-      // Extract wikilinks for backlink graph
-      const outgoingLinks = extractWikilinks(body, frontmatter);
-      for (const target of outgoingLinks) {
-        const normalized = target.toLowerCase();
-        if (!backlinkGraph.has(normalized)) {
-          backlinkGraph.set(normalized, new Set());
-        }
-        backlinkGraph.get(normalized).add(key);
+/**
+ * Merge the angles into one deduplicated candidate window. A candidate keeps
+ * its best rank and best position across angles, plus the labels of every angle
+ * that surfaced it (the multi-angle agreement bonus, and the `why_matched`
+ * provenance the reranking agent reads).
+ */
+function mergeAngles(angles) {
+  const byId = new Map();
+  for (const angle of angles) {
+    if (!angle) continue;
+    angle.rows.forEach((row, position) => {
+      const existing = byId.get(row.id);
+      if (existing) {
+        existing.rank = Math.max(existing.rank, row.rank);
+        existing.bestPosition = Math.min(existing.bestPosition, position);
+        if (!existing.snippet && row.snippet) existing.snippet = row.snippet;
+        if (!existing.angles.includes(angle.label)) existing.angles.push(angle.label);
+        return;
       }
-
-      // Parse timestamps
-      const updatedAt = parseIsoDate(frontmatter.updated);
-      const createdAt = parseIsoDate(frontmatter.created);
-
-      specs.set(key, {
-        basename: fileBase,
-        id: frontmatter.id || fileBase,
-        name: frontmatter.name || fileBase,
-        type: frontmatter.type || 'unknown',
-        status: String(frontmatter.status || 'open').toLowerCase(),
-        tags: normalizeTags(frontmatter.tags),
-        // Authored frontmatter.summary, carried so candidate cards can prefer it
-        // over the body-derived extractSummaryLine() fallback (authored wins).
-        summary: typeof frontmatter.summary === 'string' ? frontmatter.summary : null,
-        relPath: relative(vaultRoot, filePath).replace(/\\/g, '/'),
-        titleTokens,
-        termFrequency,
-        docLength: allTokens.length,
-        backlinkCount: 0, // computed in phase 3
-        forwardRefCount: outgoingLinks.size,
-        updatedAt,
-        createdAt,
+      byId.set(row.id, {
+        id: row.id,
+        // `basename` is unknown until the bounded metadata read below; the id is
+        // a resolvable stand-in ([[HS-YIR1]] — `/api/specs/:id` takes either).
+        basename: row.id,
+        name: row.name,
+        type: row.type,
+        status: row.status,
+        rank: row.rank,
+        snippet: row.snippet,
+        bestPosition: position,
+        angles: [angle.label],
+        // Filled by enrichCandidates; defaults keep every scorer total.
+        tags: [],
+        relPath: '',
+        updatedAt: 0,
+        summary: null,
+        enriched: false,
       });
+    });
+  }
+  return [...byId.values()];
+}
 
-      corpus.totalDocs++;
-      totalLength += allTokens.length;
+/**
+ * Bounded metadata read for the top {@link ENRICH_CAP} candidates: basename (so
+ * the report can emit a resolving `[[wikilink]]`), tags (tag weight + the focus
+ * resolver's strict-tag test), authored `summary:` and the `updated` stamp
+ * (recency weight). Best-effort and parallel — a miss leaves the row scored on
+ * rank + status, it never falls back to a file read.
+ */
+async function enrichCandidates(fetchImpl, candidates, cap = ENRICH_CAP) {
+  const targets = candidates.slice(0, Math.max(0, cap));
+  await mapWithConcurrency(targets, async (candidate) => {
+    try {
+      const res = await fetchImpl(`/api/specs/${encodeURIComponent(candidate.id)}`);
+      if (!res || !res.ok) return;
+      const spec = res.body && typeof res.body === 'object' ? res.body.spec : null;
+      if (!spec || typeof spec !== 'object') return;
+      applySpecMetadata(candidate, spec);
     } catch {
-      // Skip unreadable files
+      // Best-effort: an enrichment miss degrades the row's signals, never the call.
     }
-  }
+  });
+  return candidates;
+}
 
-  corpus.avgDocLength = corpus.totalDocs > 0 ? totalLength / corpus.totalDocs : 0;
+/** Fold a `SpecSummary`-shaped record (list row or `/api/specs/:id`) onto a candidate. */
+function applySpecMetadata(candidate, spec) {
+  candidate.enriched = true;
+  if (spec.basename) candidate.basename = String(spec.basename);
+  if (spec.relPath) candidate.relPath = String(spec.relPath);
+  if (spec.name) candidate.name = String(spec.name);
+  if (spec.type) candidate.type = String(spec.type);
+  if (spec.status) candidate.status = String(spec.status).toLowerCase();
+  candidate.tags = normalizeTags(spec.tags);
+  candidate.updatedAt = parseIsoDate(spec.updatedAt || spec.updated);
+  const authored = spec.frontmatter && typeof spec.frontmatter.summary === 'string'
+    ? spec.frontmatter.summary.trim()
+    : typeof spec.summary === 'string'
+      ? spec.summary.trim()
+      : '';
+  if (authored) candidate.summary = authored;
+  return candidate;
+}
 
-  // Phase 2: Compute document frequency (IDF denominator)
-  for (const spec of specs.values()) {
-    const uniqueTerms = new Set(spec.termFrequency.keys());
-    for (const term of uniqueTerms) {
-      corpus.documentFrequency.set(
-        term,
-        (corpus.documentFrequency.get(term) || 0) + 1
-      );
-    }
-  }
+/**
+ * `GET /api/specs?tag=…` — ONE bounded, tag-filtered list page. The filter is
+ * pushed into SQL by the endpoint, so this is the strict-tag lookup without a
+ * corpus: the rows it returns are exactly the specs carrying that tag, with the
+ * metadata (tags, basename, updated) the scorer needs already on them.
+ */
+async function fetchTagPage(fetchImpl, tag, limit) {
+  const params = new URLSearchParams({
+    tag,
+    limit: String(clampInt(limit, 1, SEARCH_ANGLE_LIMIT, SEARCH_ANGLE_LIMIT)),
+    fields: 'id,name,type,status,tags,basename,relPath,updated,updatedAt',
+  });
+  const res = await fetchImpl(`/api/specs?${params.toString()}`);
+  if (!res || !res.ok) return [];
+  const body = res.body && typeof res.body === 'object' ? res.body : {};
+  return Array.isArray(body.specs) ? body.specs : [];
+}
 
-  // Phase 3: Compute backlink counts
-  for (const spec of specs.values()) {
-    const key = spec.basename.toLowerCase();
-    const sources = backlinkGraph.get(key);
-    spec.backlinkCount = sources ? sources.size : 0;
+/** `GET /api/scheduler/known-tags` — the closed tag vocabulary, one bounded page. */
+async function fetchKnownTags(fetchImpl, limit = KNOWN_TAGS_PAGE) {
+  try {
+    const res = await fetchImpl(`/api/scheduler/known-tags?limit=${clampInt(limit, 1, 500, KNOWN_TAGS_PAGE)}`);
+    if (!res || !res.ok) return [];
+    const body = res.body && typeof res.body === 'object' ? res.body : {};
+    return (Array.isArray(body.tags) ? body.tags : [])
+      .map((t) => (typeof t === 'string' ? t : t && t.name))
+      .filter((t) => typeof t === 'string' && t.trim())
+      .map((t) => t.trim().toLowerCase());
+  } catch {
+    return [];
   }
-
-  const buildTimeMs = Date.now() - startMs;
-
-  return {
-    corpus,
-    specs,
-    backlinkGraph,
-    lastBuilt: Date.now(),
-    buildTimeMs,
-  };
 }
 
 /**
- * Get or rebuild the cached index.
+ * `GET /api/specs/resolve?q=…` — the one bounded call that turns an anchor
+ * basename into `{ basename, id, relPath }`. Used only for the small static
+ * anchor list; returns null when the anchor is absent from this vault.
  */
-function getOrBuildIndex(vaultRoot) {
-  const now = Date.now();
-  // Invalidate on vaultRoot change as well as TTL: a single process is normally
-  // one vault, but the cache must not hand back a different root's index (would
-  // silently mis-resolve, and makes the substrate untestable across vaults).
-  if (cachedIndex && cachedVaultRoot === vaultRoot && (now - cacheTimestamp) < CACHE_TTL_MS) {
-    return cachedIndex;
+async function resolveSpecRef(fetchImpl, token) {
+  try {
+    const res = await fetchImpl(`/api/specs/resolve?q=${encodeURIComponent(token)}`);
+    if (!res || !res.ok) return null;
+    const resolved = res.body && typeof res.body === 'object' ? res.body.resolved : null;
+    return resolved && typeof resolved === 'object' ? resolved : null;
+  } catch {
+    return null;
   }
-  cachedIndex = buildVaultIndex(vaultRoot);
-  cacheTimestamp = now;
-  cachedVaultRoot = vaultRoot;
-  return cachedIndex;
 }
 
 /**
- * Extract wikilinks from body text and frontmatter.
+ * Extract wikilinks from body text and frontmatter. Retained as a pure helper
+ * (the backlink GRAPH it used to feed is gone with the vault walk) — callers
+ * still use it to read a spec's declared edges out of a record they already
+ * hold, without touching disk.
  */
 function extractWikilinks(body, frontmatter) {
   const links = new Set();
@@ -364,30 +550,29 @@ function parseIsoDate(dateStr) {
 }
 
 /**
- * BM25 scoring - industry standard for document retrieval.
- * k1 controls term frequency saturation (1.2-2.0 typical)
- * b controls document length normalization (0.75 typical)
+ * Retrieval recall for one candidate — the primary signal, replacing the
+ * in-process BM25 that needed a whole-vault corpus.
+ *
+ * Three bounded terms, each in [0, 1]-ish so the weighted sum stays comparable
+ * to the old tfidf term:
+ *  - `rank`: the endpoint's `ts_rank_cd`, clamped at 1 (values are small and
+ *    long-tailed; the clamp stops one lucky row from dominating).
+ *  - positional: how near the top of its best angle the row came back. This is
+ *    what carries the DISK read tier, whose in-memory search returns `rank: 0`
+ *    for every row but still returns them in relevance order.
+ *  - agreement: +0.25 per extra angle that surfaced the same spec (capped),
+ *    because a spec found by both the broad query and a type-filtered angle is
+ *    more likely to be the one the planner wants.
  */
-function computeBM25(queryTerms, spec, corpus) {
-  let score = 0;
-  const { avgDocLength, totalDocs, documentFrequency } = corpus;
-
-  for (const term of queryTerms) {
-    const tf = spec.termFrequency.get(term) || 0;
-    if (tf === 0) continue;
-
-    const df = documentFrequency.get(term) || 0;
-    // IDF with smoothing to handle rare terms
-    const idf = Math.log((totalDocs - df + 0.5) / (df + 0.5) + 1);
-
-    // Length-normalized term frequency
-    const lengthNorm = 1 - BM25_B + BM25_B * (spec.docLength / (avgDocLength || 1));
-    const tfNorm = (tf * (BM25_K1 + 1)) / (tf + BM25_K1 * lengthNorm);
-
-    score += idf * tfNorm;
-  }
-
-  return score;
+function computeRetrievalScore(candidate) {
+  const rank = Number.isFinite(candidate?.rank) && candidate.rank > 0
+    ? Math.min(candidate.rank, 1)
+    : 0;
+  const position = Number.isFinite(candidate?.bestPosition) ? candidate.bestPosition : 0;
+  const positional = Math.max(0, 1 - position / SEARCH_ANGLE_LIMIT);
+  const angles = Array.isArray(candidate?.angles) ? candidate.angles.length : 1;
+  const agreement = Math.min(Math.max(angles - 1, 0), 3) * 0.25;
+  return rank + positional + agreement;
 }
 
 /**
@@ -408,17 +593,6 @@ function computeTitleBoost(queryTerms, spec) {
   return presentCount * 2;
 }
 
-/**
- * Backlink importance - specs linked by many others are more authoritative.
- * Uses log scale to prevent runaway scores.
- */
-function computeBacklinkScore(spec) {
-  const count = spec.backlinkCount;
-  if (count === 0) return 0;
-  // log1p handles 0 gracefully: 1 link ~= 0.69, 5 ~= 1.79, 10 ~= 2.40, 30 ~= 3.43
-  return Math.min(Math.log1p(count), 3);
-}
-
 /**
  * Recency score - recently updated specs are more relevant.
  * Decay function: full score for last 7 days, decays over 90 days.
@@ -490,136 +664,59 @@ function computeAreaScore(areas, spec) {
 }
 
 /**
- * Enhanced scoreSpec using TF-IDF index.
- * Returns combined score with all factors.
+ * Rerank ONE candidate from the bounded retrieval window. The endpoint already
+ * did the lexical work (`ts_rank_cd` over a weighted tsvector); this adds the
+ * signals that do not live in the tsvector — the caller's explicit tags, the
+ * code area, the spec's status and how recently it moved.
+ *
+ * `candidate` is the shape {@link mergeAngles} mints: `{ name, status, tags,
+ * relPath, updatedAt, rank, bestPosition, angles }`. Tags / relPath / updatedAt
+ * are present only for rows the bounded metadata read covered; the untouched
+ * defaults (`[]`, `''`, `0`) make every scorer total, so a row outside
+ * {@link ENRICH_CAP} scores on recall + status rather than erroring.
  */
-function scoreSpecWithIndex(queryTokens, explicitTags, areas, spec, corpus) {
-  const now = Date.now();
+function scoreCandidate(queryTokens, explicitTags, areas, candidate, now = Date.now()) {
   const reasons = [];
 
-  // Component scores
-  const bm25 = computeBM25(queryTokens, spec, corpus);
-  const titleBoost = computeTitleBoost(queryTokens, spec);
-  const tagScore = computeTagScore(queryTokens, explicitTags, spec);
-  const areaScore = computeAreaScore(areas, spec);
-  const statusBoost = computeStatusBoost(spec.status);
-  const backlinkScore = computeBacklinkScore(spec);
-  const recencyScore = computeRecencyScore(spec, now);
-
-  // Build reasons for debugging/display
-  if (bm25 > 0.5) reasons.push(`tfidf:${bm25.toFixed(1)}`);
+  const recall = computeRetrievalScore(candidate);
+  const titleBoost = computeTitleBoost(queryTokens, candidate);
+  const tagScore = computeTagScore(queryTokens, explicitTags, candidate);
+  const areaScore = computeAreaScore(areas, candidate);
+  const statusBoost = computeStatusBoost(candidate.status);
+  const recencyScore = computeRecencyScore(candidate, now);
+
+  // Build reasons for debugging/display — these become `why_matched` on the card.
+  if (candidate.rank > 0) reasons.push(`rank:${candidate.rank.toFixed(3)}`);
+  else if (recall > 0) reasons.push(`recall:${recall.toFixed(2)}`);
+  if (Array.isArray(candidate.angles) && candidate.angles.length > 1) {
+    reasons.push(`angles:${candidate.angles.length}`);
+  }
   if (titleBoost >= 5) reasons.push(`title:${titleBoost}`);
   if (tagScore > 0) reasons.push(`tag:${tagScore}`);
-  if (backlinkScore > 1) reasons.push(`links:${spec.backlinkCount}`);
   if (recencyScore > 1) reasons.push('recent');
-  if (areaScore > 0) reasons.push(`area`);
+  if (areaScore > 0) reasons.push('area');
 
   // Weighted combination
   const score =
-    (bm25 * SCORE_WEIGHTS.tfidf) +
+    (recall * SCORE_WEIGHTS.rank) +
     (titleBoost * SCORE_WEIGHTS.title) +
     (tagScore * SCORE_WEIGHTS.tag) +
     (areaScore * SCORE_WEIGHTS.area) +
     (statusBoost * SCORE_WEIGHTS.status) +
-    (backlinkScore * SCORE_WEIGHTS.backlink) +
     (recencyScore * SCORE_WEIGHTS.recency);
 
   return {
     score,
     reasons,
-    bm25,
+    recall,
     titleBoost,
     tagScore,
     areaScore,
     statusBoost,
-    backlinkScore,
     recencyScore,
   };
 }
 
-function parseFrontmatter(content) {
-  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
-  if (!match) return { frontmatter: {}, body: content };
-  const yamlBlock = match[1];
-  const body = content.slice(match[0].length).trim();
-  return { frontmatter: parseSimpleYaml(yamlBlock), body };
-}
-
-function parseSimpleYaml(yaml) {
-  const result = {};
-  const lines = yaml.split(/\r?\n/);
-  let currentArray = null;
-  // Block-scalar collection state (`key: |` / `key: >`). The vast majority of
-  // authored `summary:` values in this vault are literal block scalars, so the
-  // brief consumer must read them as strings — not drop them — to agree with the
-  // index consumer (which uses a real YAML parser). [[HS-TASK-ECOI]]
-  let blockKey = null;
-  let blockLines = null;
-  let blockFold = false;
-
-  const flushBlock = () => {
-    if (blockKey === null) return;
-    // Strip the common leading indentation of the block's content lines, then
-    // join: literal (`|`) keeps line breaks; folded (`>`) joins with spaces.
-    const contentful = blockLines.filter((l) => l.trim() !== '');
-    const minIndent = contentful.length
-      ? Math.min(...contentful.map((l) => l.match(/^(\s*)/)[1].length))
-      : 0;
-    const stripped = blockLines.map((l) => l.slice(minIndent));
-    result[blockKey] = blockFold
-      ? stripped.map((l) => l.trim()).filter(Boolean).join(' ')
-      : stripped.join('\n').replace(/\n+$/, '');
-    blockKey = null;
-    blockLines = null;
-    blockFold = false;
-  };
-
-  for (const line of lines) {
-    if (blockKey !== null) {
-      // Blank or indented lines belong to the active block scalar; a line at the
-      // key column (non-indented, non-blank) ends it and is parsed normally.
-      if (line.trim() === '' || /^\s/.test(line)) {
-        blockLines.push(line);
-        continue;
-      }
-      flushBlock();
-    }
-
-    if (!line.trim() || line.trim().startsWith('#')) continue;
-
-    const arrayItemMatch = line.match(/^(\s+)-\s*(.*)$/);
-    if (arrayItemMatch && currentArray) {
-      const value = arrayItemMatch[2].trim().replace(/^["']|["']$/g, '');
-      currentArray.push(value);
-      continue;
-    }
-
-    const keyMatch = line.match(/^(\w[\w_-]*):\s*(.*)$/);
-    if (keyMatch) {
-      const key = keyMatch[1];
-      const value = keyMatch[2].trim();
-      currentArray = null;
-
-      if (/^[|>][-+\d]*$/.test(value)) {
-        // Block scalar: `|`, `>`, or a header variant (`|-`, `>+`, `|2`, …).
-        blockKey = key;
-        blockLines = [];
-        blockFold = value.startsWith('>');
-        result[key] = '';
-      } else if (value === '') {
-        result[key] = [];
-        currentArray = result[key];
-      } else if (value.startsWith('[') && value.endsWith(']')) {
-        result[key] = value.slice(1, -1).split(',').map((s) => s.trim().replace(/^["']|["']$/g, ''));
-      } else {
-        result[key] = value.replace(/^["']|["']$/g, '');
-      }
-    }
-  }
-  flushBlock();
-  return result;
-}
-
 function tokenize(text) {
   if (!text) return [];
   return text
@@ -629,123 +726,12 @@ function tokenize(text) {
     .filter((t) => t.length > 2 && !STOPWORDS.has(t));
 }
 
-function walkMdFiles(dir, out = []) {
-  if (!existsSync(dir)) return out;
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    if (SKIP_DIRS.has(entry.name)) continue;
-    const full = join(dir, entry.name);
-    if (entry.isDirectory()) walkMdFiles(full, out);
-    else if (entry.name.endsWith('.md')) out.push(full);
-  }
-  return out;
-}
-
-function resolveSpecPath(vaultRoot, anchor) {
-  const specsRoot = join(vaultRoot, 'specs');
-  const candidates = [
-    join(specsRoot, `${anchor.basename}.md`),
-    anchor.subdir ? join(specsRoot, anchor.subdir, `${anchor.basename}.md`) : null,
-  ].filter(Boolean);
-
-  for (const p of candidates) {
-    if (existsSync(p)) return relative(vaultRoot, p).replace(/\\/g, '/');
-  }
-
-  const all = walkMdFiles(specsRoot);
-  const found = all.find((f) => basename(f, '.md') === anchor.basename);
-  return found ? relative(vaultRoot, found).replace(/\\/g, '/') : null;
-}
-
 function normalizeTags(tags) {
   if (!tags) return [];
   if (Array.isArray(tags)) return tags.map((t) => String(t).toLowerCase());
   return [String(tags).toLowerCase()];
 }
 
-/**
- * Score a spec using the TF-IDF index if available, otherwise fall back to simple scoring.
- * This function bridges the old API (file-based) with the new index-based scoring.
- */
-function scoreSpec(filePath, content, tokens, explicitTags, areas, index = null) {
-  const fileBase = basename(filePath, '.md');
-  const key = fileBase.toLowerCase();
-
-  // If we have an index, use enhanced TF-IDF scoring
-  if (index && index.specs.has(key)) {
-    const spec = index.specs.get(key);
-    const result = scoreSpecWithIndex(tokens, explicitTags, areas, spec, index.corpus);
-
-    return {
-      score: result.score,
-      reason: result.reasons.slice(0, 4).join(', ') || 'low relevance',
-      frontmatter: {
-        id: spec.id,
-        name: spec.name,
-        type: spec.type,
-        status: spec.status,
-        tags: spec.tags,
-        summary: spec.summary ?? null,
-      },
-      basename: spec.basename,
-      status: spec.status,
-      // Extra debug info for index-based scoring
-      _indexed: true,
-      _backlinkCount: spec.backlinkCount,
-      _bm25: result.bm25,
-    };
-  }
-
-  // Fallback: simple scoring for unindexed files (should be rare)
-  const { frontmatter, body } = parseFrontmatter(content);
-  const name = String(frontmatter.name || frontmatter.title || fileBase).toLowerCase();
-  const specTags = normalizeTags(frontmatter.tags);
-  const status = String(frontmatter.status || 'open').toLowerCase();
-
-  let score = 0;
-  const reasons = [];
-
-  for (const tok of tokens) {
-    if (name.includes(tok)) {
-      score += 2;
-      reasons.push(`title:${tok}`);
-    }
-    const bodyHits = (body.toLowerCase().match(new RegExp(`\\b${tok}\\b`, 'g')) || []).length;
-    if (bodyHits > 0) {
-      score += Math.min(bodyHits, 3);
-      reasons.push(`body:${tok}`);
-    }
-  }
-
-  for (const tag of explicitTags) {
-    if (specTags.includes(tag.toLowerCase())) {
-      score += 3;
-      reasons.push(`tag:${tag}`);
-    }
-  }
-
-  for (const area of areas) {
-    const boost = AREA_BOOSTS[area];
-    if (!boost) continue;
-    const pathLower = filePath.replace(/\\/g, '/').toLowerCase();
-    if (boost.paths.some((p) => pathLower.includes(p)) || boost.tags.some((t) => specTags.includes(t))) {
-      score += 2;
-      reasons.push(`area:${area}`);
-    }
-  }
-
-  if (status === 'open' || status === 'in_progress') {
-    score += 1;
-  }
-
-  return {
-    score,
-    reason: reasons.slice(0, 4).join(', ') || 'low relevance',
-    frontmatter,
-    basename: fileBase,
-    status,
-    _indexed: false,
-  };
-}
 
 function detectPillars(text) {
   const lower = text.toLowerCase();
@@ -767,32 +753,6 @@ function detectSurfaces(text, areas) {
   return [...surfaces];
 }
 
-function findLargeMarkdownFiles(vaultRoot, limit = 8) {
-  const roots = [vaultRoot];
-  const large = [];
-
-  for (const root of roots) {
-    if (!existsSync(root)) continue;
-    const files = walkMdFiles(root);
-    for (const f of files) {
-      if (f.includes('node_modules')) continue;
-      try {
-        const content = readFileSync(f, 'utf8');
-        const lines = content.split(/\r?\n/).length;
-        if (lines >= 500) {
-          large.push({
-            path: relative(vaultRoot, f).replace(/\\/g, '/'),
-            lines,
-          });
-        }
-      } catch {
-        /* skip */
-      }
-    }
-  }
-
-  return large.sort((a, b) => b.lines - a.lines).slice(0, limit);
-}
 
 function buildCodebaseHints(topic, summary, areas, tokens) {
   const queries = [...new Set([...tokens.slice(0, 6), topic].filter(Boolean))];
@@ -811,23 +771,6 @@ function buildCodebaseHints(topic, summary, areas, tokens) {
   };
 }
 
-function guessManifestPhase(vaultRoot, tokens) {
-  const roadmapPath = join(vaultRoot, 'specs/roadmaps/HS-ROADMAP-product-master.md');
-  if (!existsSync(roadmapPath) || tokens.length === 0) return null;
-
-  try {
-    const content = readFileSync(roadmapPath, 'utf8').toLowerCase();
-    const phases = content.match(/phase[_\s-]*id[:\s]+["']?(P\d+)/gi) || [];
-    if (phases.length === 0) return 'See [[HS-ROADMAP-product-master]] — agent should read manifest';
-    for (const tok of tokens) {
-      if (content.includes(tok)) return 'Match found in product roadmap — read manifest for phase';
-    }
-  } catch {
-    return null;
-  }
-  return null;
-}
-
 function buildPlanContextSection(topic, landscape, hooks, placement, suggestedRelated, conflictAnalysis) {
   const lines = [
     '## Grounding',
@@ -927,50 +870,6 @@ function buildOptimizedOutline(topic) {
   ].join('\n');
 }
 
-/**
- * Extract a one-line summary from a spec's markdown: the first meaningful prose
- * line after the frontmatter block, with heading/callout/link syntax stripped
- * and capped at `maxLen` chars. Used to build body-free candidate cards — the
- * LLM triages on these, then deep-reads only the shortlist it selects
- * ([[retrieval-intake-architecture]] §How the LLM-as-reranker works).
- * @param {string} content
- * @param {number} [maxLen]
- * @returns {string}
- */
-function extractSummaryLine(content, maxLen = 140) {
-  let body = String(content || '');
-  // Drop a leading YAML frontmatter block.
-  if (body.startsWith('---')) {
-    const close = body.indexOf('\n---', 3);
-    if (close !== -1) {
-      const nl = body.indexOf('\n', close + 1);
-      body = nl !== -1 ? body.slice(nl + 1) : '';
-    }
-  }
-  const cap = (s) => (s.length > maxLen ? `${s.slice(0, maxLen - 1).trimEnd()}…` : s);
-  let firstHeading = ''; // fallback if the doc is heading-only (title echoes anyway)
-  for (let raw of body.split('\n')) {
-    let line = raw.trim();
-    if (!line) continue;
-    if (line.startsWith('|') || line.startsWith('<!--') || line.startsWith('---') || line.startsWith('```')) continue;
-    const isHeading = /^#{1,6}\s/.test(line);
-    line = line.replace(/^#{1,6}\s+/, ''); // de-head
-    line = line.replace(/^>+\s?/, '').replace(/^\[!\w+\][-+]?\s*/, ''); // de-callout
-    line = line.replace(/\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g, '$1'); // [[wikilink]] → text
-    line = line.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1'); // [md](link) → text
-    line = line.replace(/[*_`]/g, '').trim(); // strip emphasis
-    if (!line) continue;
-    // Prefer the first prose/callout line; the leading H1 usually just echoes
-    // the title, so skip headings unless that is all the doc has.
-    if (isHeading) {
-      if (!firstHeading) firstHeading = line;
-      continue;
-    }
-    return cap(line);
-  }
-  return cap(firstHeading);
-}
-
 /**
  * @param {string} vaultRoot
  * @param {object} args
@@ -999,7 +898,7 @@ function buildGroundReport(ctx) {
 
     L.push('', '## 1. Scope');
     L.push(`- Workspace: \`${basename(vaultRoot)}\` · surfaces: ${surfaces.join(', ') || '—'} · pillars: ${pillarIds.join(', ') || '—'}`);
-    L.push(`- Index: in-memory lexical (BM25 + signals), built per call · grounded ${groundedAt}`);
+    L.push(`- Retrieval: bounded lexical search via \`GET /api/search\` (top-k per angle, reranked) · grounded ${groundedAt}`);
 
     L.push('', '## 2. Relevant specs');
     L.push('_Lexical wide-recall — YOU are the reranker. Open only the shortlist you judge relevant; do not read them all._');
@@ -1057,7 +956,7 @@ function buildGroundReport(ctx) {
   return { report_markdown: report, token_estimate: estTokens(report), truncated };
 }
 
-export function executeGroundPlan(vaultRoot, args = {}) {
+export async function executeGroundPlan(vaultRoot, args = {}, { fetchImpl = defaultConsoleFetch } = {}) {
   const topic = String(args.topic || '').trim();
   if (!topic) {
     return { error: 'MISSING_TOPIC', message: 'topic is required (short title of feature/request)' };
@@ -1066,10 +965,7 @@ export function executeGroundPlan(vaultRoot, args = {}) {
   const summary = String(args.summary || '').trim();
   const explicitTags = normalizeTags(args.tags || []);
   const areas = Array.isArray(args.areas)
-    ? args.areas.map((a) => {
-        const v = String(a).toLowerCase();
-        return v === 'cockpit' ? 'console' : v; // legacy-cockpit area alias
-      })
+    ? args.areas.map((a) => String(a).toLowerCase())
     : [];
   // Wide-recall window: the user's own LLM is the reranker, so HeySpex retrieves
   // WIDE (Recall@N is the metric, not nDCG@10 ordering). Default 30, max 50 — per
@@ -1085,17 +981,65 @@ export function executeGroundPlan(vaultRoot, args = {}) {
   const combinedText = `${topic} ${summary}`;
   const tokens = [...new Set([...tokenize(topic), ...tokenize(summary)])];
 
-  // Build or retrieve cached TF-IDF index for enhanced scoring
-  const index = getOrBuildIndex(vaultRoot);
+  // ── Retrieval: a fixed set of bounded angles against /api/search ───────────
+  // The query text the endpoint parses (`websearch_to_tsquery`) is the topic and
+  // summary as typed; the type-filtered angles guarantee the buckets a plan needs
+  // (governing protocols, memory decisions/lessons) are represented even when the
+  // broad angle is crowded by epics and tasks. Every call is `?limit=` bounded.
+  const searchQuery = combinedText.trim() || topic;
+  const tagQuery = explicitTags.join(' ').trim();
+  const angleSpecs = [
+    { label: 'broad', q: searchQuery, want: limit },
+    { label: 'work', q: searchQuery, want: 10, type: WORK_TYPE_FILTER },
+    { label: 'protocols', q: searchQuery, want: 8, type: PROTOCOL_TYPE_FILTER },
+    { label: 'memories', q: searchQuery, want: 8, type: MEMORY_TYPE_FILTER },
+    ...(tagQuery && tagQuery !== searchQuery
+      ? [{ label: 'tags', q: tagQuery, want: 10 }]
+      : []),
+  ];
+
+  let angles;
+  try {
+    angles = await Promise.all(angleSpecs.map((spec) => fetchSearchAngle(fetchImpl, spec)));
+  } catch (err) {
+    // Retrieval is remote and there is NO disk fallback ([[no-fallbacks-by-default]]):
+    // a grounding report built from a stale local walk is exactly the failure this
+    // task removed. Say the console is unreachable and stop.
+    return {
+      error: err instanceof ConsoleRetrievalError ? err.code : 'CONSOLE_UNREACHABLE',
+      message:
+        `Pre-plan grounding retrieves through the console search endpoint (GET /api/search) and it did not answer (${err?.message ?? err}). `
+        + 'Call heyspex_dev_server_status before concluding the console is down, then re-run heyspex_ground_plan. '
+        + 'Do NOT hand-walk the vault to substitute for grounding.',
+      topic,
+    };
+  }
 
-  const anchors = ANCHOR_SPECS.map((a) => {
-    const path = resolveSpecPath(vaultRoot, a);
+  // ── Rerank: two passes over the SAME bounded window ────────────────────────
+  // Pass 1 ranks on what every row carries (rank/position/status/title) so the
+  // enrichment cap lands on the rows most likely to matter; pass 2 re-scores with
+  // the tags and `updated` stamps the bounded metadata read added.
+  const candidates = mergeAngles(angles);
+  const rank = (list) => {
+    const now = Date.now();
+    for (const c of list) c._scored = scoreCandidate(tokens, explicitTags, areas, c, now);
+    list.sort((a, b) => b._scored.score - a._scored.score || a.id.localeCompare(b.id));
+    return list;
+  };
+  rank(candidates);
+  await enrichCandidates(fetchImpl, candidates, ENRICH_CAP);
+  rank(candidates);
+
+  // Anchors: the small static constitution list. One bounded resolve per anchor
+  // (5 calls, never a folder scan) so `found` still tells the truth per vault.
+  const anchors = await mapWithConcurrency(ANCHOR_SPECS, async (a) => {
+    const resolved = await resolveSpecRef(fetchImpl, a.basename);
     return {
-      basename: a.basename,
-      path: path || `specs/${a.subdir ? `${a.subdir}/` : ''}${a.basename}.md`,
+      basename: resolved?.basename || a.basename,
+      path: resolved?.relPath || `specs/${a.subdir ? `${a.subdir}/` : ''}${a.basename}.md`,
       why: a.why,
       tier: 'anchor',
-      found: Boolean(path),
+      found: Boolean(resolved),
     };
   });
 
@@ -1113,54 +1057,29 @@ export function executeGroundPlan(vaultRoot, args = {}) {
     future_considerations: [],
   };
 
-  const scored = [];
-
-  for (const { rel, bucket } of SCAN_DIRS) {
-    const dir = join(vaultRoot, rel);
-    const files = walkMdFiles(dir);
-    for (const filePath of files) {
-      try {
-        const content = readFileSync(filePath, 'utf8');
-        const { score, reason, frontmatter, basename: fileBase, status } = scoreSpec(
-          filePath,
-          content,
-          tokens,
-          explicitTags,
-          areas,
-          index, // Pass index for TF-IDF scoring
-        );
-        if (score <= 0 && bucket !== 'memories') continue;
-
-        const entry = {
-          basename: fileBase,
-          id: frontmatter.id || fileBase,
-          name: frontmatter.name || fileBase,
-          path: relative(vaultRoot, filePath).replace(/\\/g, '/'),
-          status,
-          relevance: score,
-          reason,
-          type: frontmatter.type || bucket,
-          // Authored frontmatter.summary, carried so the candidate card can
-          // prefer it over the body-derived extractSummaryLine() fallback
-          // (same precedence as the index/brief consumers: authored wins).
-          summary: typeof frontmatter.summary === 'string' ? frontmatter.summary : null,
-        };
-
-        scored.push({ bucket, entry });
-
-        if (discovered[bucket]) {
-          discovered[bucket].push(entry);
-        }
-      } catch {
-        /* skip unreadable */
-      }
-    }
-  }
-
-  scored.sort((a, b) => b.entry.relevance - a.entry.relevance);
+  const scored = candidates.map((c) => {
+    const bucket = TYPE_BUCKETS[c.type] || null;
+    const entry = {
+      basename: c.basename,
+      id: c.id,
+      name: c.name,
+      // Known only for rows the bounded metadata read covered; `null` elsewhere —
+      // a candidate is addressed by id, never by a path this module guessed.
+      path: c.relPath || null,
+      status: c.status,
+      relevance: c._scored.score,
+      reason: c._scored.reasons.slice(0, 4).join(', ') || 'low relevance',
+      type: c.type,
+      tags: c.tags,
+      // Authored `summary:` when the metadata read returned one; the endpoint's
+      // `ts_headline` snippet otherwise (same job — a body-free triage line).
+      summary: c.summary || c.snippet || null,
+    };
+    if (bucket && discovered[bucket]) discovered[bucket].push(entry);
+    return { bucket, entry };
+  });
 
   for (const key of Object.keys(discovered)) {
-    discovered[key].sort((a, b) => b.relevance - a.relevance);
     discovered[key] = discovered[key].slice(0, limit);
   }
 
@@ -1183,18 +1102,12 @@ export function executeGroundPlan(vaultRoot, args = {}) {
   // one-line summary, and the why-matched lexical signals. Bodies are NOT
   // included — the agent deep-reads only the shortlist it selects (two-stage).
   const candidateCards = topEntries.map((e) => {
-    // Precedence: authored frontmatter.summary wins over the body-derived
-    // extractSummaryLine() fallback, matching the index (U2) and /hs-next brief
-    // consumers so the same spec reads identically across surfaces. The body
-    // fallback is preserved byte-for-byte for specs with no authored summary.
-    let summary = e.summary && String(e.summary).trim() ? String(e.summary).trim() : '';
-    if (!summary) {
-      try {
-        summary = extractSummaryLine(readFileSync(join(vaultRoot, e.path), 'utf8'));
-      } catch {
-        /* unreadable — leave summary blank */
-      }
-    }
+    // Precedence: authored frontmatter.summary (from the bounded metadata read)
+    // wins over the endpoint's `ts_headline` snippet, matching the index (U2) and
+    // /hs-next brief consumers so the same spec reads identically across surfaces.
+    // The old body-derived fallback is gone with the file walk — the snippet is
+    // the endpoint's own answer to the same question, computed over the page only.
+    const summary = e.summary && String(e.summary).trim() ? String(e.summary).trim() : '';
     return {
       id: e.id,
       basename: e.basename,
@@ -1231,8 +1144,13 @@ export function executeGroundPlan(vaultRoot, args = {}) {
   const possibleDuplicates = openWork.filter((e) => e.relevance >= 4).slice(0, 5);
 
   const protocols = discovered.protocols.slice(0, 8);
+  // Decisions and lessons are the memories that CONSTRAIN a plan (references and
+  // ui-patterns inform it). Selected on the row's `type:` — the folder-path test
+  // this replaces needed a walked path, and `path` is now null for any row the
+  // bounded metadata read did not cover.
   const decisionsToHeed = discovered.memories
-    .filter((m) => m.path.includes('/decisions/') || m.path.includes('/lessons/'))
+    .filter((m) => m.type === 'memory-decision' || m.type === 'memory-lesson'
+      || (m.path || '').includes('/decisions/') || (m.path || '').includes('/lessons/'))
     .slice(0, 8);
 
   const horizonItems = discovered.future_considerations.slice(0, 5);
@@ -1262,7 +1180,11 @@ export function executeGroundPlan(vaultRoot, args = {}) {
   const architecturePlacement = {
     surfaces,
     dependency_specs: dependencySpecs,
-    manifest_phase_hint: guessManifestPhase(vaultRoot, tokens),
+    // Retired with the file walk ([[HS-YIR1-8KK5-TASK-planning-grounding-retrieves-through]]):
+    // the hint was a substring scan of specs/roadmaps/HS-ROADMAP-product-master.md
+    // read from disk on every call, and it only ever said "go read the manifest".
+    // The roadmap is one bounded read away for an agent that wants it.
+    manifest_phase_hint: null,
   };
 
   const northStarPillarHooks = {
@@ -1343,12 +1265,13 @@ export function executeGroundPlan(vaultRoot, args = {}) {
   }
   enhancementSuggestions.push('Add child tasks for each implementation phase after stress-test.');
 
+  // `large_markdown_files` is GONE, not emptied-in-place: it walked every .md in
+  // the workspace to measure line counts, and the field itself was the
+  // read-everything anti-pattern the bounded report exists to retire
+  // ([[hs-ground-as-built-mcp-failure-modes]]).
   const codebaseHints = includeCodebaseHints
-    ? {
-        ...buildCodebaseHints(topic, summary, areas, tokens),
-        large_markdown_files: findLargeMarkdownFiles(vaultRoot),
-      }
-    : { grep_queries: [], path_globs: [], large_markdown_files: [] };
+    ? buildCodebaseHints(topic, summary, areas, tokens)
+    : { grep_queries: [], path_globs: [] };
 
   const landscapeSummary = {
     open_work: openWork.map((e) => ({ basename: e.basename, id: e.id, reason: e.reason })),
@@ -1497,7 +1420,6 @@ export function executeGroundPlan(vaultRoot, args = {}) {
         ? {
             grep_queries: codebaseHints.grep_queries?.slice(0, 5) || [],
             path_globs: codebaseHints.path_globs?.slice(0, 4) || [],
-            large_markdown_files: (codebaseHints.large_markdown_files || []).slice(0, 5),
           }
         : null,
 
@@ -1562,18 +1484,30 @@ export function executeGroundPlan(vaultRoot, args = {}) {
 
 /**
  * `heyspex_resolve_focus_intent` lexical resolver — [[HS-TASK-JZEJ]] /
- * [[HS-EPIC-OOPW]] Phase B. Given a free-text focus utterance, run lexical
- * wide-recall over the vault (titles/bodies/tags) and return a WIDE candidate
- * window for the calling agent's LLM to rerank. HeySpex hosts no reranker — the
- * metric is Recall@N, not nDCG@10 ordering
+ * [[HS-EPIC-OOPW]] Phase B. Given a free-text focus utterance, retrieve a WIDE
+ * candidate window through the console search endpoint and hand it to the
+ * calling agent's LLM to rerank. HeySpex hosts no reranker — the metric is
+ * Recall@N, not nDCG@10 ordering
  * ([[llm-as-reranker-lexical-recall-companion-architecture]]). No embeddings /
  * vector store (Gate B = NO,
  * [[retrieval-substrate-lexical-not-embeddings-gate-b-no]]); consumes the same
- * scoring substrate as hs-ground ([[HS-EPIC-G7R2-hs-ground-three-tier-grounding-report]]).
+ * bounded retrieval substrate as hs-ground
+ * ([[HS-EPIC-G7R2-hs-ground-three-tier-grounding-report]]).
+ *
+ * Retrieval is TWO bounded angle families, never a vault walk
+ * ([[HS-YIR1-8KK5-TASK-planning-grounding-retrieves-through]]):
+ *  1. `GET /api/search?q=<utterance>` — the lexical window, `?limit=`-bounded
+ *     and continued with the endpoint's cursor up to `top_k` rows.
+ *  2. For each tag of the closed vocabulary the utterance names EXACTLY
+ *     (`GET /api/scheduler/known-tags`, one page), one bounded
+ *     `GET /api/specs?tag=…` page. Those rows are the strict-tag matches, and
+ *     they arrive carrying their own tags — so the strict/lexical split is now
+ *     an indexed filter instead of a scan over every spec's frontmatter.
  *
  * The utterance need not contain a spec's exact tag word: utterance tokens match
- * tags via computeTagScore's substring pass and match titles/bodies via BM25, so
- * tag-relevant specs still surface — the core Phase-B behavior.
+ * titles/bodies through the tsvector and tags through computeTagScore's
+ * substring pass, so tag-relevant specs still surface — the core Phase-B
+ * behavior.
  *
  * Returns an empty window (NOT an error) for an empty/stopword-only utterance,
  * an empty vault, or no matches — the caller's documented fallback is
@@ -1585,11 +1519,12 @@ export function executeGroundPlan(vaultRoot, args = {}) {
  * counts (plus a ready-to-read `beat`) the confirmation copy reads back
  * ([[HS-TASK-5DB2]] / focus-intent-routing §5–6).
  *
- * @param {string} vaultRoot - Root of the vault (contains specs/).
+ * @param {string} vaultRoot - Retained for signature parity; nothing here reads it.
  * @param {{ utterance?: string, top_k?: number }} args
- * @returns {{ utterance: string, candidates: Array<{ spec: string, tags: string[], match_kind: 'strict-tag' | 'lexical', why: string }>, total: number, justification: { strict_tag_matches: number, lexically_adjacent: number, beat: string } }}
+ * @param {{ fetchImpl?: Function }} [deps] - Console transport (stubbed in tests).
+ * @returns {Promise<{ utterance: string, candidates: Array<{ spec: string, tags: string[], match_kind: 'strict-tag' | 'lexical', why: string }>, total: number, justification: { strict_tag_matches: number, lexically_adjacent: number, beat: string } }>}
  */
-export function resolveFocusIntentLexical(vaultRoot, args = {}) {
+export async function resolveFocusIntentLexical(vaultRoot, args = {}, { fetchImpl = defaultConsoleFetch } = {}) {
   const utterance = String(args.utterance || '').trim();
   // Wide-recall window: default 30, capped at 50 — same Recall@N bounds as
   // executeGroundPlan's limit. top_k narrows the window when the caller asks.
@@ -1600,64 +1535,97 @@ export function resolveFocusIntentLexical(vaultRoot, args = {}) {
     return { utterance, candidates: [], total: 0, justification: buildFocusJustification(0, 0) };
   }
 
-  // Graceful degradation ([[HS-TASK-09I3]] / focus-intent-routing §6). The
-  // lexical recall primitive can be unreachable in a constrained environment —
-  // no vault index, an IO error while walking specs/, a corpus that fails to
-  // load. When the index build or scoring throws, the resolver must NOT raise:
-  // it degrades to an empty window flagged `degraded`, routing the caller to the
-  // Phase-A exact-tag NLU floor (heyspex_list_known_tags + closed-vocab mapping)
-  // instead of erroring out. An empty *result* (no matches, empty vault) is a
-  // success and is handled by the normal path below — only a thrown primitive
-  // trips degradation.
-  let index;
-  try {
-    index = getOrBuildIndex(vaultRoot);
-  } catch (err) {
-    return buildDegradedFocusResult(utterance, err);
-  }
-
   // A candidate is a STRICT-TAG match when the user named one of its tags
   // exactly — every token of some tag is present in the utterance (so multi-word
   // tags like `focus-areas` match "focus areas"). Everything else that still
   // surfaced matched on title/body/substring only — "lexically adjacent". The
   // split feeds the confirmation copy ([[HS-TASK-5DB2]]).
   const utteranceSet = new Set(tokens);
-  const isStrictTagHit = (specTags) =>
-    (specTags || []).some((tag) => {
-      const t = String(tag).toLowerCase();
-      if (utteranceSet.has(t)) return true;
-      const parts = tokenize(t);
-      return parts.length > 0 && parts.every((p) => utteranceSet.has(p));
-    });
+  const namesTagExactly = (tag) => {
+    const t = String(tag || '').toLowerCase();
+    if (!t) return false;
+    if (utteranceSet.has(t)) return true;
+    const parts = tokenize(t);
+    return parts.length > 0 && parts.every((p) => utteranceSet.has(p));
+  };
+  const isStrictTagHit = (specTags) => (specTags || []).some(namesTagExactly);
 
+  // Graceful degradation ([[HS-TASK-09I3]] / focus-intent-routing §6). The
+  // recall primitive is now REMOTE, so the unreachable case is a dead console,
+  // a 401, or a 5xx rather than an IO error walking specs/. The resolver must
+  // NOT raise: it degrades to an empty window flagged `degraded`, routing the
+  // caller to the Phase-A exact-tag NLU floor (heyspex_list_known_tags +
+  // closed-vocab mapping). An empty *result* (no matches, empty vault) is a
+  // success and is handled by the normal path below.
   let candidates;
   try {
-    const scored = [];
-    for (const spec of index.specs.values()) {
-      // No explicit tags / areas from a free-text utterance — the utterance
-      // tokens carry the tag matching (computeTagScore substring pass) and the
-      // lexical hit.
-      const result = scoreSpecWithIndex(tokens, [], [], spec, index.corpus);
-      // Gate on the LEXICAL signal (BM25 + title + tag), not the full weighted
-      // score: status/backlink/recency boosts are unconditional, so the full
-      // score is > 0 for every open spec even with no term match. A candidate
-      // must actually match the utterance; the boosts only tie-break among
-      // matches.
-      const lexical = result.bm25 + result.titleBoost + result.tagScore;
-      if (lexical <= 0) continue;
-      scored.push({ spec, result });
-    }
+    // Angle 1 — the lexical window, bounded per call and cursor-paged to top_k.
+    const lexicalAngle = await fetchSearchAngle(fetchImpl, {
+      label: 'utterance',
+      q: utterance,
+      want: topK,
+    });
 
-    // Deterministic order: score desc, basename asc as a stable tiebreak.
-    scored.sort(
-      (a, b) => b.result.score - a.result.score || a.spec.basename.localeCompare(b.spec.basename),
+    // Angle 2 — strict-tag lookup over the closed vocabulary. Best-effort: a
+    // known-tags failure costs the strict/lexical distinction, not the window.
+    const knownTags = await fetchKnownTags(fetchImpl);
+    const namedTags = knownTags.filter(namesTagExactly).slice(0, 3);
+    const tagPages = await Promise.all(
+      namedTags.map((tag) => fetchTagPage(fetchImpl, tag, Math.min(topK, SEARCH_ANGLE_LIMIT))),
     );
 
-    candidates = scored.slice(0, topK).map(({ spec, result }) => ({
-      spec: spec.id,
-      tags: spec.tags,
-      match_kind: isStrictTagHit(spec.tags) ? 'strict-tag' : 'lexical',
-      why: result.reasons.slice(0, 4).join(', ') || 'low relevance',
+    const byId = new Map(mergeAngles([lexicalAngle]).map((c) => [c.id, c]));
+    // Fold the tag rows in: they carry tags/basename/updated already, so they
+    // both enrich a row the search angle found and add rows it did not.
+    tagPages.forEach((rows, tagIndex) => {
+      rows.forEach((row, position) => {
+        const id = String((row && row.id) || '').trim();
+        if (!id) return;
+        let candidate = byId.get(id);
+        if (!candidate) {
+          candidate = {
+            id,
+            basename: id,
+            name: String(row.name || id),
+            type: String(row.type || 'unknown'),
+            status: String(row.status || 'open').toLowerCase(),
+            rank: 0,
+            snippet: '',
+            bestPosition: position,
+            angles: [],
+            tags: [],
+            relPath: '',
+            updatedAt: 0,
+            summary: null,
+            enriched: false,
+          };
+          byId.set(id, candidate);
+        }
+        const label = `tag:${namedTags[tagIndex]}`;
+        if (!candidate.angles.includes(label)) candidate.angles.push(label);
+        candidate.bestPosition = Math.min(candidate.bestPosition, position);
+        applySpecMetadata(candidate, row);
+      });
+    });
+
+    const window = [...byId.values()];
+    const now = Date.now();
+    const rescore = () => {
+      for (const c of window) c._scored = scoreCandidate(tokens, [], [], c, now);
+      window.sort((a, b) => b._scored.score - a._scored.score || a.basename.localeCompare(b.basename));
+    };
+    // Enrich the top of the window so `tags` are real for the rows the caller
+    // will actually read back, bounded by ENRICH_CAP (constant, never N).
+    rescore();
+    await enrichCandidates(fetchImpl, window.filter((c) => !c.enriched), ENRICH_CAP);
+    // Deterministic order: score desc, basename asc as a stable tiebreak.
+    rescore();
+
+    candidates = window.slice(0, topK).map((c) => ({
+      spec: c.id,
+      tags: c.tags,
+      match_kind: isStrictTagHit(c.tags) ? 'strict-tag' : 'lexical',
+      why: c._scored.reasons.slice(0, 4).join(', ') || 'low relevance',
     }));
   } catch (err) {
     return buildDegradedFocusResult(utterance, err);
@@ -1877,22 +1845,23 @@ function detectTagDrift(tags, options = {}) {
 
 // Export scoring functions for unit testing (HS-TASK-9HIW)
 export {
-  computeBM25,
+  computeRetrievalScore,
   computeTitleBoost,
-  computeBacklinkScore,
   computeRecencyScore,
   computeStatusBoost,
   computeTagScore,
   computeAreaScore,
-  scoreSpecWithIndex,
+  scoreCandidate,
   tokenize,
   detectTagDrift,
   extractWikilinks,
   normalizeWikilinkTarget,
   parseIsoDate,
+  ConsoleRetrievalError,
   SCORE_WEIGHTS,
-  BM25_K1,
-  BM25_B,
+  SEARCH_ANGLE_LIMIT,
+  MAX_PAGES_PER_ANGLE,
+  ENRICH_CAP,
   STOPWORDS,
   AREA_BOOSTS,
 };
@@ -1913,7 +1882,7 @@ export function buildGroundWorkflowMarkdown(options = {}) {
     '',
     '1. Call MCP tool **`heyspex_ground_plan`** with `topic`, `summary`, `tags`, `areas`.',
     '2. Call **`heyspex_recall_memories`** (`limit: 10`, tags from tool `recall_memories_hint`).',
-    '3. Read **all anchors** + top 5–8 discovered specs; any `.md` ≥500 lines in `large_markdown_files` → read fully.',
+    '3. Read **all anchors** + only the shortlist you rerank out of `candidate_cards` — never every discovered spec.',
     '4. Explore codebase via `codebase_hints` (`rg`, Glob) — not README-only ([[audit-read-plan-files-not-just-readmes]]).',
     '5. **Deliverable:** `## Landscape scan` — table of specs/code with relevance; duplicates and surprises.',
     '',