@tpsdev-ai/flair 0.21.0 → 0.22.1

package/dist/resources/Relationship.js

@@ -2,7 +2,10 @@ import { databases } from "@harperfast/harper";
 import { resolveAgentAuth, allowVerified } from "./agent-auth.js";
 import { checkRateLimit, rateLimitResponse } from "./rate-limiter.js";
 import { localInstanceId } from "./instance-identity.js";
+import { buildProvenance } from "./provenance.js";
 const NOT_FOUND = () => new Response(JSON.stringify({ error: "not found" }), { status: 404, headers: { "Content-Type": "application/json" } });
+const UNAUTH = () => new Response(JSON.stringify({ error: "authentication required" }), { status: 401, headers: { "Content-Type": "application/json" } });
+const FORBIDDEN = (msg) => new Response(JSON.stringify({ error: msg }), { status: 403, headers: { "Content-Type": "application/json" } });
 /**
  * Relationship resource — entity-to-entity relationships with temporal validity.
  *
@@ -86,18 +89,73 @@ export class Relationship extends databases.flair.Relationship {
             operator: "and",
         });
     }
+    /**
+     * ─── Auth reconcile (relationship-write-path, folded K&S refinement) ──────
+     *
+     * Upgraded from the older `request.tpsAgent`-direct pattern to
+     * `resolveAgentAuth` — matching Memory.post()/put() — so this write path
+     * gets the SAME three-way verdict handling (anonymous denied, verified
+     * agent stamped from the SIGNATURE never the body, internal/admin
+     * unfiltered) instead of a parallel, easy-to-drift auth mechanism. K&S both
+     * flagged this as a real divergence (the previous code had no
+     * internal/admin verdict paths at all — see the doc below for why that
+     * never actually bit anyone in practice, but was still the wrong shape to
+     * build the new ergonomic write surfaces on top of).
+     *
+     * - `anonymous` → 401, same as before.
+     * - `agent` + non-admin: a body-supplied `agentId` that MISMATCHES the
+     *   verified identity is rejected outright (403) rather than silently
+     *   overwritten — a clearer signal than Memory.post()'s "validate, don't
+     *   stamp" idiom. `content.agentId` is then ALWAYS set from `auth.agentId`
+     *   (never left as whatever the body claimed, even when it already
+     *   matched) — the non-negotiable "agentId comes from the verdict, never
+     *   the body" rule, applied unconditionally rather than only on mismatch.
+     * - `agent` + admin: `content.agentId` passes through UNFILTERED, matching
+     *   the existing admin-bypass idiom already used by get()/search()/
+     *   delete() below (an admin/migration tool may legitimately write on
+     *   another agent's behalf).
+     * - `internal` (no HTTP request at all — a trusted in-process call):
+     *   `content.agentId` also passes through unchanged. No in-process
+     *   Relationship writer exists today (openclaw's integration writes via a
+     *   real signed HTTP PUT, landing on the `agent` branch above), so this is
+     *   forward-looking parity with Memory.post()/put() rather than a path
+     *   this PR's callers actually exercise — but it closes the SAME latent gap
+     *   the old code had: `request?.tpsAgent` was falsy for BOTH an anonymous
+     *   HTTP caller and a true internal call, so an internal caller would have
+     *   been wrongly 401'd too. resolveAgentAuth distinguishes the two.
+     */
     async put(content) {
         const ctx = this.getContext?.();
-        const request = ctx?.request ?? ctx;
-        const authAgent = request?.tpsAgent;
-        if (!authAgent) {
-            return new Response(JSON.stringify({ error: "authentication required" }), {
-                status: 401, headers: { "content-type": "application/json" },
+        const auth = await resolveAgentAuth(ctx);
+        if (auth.kind === "anonymous") {
+            return UNAUTH();
+        }
+        if (auth.kind === "agent") {
+            if (!auth.isAdmin) {
+                if (content?.agentId && content.agentId !== auth.agentId) {
+                    return FORBIDDEN("cannot write a relationship owned by another agent");
+                }
+                content.agentId = auth.agentId;
+            }
+            // admin: content.agentId left as provided (unfiltered) — see doc above.
+        }
+        // internal: content.agentId left as provided (unfiltered) — see doc above.
+        if (!content.agentId || typeof content.agentId !== "string") {
+            return new Response(JSON.stringify({ error: "agentId is required" }), {
+                status: 400, headers: { "content-type": "application/json" },
             });
         }
-        const rl = checkRateLimit(authAgent);
-        if (!rl.allowed)
-            return rateLimitResponse(rl.retryAfterMs, "relationship");
+        // Rate limit keyed on the RESOLVED agentId (never a client-supplied one)
+        // — matches Memory.post()'s intent, extended to cover every
+        // resolveAgentAuth path (credentialed super_user, verifyAgentRequest
+        // fallback), not just the gate's own `tpsAgent` annotation. Internal
+        // calls have no per-agent identity to key on and are trusted, so they're
+        // exempt — same as Memory.post()'s `if (authenticatedAgent)` guard.
+        if (auth.kind === "agent") {
+            const rl = checkRateLimit(auth.agentId);
+            if (!rl.allowed)
+                return rateLimitResponse(rl.retryAfterMs, "relationship");
+        }
         // Validate required fields
         if (!content.subject || typeof content.subject !== "string") {
             return new Response(JSON.stringify({ error: "subject is required (string)" }), {
@@ -114,9 +172,9 @@ export class Relationship extends databases.flair.Relationship {
                 status: 400, headers: { "content-type": "application/json" },
             });
         }
-        // Normalize
+        // Normalize — lowercasing is load-bearing: MemoryBootstrap.ts's attention
+        // read matches lowercased predicted subjects against subject/object.
         const now = new Date().toISOString();
-        content.agentId = authAgent;
         content.subject = content.subject.toLowerCase();
         content.predicate = content.predicate.toLowerCase();
         content.object = content.object.toLowerCase();
@@ -125,6 +183,13 @@ export class Relationship extends databases.flair.Relationship {
         content.validFrom = content.validFrom || now;
         // validTo left as null/undefined for active relationships
         content.confidence = content.confidence ?? 1.0;
+        // Write-time provenance stamp (relationship-write-path, folded K&S
+        // refinement) — reuses Memory's buildProvenance EXACTLY (./provenance.ts),
+        // same `{v, verified, claimed?}` shape, no Relationship-specific format.
+        // Additive/nullable on the schema side (schemas/memory.graphql) — a
+        // pre-existing row with no provenance field reads back `undefined`,
+        // unchanged behavior (migration-equivalence gate).
+        content.provenance = buildProvenance(auth, content.createdAt, content);
         // Write-time originatorInstanceId stamp (federation-edge-hardening slice
         // 1) — see resources/Memory.ts's stampOriginatorInstanceId doc for the
         // full contract. No-op if already set (never fires for a genuine local
@@ -135,24 +200,32 @@ export class Relationship extends databases.flair.Relationship {
         }
         return super.put(content);
     }
+    /**
+     * Same auth reconcile as put() above — resolveAgentAuth replaces the
+     * `request.tpsAgent`-direct pattern. K&S both independently caught that
+     * delete() had the identical divergence the spec text only named on
+     * put(): anonymous and true-internal calls were indistinguishable (both
+     * read as a falsy `authAgent`), and there was no admin/internal verdict
+     * handling. Ownership-check logic (own-agent-or-admin) is otherwise
+     * unchanged — see test/integration/relationship-delete-authz.test.ts's doc
+     * comment for why calling `super.get()` with no target argument still
+     * resolves the URL-bound target record (a Harper Table-resource
+     * invariant), not an empty/collection result.
+     */
     async delete(_) {
         const ctx = this.getContext?.();
-        const request = ctx?.request ?? ctx;
-        const authAgent = request?.tpsAgent;
-        const isAdminAgent = request?.tpsAgentIsAdmin ?? false;
-        if (!authAgent) {
-            return new Response(JSON.stringify({ error: "authentication required" }), {
-                status: 401, headers: { "content-type": "application/json" },
-            });
+        const auth = await resolveAgentAuth(ctx);
+        if (auth.kind === "anonymous") {
+            return UNAUTH();
         }
-        // Non-admin: verify ownership before delete
-        if (!isAdminAgent) {
-            const existing = await super.get();
-            if (existing?.agentId && existing.agentId !== authAgent) {
-                return new Response(JSON.stringify({ error: "cannot delete another agent's relationship" }), {
-                    status: 403, headers: { "content-type": "application/json" },
-                });
-            }
+        // Trusted internal call or admin agent — unfiltered, unchanged behavior.
+        if (auth.kind === "internal" || (auth.kind === "agent" && auth.isAdmin)) {
+            return super.delete(_);
+        }
+        // Non-admin agent: verify ownership before delete.
+        const existing = await super.get();
+        if (existing?.agentId && existing.agentId !== auth.agentId) {
+            return FORBIDDEN("cannot delete another agent's relationship");
         }
         return super.delete(_);
     }

package/dist/resources/SemanticSearch.js

@@ -1,31 +1,26 @@
 import { Resource, databases } from "@harperfast/harper";
 import { resolveAgentAuth, allowVerified } from "./agent-auth.js";
 import { getEmbedding, getMode } from "./embeddings-provider.js";
-import { patchRecord, withDetachedTxn } from "./table-helpers.js";
+import { patchRecord } from "./table-helpers.js";
 import { checkRateLimit, rateLimitResponse } from "./rate-limiter.js";
-import { wrapUntrusted } from "./content-safety.js";
 import { resolveReadScope } from "./memory-read-scope.js";
-import { cosineSimilarity } from "./dedup.js";
 import { isRerankEnabled, getRerankTopN, getRerankBudgetMs, getRerankMinCandidates, rerankCandidates, } from "./rerank-provider.js";
-// Temporal decay + relevance scoring (incl. the OPS-AYGD retrievalBoost cap +
-// relevance floor) lives in ./scoring.ts — a Harper-free module so it can be
-// unit-tested directly (see test/unit/temporal-scoring.test.ts).
-import { compositeScore } from "./scoring.js";
-// BM25 + union-RRF hybrid retrieval (FLAIR-BM25-HYBRID-RETRIEVAL).
-// Harper-free modules so the BM25 scoring, the candidate-union RRF, and the
-// SECURITY conditions-filter are unit-tested against the shipped code.
-import { buildBM25, fuseRrfNormalized, hybridEnabled, SEM_LIMIT } from "./bm25.js";
-import { isAllowedBm25Candidate } from "./bm25-filter.js";
-// Convert HNSW cosine distance (1 - similarity) to similarity score
-function distanceToSimilarity(distance) {
-    return 1 - distance;
-}
+// The BM25 + union-RRF hybrid path is feature-flagged via hybridEnabled()
+// (imported from ./bm25 — Harper-free so it's unit-testable). Default is ON as
+// of 2026-07-08 (see ./bm25.ts's hybridEnabled() doc); set
+// FLAIR_HYBRID_RETRIEVAL=false to revert to the legacy HNSW + +0.05
+// keyword-bump path, byte-identical to the original pre-hybrid behavior.
+import { hybridEnabled } from "./bm25.js";
+// The actual HNSW/BM25 retrieval + post-retrieval filtering (temporal/
+// supersede/isAllowed) now lives in the pure, side-effect-free
+// retrieveCandidates() core (flair-bootstrap-scale-fix, Kern-approved
+// extraction) — MemoryBootstrap.ts calls the SAME core bare, without
+// tripping this file's rate-limit/reranker/hit-tracking side effects. See
+// resources/semantic-retrieval-core.ts's module doc for the full boundary.
+import { retrieveCandidates } from "./semantic-retrieval-core.js";
 // Candidate multiplier: fetch more candidates than needed from the HNSW index
 // so composite re-ranking has enough headroom to reorder results.
 const CANDIDATE_MULTIPLIER = 5;
-// The BM25 + union-RRF hybrid path is feature-flagged via hybridEnabled()
-// (imported from ./bm25 — Harper-free so it's unit-testable). Flag OFF (default)
-// → byte-identical to the pre-hybrid HNSW + +0.05 keyword-bump behavior.
 export class SemanticSearch extends Resource {
     // Self-authorize via the Ed25519 agent verify instead of relying on the auth
     // gate's admin super_user elevation (removed in the auth reshape). Any
@@ -38,7 +33,34 @@ export class SemanticSearch extends Resource {
         return allowVerified(this.getContext?.());
     }
     async post(data) {
-        const { agentId: bodyAgentId, q, queryEmbedding, tag, subject, subjects, limit = 10, includeSuperseded = false, scoring = "composite", minScore = 0, since, asOf } = data || {};
+        // Default scoring is "raw", not "composite" (flair#623 follow-up, measured
+        // 2026-07-08). recall-eval on the live corpus with BM25 hybrid retrieval
+        // active (default since eb26890) showed composite was net-HARMFUL at the
+        // time: Δp@3 (composite − raw) = -0.38 to -0.50 across repeated runs, MRR
+        // 0.44→0.06-0.44. Root cause: compositeScore's durability-weight ×
+        // recency-decay multiplier applied UNCONDITIONALLY (no relevance gate,
+        // unlike retrievalBoost's RBOOST_RELEVANCE_FLOOR), so a `permanent`
+        // -durability or freshly-created LOW-relevance record could outrank the
+        // objectively best-matching `persistent`/older record. Now that BM25+RRF
+        // fusion normalizes rawScore into a tight [0,1] band, an unbounded
+        // durability/recency multiplier is often bigger than the actual relevance
+        // gap between candidates.
+        //
+        // FIXED (flair#623 follow-up, 2026-07-08, see ./scoring.ts's
+        // COMPOSITE_DISCOUNT_FLOOR / COMPOSITE_RELEVANCE_FLOOR): compositeScore's
+        // durability/recency multiplier is now bounded to a small (~2%) nudge and
+        // relevance-gated, the same way RBOOST_CAP/RBOOST_RELEVANCE_FLOOR already
+        // bound the retrieval-popularity boost — `scoring: "composite"` no longer
+        // reproduces the magnet/inversion bug (recall-harness: p@3 and MRR both
+        // now match raw exactly on its 87-record corpus). The default REMAINS
+        // "raw" anyway: on that same corpus, a relevance-gated composite only
+        // MATCHES raw's precision, it doesn't beat it, so there is no measured
+        // upside to switching the default, only the (now-closed) downside risk
+        // for anyone who explicitly opts into "composite". Re-run
+        // recall-harness (test/bench/recall-harness/run.ts) and `recall-eval.mjs`
+        // before reconsidering this default if the compositeScore formula or
+        // corpus changes.
+        const { agentId: bodyAgentId, q, queryEmbedding, tag, subject, subjects, limit = 10, includeSuperseded = false, scoring = "raw", minScore = 0, since, asOf } = data || {};
         // Authenticated identity lives on the Harper Resource context (getContext().request).
         // `this.request` is NOT populated on Harper v5 Resources — prior reads here
         // silently returned undefined and the defense-in-depth scope check below
@@ -92,8 +114,9 @@ export class SemanticSearch extends Resource {
         if (!qEmb && q) {
             // Always attempt embedding generation — getEmbedding() handles init internally.
             // Don't gate on getMode() which may return "none" before init completes in worker threads.
+            // flair#504 Phase 2: 'query' — this is a search query, not stored content.
             try {
-                qEmb = await getEmbedding(String(q).slice(0, 8000));
+                qEmb = await getEmbedding(String(q).slice(0, 8000), "query");
             }
             catch { }
         }
@@ -155,7 +178,6 @@ export class SemanticSearch extends Resource {
                 });
             }
         }
-        const results = [];
         const hybrid = hybridEnabled();
         // When the reranker is on, widen the legacy HNSW fetch so it has a deeper
         // pool to re-score (retrieve topN → rerank → slice to limit). Decoupled
@@ -167,310 +189,56 @@ export class SemanticSearch extends Resource {
         // path produced `filteredResults`.
         const rerankOn = isRerankEnabled();
         const rerankTopN = getRerankTopN();
-        if (hybrid) {
-            // ─── BM25 + union-RRF hybrid path ────────────────────────────────────
-            // 1. Semantic candidates via HNSW (unchanged fetch). 2. BM25 lexical pass
-            //    over the SCOPED corpus. 3. SECURITY: the BM25 candidate set is filtered
-            //    by the SAME conditions[] + temporal filters BEFORE fusion (the corpus
-            //    is fetched with those conditions, AND re-checked in-process as
-            //    defense-in-depth) so no other agent's memory is ever scored or fused.
-            //    4. Candidate-union RRF → normalize → feed as rawScore to compositeScore.
-            const ctx = this.getContext?.();
-            // ── (a) Semantic candidate records (best-first) ──────────────────────
-            // Same HNSW query as the legacy path; we keep the raw records so the BM25
-            // pass + fusion can re-derive rawScore. No embedding → empty semantic list
-            // (RRF degrades naturally to BM25-only).
-            const semRecords = [];
-            const semIds = [];
-            if (qEmb) {
-                const candidateLimit = limit * CANDIDATE_MULTIPLIER;
-                const semQuery = {
-                    sort: { attribute: "embedding", target: qEmb, distance: "cosine" },
-                    select: ["id", "agentId", "content", "contentHash", "visibility", "tags", "durability",
-                        "source", "createdAt", "updatedAt", "expiresAt", "retrievalCount", "lastRetrieved",
-                        "promotionStatus", "promotedAt", "promotedBy", "archived", "archivedAt", "archivedBy",
-                        "parentId", "derivedFrom", "sessionId", "lastReflected", "supersedes", "subject",
-                        "validFrom", "validTo", "_safetyFlags", "$distance"],
-                    limit: candidateLimit,
-                };
-                if (conditions.length > 0)
-                    semQuery.conditions = conditions;
-                const semResults = withDetachedTxn(ctx, () => databases.flair.Memory.search(semQuery));
-                for await (const record of semResults) {
-                    // Same per-record temporal gate as the legacy HNSW loop.
-                    if (record.expiresAt && Date.parse(record.expiresAt) < Date.now())
-                        continue;
-                    if (sinceDate && record.createdAt && new Date(record.createdAt) < sinceDate)
-                        continue;
-                    if (asOf && record.validFrom && record.validFrom > asOf)
-                        continue;
-                    if (asOf && record.validTo && record.validTo <= asOf)
-                        continue;
-                    // Unconditional past-validTo exclusion (see legacy HNSW
-                    // loop below for the full rationale) — applies regardless of asOf.
-                    if (record.validTo && Date.parse(record.validTo) < Date.now())
-                        continue;
-                    semRecords.push(record);
-                    semIds.push(record.id);
-                }
-            }
-            // ── (b) BM25 candidate records over the SCOPED corpus ────────────────
-            // SECURITY: fetch the corpus WITH the same conditions[] so Harper applies
-            // the agent boundary, then re-apply the identical predicate + temporal
-            // filters in-process (isAllowedBm25Candidate) BEFORE building/scoring the
-            // index. The BM25 index therefore only ever contains the caller's allowed
-            // memories — no other agent's content/term-frequency enters scoring or
-            // fusion. This is the conditions-filter-before-fusion gate.
-            // Explicit select (same fields as the HNSW path, no embedding / $distance)
-            // so the large embedding vector is never fetched into the BM25 corpus and
-            // never spread into a result payload.
-            const corpusSelect = ["id", "agentId", "content", "contentHash", "visibility", "tags", "durability",
-                "source", "createdAt", "updatedAt", "expiresAt", "retrievalCount", "lastRetrieved",
-                "promotionStatus", "promotedAt", "promotedBy", "archived", "archivedAt", "archivedBy",
-                "parentId", "derivedFrom", "sessionId", "lastReflected", "supersedes", "subject",
-                "validFrom", "validTo", "_safetyFlags"];
-            const corpusQuery = conditions.length > 0
-                ? { conditions, select: corpusSelect }
-                : { select: corpusSelect };
-            const corpusResults = withDetachedTxn(ctx, () => databases.flair.Memory.search(corpusQuery));
-            const allowedById = new Map();
-            const bm25Docs = [];
-            for await (const record of corpusResults) {
-                // Defense-in-depth: re-check the SAME conditions[] + temporal filters
-                // in-process. Even if a Harper query change ever let an out-of-scope
-                // record through, it is dropped here BEFORE it can be BM25-scored/fused.
-                if (!isAllowedBm25Candidate(record, conditions, { sinceDate, asOf }))
-                    continue;
-                allowedById.set(record.id, record);
-                bm25Docs.push({ id: record.id, content: record.content });
-            }
-            // Carry semantic candidates that survived their temporal gate into the
-            // allowed map too (so a fused id always resolves to a record). Semantic
-            // records were fetched with the SAME conditions[], so they're in-scope.
-            for (const r of semRecords) {
-                if (!allowedById.has(r.id)) {
-                    const { $distance, ...rest } = r;
-                    allowedById.set(r.id, rest);
-                }
-            }
-            // ── (c) BM25 lexical ranking → top SEM_LIMIT (only when q present) ────
-            let bm25Ids = [];
-            if (q) {
-                const bm25 = buildBM25(bm25Docs);
-                const ranked = bm25.rank(String(q));
-                // Drop zero-score docs (no query-term overlap → contribute nothing) and
-                // cap at SEM_LIMIT — the production BM25 candidate window.
-                bm25Ids = ranked.filter(r => r.score > 0).slice(0, SEM_LIMIT).map(r => r.id);
-            }
-            // ── (d) Candidate-union RRF → normalized [0,1] rawScore ──────────────
-            // Union dedupes semantic ∪ bm25 ids; absent-from-a-list = 0 contribution.
-            const fused = fuseRrfNormalized(semIds, bm25Ids);
-            for (const [id, rrfRaw] of fused) {
-                const record = allowedById.get(id);
-                if (!record)
-                    continue; // should not happen — union ⊆ allowed
-                const rawScore = rrfRaw; // already normalized to [0,1]
-                let finalScore = scoring === "raw" ? rawScore : compositeScore(rawScore, record);
-                if (temporalBoost > 1.0)
-                    finalScore *= temporalBoost;
-                const isFlagged = record._safetyFlags && Array.isArray(record._safetyFlags) && record._safetyFlags.length > 0;
-                const source = record.agentId !== agentId ? record.agentId : undefined;
-                results.push({
-                    ...record,
-                    content: isFlagged ? wrapUntrusted(record.content, source) : record.content,
-                    _score: Math.round(finalScore * 1000) / 1000,
-                    _rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
-                    _source: source,
-                });
-            }
-        }
-        else if (qEmb) {
-            // ─── HNSW vector search path (legacy, hybrid flag OFF) ────────────────
-            const candidateLimit = rerankOn
-                ? Math.max(limit * CANDIDATE_MULTIPLIER, rerankTopN)
-                : limit * CANDIDATE_MULTIPLIER;
-            const query = {
-                sort: { attribute: "embedding", target: qEmb, distance: "cosine" },
-                select: ["id", "agentId", "content", "contentHash", "visibility", "tags", "durability",
-                    "source", "createdAt", "updatedAt", "expiresAt", "retrievalCount", "lastRetrieved",
-                    "promotionStatus", "promotedAt", "promotedBy", "archived", "archivedAt", "archivedBy",
-                    "parentId", "derivedFrom", "sessionId", "lastReflected", "supersedes", "subject",
-                    "validFrom", "validTo", "_safetyFlags", "$distance"],
-                limit: candidateLimit,
-            };
-            if (conditions.length > 0) {
-                query.conditions = conditions;
-            }
-            // MemoryGrant.search above left a closed transaction in ctx's chain —
-            // detach it so Harper builds a fresh transaction for this Memory read.
-            const ctx = this.getContext?.();
-            const memoryResults = withDetachedTxn(ctx, () => databases.flair.Memory.search(query));
-            for await (const record of memoryResults) {
-                if (record.expiresAt && Date.parse(record.expiresAt) < Date.now())
-                    continue;
-                if (sinceDate && record.createdAt && new Date(record.createdAt) < sinceDate)
-                    continue;
-                // Temporal validity: if asOf is specified, only include memories valid at that point
-                if (asOf && record.validFrom && record.validFrom > asOf)
-                    continue;
-                if (asOf && record.validTo && record.validTo <= asOf)
-                    continue;
-                // A past validTo ALWAYS means the record has been closed out
-                // (server supersede path — Memory.ts closeSupersededRecord — sets
-                // validTo without necessarily setting `archived`). Unconditional, not
-                // gated on `asOf`, so a server-superseded record can't resurface in
-                // the DEFAULT recall path just because its successor isn't
-                // co-present in this result set (the supersededIds filter further
-                // down only catches co-presence). A record with no validTo, or a
-                // future validTo, is unaffected.
-                if (record.validTo && Date.parse(record.validTo) < Date.now())
-                    continue;
-                let semanticScore;
-                if (record.$distance !== undefined) {
-                    semanticScore = distanceToSimilarity(record.$distance);
-                }
-                else {
-                    // ─── Harper's cosine-sort query omits $distance for a
-                    // SINGLETON post-filter result set — the SAME quirk root-caused and
-                    // fixed for the dedup path (resources/Memory.ts
-                    // findConservativeDedupMatch / resources/dedup.ts cosineSimilarity).
-                    // Sort ORDER is still correct; only the numeric `$distance`
-                    // annotation is missing on that one record, regardless of the
-                    // query's own `limit` (reproduced here with candidateLimit=50, not
-                    // just limit=1 — the trigger is the post-filter MATCH COUNT, not the
-                    // requested limit).
-                    //
-                    // Layer 1 made this common: the no-grants agent scope used
-                    // to ALWAYS be a compound `{operator:"or", conditions:[{agentId},
-                    // {visibility=="office"}]}` condition; resolveReadScope() now emits
-                    // a PLAIN single `{agentId==X}` condition for the common (no-grants)
-                    // case, so a scoped search against an agent with exactly one
-                    // matching memory hits this Harper quirk directly. Before, the OR
-                    // wrap incidentally avoided the singleton shape. The old `?? 1`
-                    // fallback silently collapsed this to similarity 0 — read by
-                    // callers (including the clean-VM CI gate's single-memory init
-                    // probe, #533) as "embeddings not loaded", which is WRONG:
-                    // embeddings ARE loaded, only the score was computed incorrectly.
-                    //
-                    // Fix: point-lookup the record by id (a plain get(), unaffected by
-                    // the sort-query quirk — selecting "embedding" directly on the SAME
-                    // sort-by-embedding query comes back as a bare scalar, per the same
-                    // investigation above) and compute cosine similarity ourselves in JS from its
-                    // real stored `embedding` vector against this query's `qEmb`, via
-                    // the same math as the ume4 fallback (dedup.ts's cosineSimilarity).
-                    // Only done on this (rare) undefined-$distance branch — never adds
-                    // a point-lookup to the normal per-record path. If the stored
-                    // embedding is missing/empty (e.g. a legacy pre-embedding record),
-                    // cosineSimilarity returns 0 — the same safe "no match" the old
-                    // `?? 1` fallback produced, never a false-high score.
-                    const full = await withDetachedTxn(ctx, () => databases.flair.Memory.get(record.id));
-                    const storedEmbedding = Array.isArray(full?.embedding) ? full.embedding : [];
-                    semanticScore = cosineSimilarity(qEmb, storedEmbedding);
-                }
-                let keywordHit = false;
-                if (q && String(record.content || "").toLowerCase().includes(String(q).toLowerCase())) {
-                    keywordHit = true;
-                }
-                const rawScore = semanticScore + (keywordHit ? 0.05 : 0);
-                let finalScore = scoring === "raw" ? rawScore : compositeScore(rawScore, record);
-                if (temporalBoost > 1.0)
-                    finalScore *= temporalBoost;
-                const { $distance, ...rest } = record;
-                const isFlagged = rest._safetyFlags && Array.isArray(rest._safetyFlags) && rest._safetyFlags.length > 0;
-                const source = record.agentId !== agentId ? record.agentId : undefined;
-                results.push({
-                    ...rest,
-                    content: isFlagged ? wrapUntrusted(rest.content, source) : rest.content,
-                    _score: Math.round(finalScore * 1000) / 1000,
-                    _rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
-                    _source: source,
-                });
-            }
-        }
-        else {
-            // ─── No embedding available — keyword-only fallback ──────────────────
-            // Full scan is only used when there's no query embedding (e.g. tag-only
-            // or subject-only searches, or when the embedding engine is unavailable).
-            const query = conditions.length > 0 ? { conditions } : {};
-            // MemoryGrant.search above left a closed transaction in ctx's chain —
-            // detach it so Harper builds a fresh transaction for this Memory read.
-            const ctx = this.getContext?.();
-            const memoryResults = withDetachedTxn(ctx, () => databases.flair.Memory.search(query));
-            for await (const record of memoryResults) {
-                if (record.expiresAt && Date.parse(record.expiresAt) < Date.now())
-                    continue;
-                if (sinceDate && record.createdAt && new Date(record.createdAt) < sinceDate)
-                    continue;
-                if (asOf && record.validFrom && record.validFrom > asOf)
-                    continue;
-                if (asOf && record.validTo && record.validTo <= asOf)
-                    continue;
-                // Unconditional past-validTo exclusion (see legacy HNSW
-                // loop above for the full rationale) — applies regardless of asOf.
-                if (record.validTo && Date.parse(record.validTo) < Date.now())
-                    continue;
-                let keywordHit = false;
-                if (q && String(record.content || "").toLowerCase().includes(String(q).toLowerCase())) {
-                    keywordHit = true;
-                }
-                const rawScore = keywordHit ? 0.05 : 0;
-                if (q && rawScore === 0)
-                    continue;
-                const { embedding, ...rest } = record;
-                let finalScore = scoring === "raw" ? rawScore : compositeScore(rawScore, rest);
-                if (temporalBoost > 1.0)
-                    finalScore *= temporalBoost;
-                const isFlagged = rest._safetyFlags && Array.isArray(rest._safetyFlags) && rest._safetyFlags.length > 0;
-                const source = record.agentId !== agentId ? record.agentId : undefined;
-                results.push({
-                    ...rest,
-                    content: isFlagged ? wrapUntrusted(rest.content, source) : rest.content,
-                    _score: Math.round(finalScore * 1000) / 1000,
-                    _rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
-                    _source: source,
-                });
-            }
-        }
-        // Build superseded set and filter (unless caller opts in to see full history)
-        let filteredResults = results;
-        if (!includeSuperseded) {
-            const supersededIds = new Set();
-            for (const r of results) {
-                if (r.supersedes)
-                    supersededIds.add(r.supersedes);
-            }
-            filteredResults = results.filter((r) => !supersededIds.has(r.id));
-        }
-        // Apply minimum score filter
-        if (minScore > 0) {
-            filteredResults = filteredResults.filter((r) => r._score >= minScore);
-        }
+        // The overfetch policy (how many raw candidates to pull from the
+        // HNSW/BM25 legs relative to what the caller ultimately wants) is THIS
+        // wrapper's decision — retrieveCandidates() never multiplies its `limit`
+        // param internally (see resources/semantic-retrieval-core.ts's doc), so
+        // every caller (this one, and MemoryBootstrap's own K formula) computes
+        // its own fetch depth. Hybrid's semantic leg is never rerank-widened
+        // (matches the pre-extraction behavior: only the legacy path widened for
+        // rerank).
+        const candidateLimit = hybrid
+            ? limit * CANDIDATE_MULTIPLIER
+            : (rerankOn ? Math.max(limit * CANDIDATE_MULTIPLIER, rerankTopN) : limit * CANDIDATE_MULTIPLIER);
+        const ctx = this.getContext?.();
+        let filteredResults = await retrieveCandidates({
+            queryEmbedding: qEmb,
+            q,
+            conditions,
+            limit: candidateLimit,
+            includeSuperseded,
+            scoring,
+            temporalBoost,
+            sinceDate,
+            asOf,
+            minScore,
+            agentId,
+            isAllowed: scope?.isAllowed,
+            hybrid,
+            ctx,
+        });
         // ─── Cross-encoder rerank (best-effort, fail-open to vector order) ───────
         // Re-scores query+candidate TOGETHER (cross-attention the pooled embedding
         // can't do) and reorders before the final slice. Reorders whatever
-        // `filteredResults` the retrieval stage above produced — legacy HNSW-only
-        // OR the BM25+union-RRF hybrid path — since both converge into
-        // the same `results`/`filteredResults` shape before this point; hybrid
-        // retrieves+fuses, the reranker only reorders the fused set. Still gated
-        // on `qEmb` (an embedding was actually generated); the pure keyword-only
-        // fallback (no qEmb at all) is untouched either way. The reranker
-        // overwrites `_score` with the rerank score (so margin measurement reads it)
-        // and preserves the semantic score as `_semScore`; `_rawScore` is left as-is
-        // so recall-bench's scoring:"raw" path stays reproducible. On init failure,
-        // timeout, or any throw, rerankCandidates returns the input UNCHANGED and we
-        // fall through to the existing vector-order sort — recall is never blocked.
+        // `filteredResults` retrieveCandidates() produced — legacy HNSW-only OR
+        // the BM25+union-RRF hybrid path — since both converge into the same
+        // shape (retrieveCandidates never exposes which leg produced a result).
+        // Still gated on `qEmb` (an embedding was actually generated); the pure
+        // keyword-only fallback (no qEmb at all) is untouched either way. The
+        // reranker overwrites `_score` with the rerank score (so margin
+        // measurement reads it) and preserves the semantic score as `_semScore`;
+        // `_rawScore` is left as-is so recall-bench's scoring:"raw" path stays
+        // reproducible. On init failure, timeout, or any throw, rerankCandidates
+        // returns the input UNCHANGED and we fall through to retrieveCandidates'
+        // own vector-order sort — recall is never blocked. retrieveCandidates
+        // already returns its output sorted best-first, so the non-rerank branch
+        // needs no additional sort here.
         if (rerankOn && qEmb && q && filteredResults.length >= getRerankMinCandidates()) {
-            // Pre-sort by vector order so the topN fed to the reranker is the most
-            // semantically-promising slice (filteredResults isn't sorted yet here).
-            filteredResults.sort((a, b) => b._score - a._score);
             filteredResults = await rerankCandidates(String(q), filteredResults, {
                 topN: rerankTopN,
                 budgetMs: getRerankBudgetMs(),
             });
         }
-        else {
-            filteredResults.sort((a, b) => b._score - a._score);
-        }
         const topResults = filteredResults.slice(0, limit);
         // Async hit tracking — don't block the response
         const now = new Date().toISOString();