@tpsdev-ai/flair 0.48.0 → 0.50.0

package/dist/resources/embeddings-boot.js

@@ -84,6 +84,7 @@
  * Harper boot), registration is skipped and logged — Harper falls back to
  * keyword-only search, matching the pre-existing degrade contract.
  */
+import { availableParallelism } from "node:os";
 import { resolveModelsDir } from "./embeddings-provider.js";
 const LOGICAL_NAME = "default";
 const MODEL_NAME = "nomic-embed-text";
@@ -148,6 +149,43 @@ const EMBEDDING_POOLING = "mean";
 function benchModelPathOverride() {
     return process.env.FLAIR_RECALL_HARNESS_MODEL_PATH || undefined;
 }
+/**
+ * llama.cpp CPU thread count passed to HFE `register({config:{threads}})`.
+ *
+ * HFE's own default is a fixed 6 (see harper-fabric-embeddings' `init()`
+ * table). flair never used to pass `threads`, so every host inherited that
+ * 6: an 8-vCPU ingest box left cores idle (flair#1330), a 4-core laptop
+ * oversubscribed. We always pass an explicit value.
+ *
+ * Default (unset / empty / non-integer / <1): `max(1, cores - 1)`. Safer
+ * than `min(6, cores)` — that still leaves the 8-vCPU case idle at 6 —
+ * and safer than using every core: Harper's JS worker (`THREADS_COUNT=1`)
+ * and the OS keep one. `availableParallelism()` (not `os.cpus().length`)
+ * so a cgroup CPU quota (Docker / k8s / Fabric) is what we count.
+ *
+ * Override: `FLAIR_EMBED_THREADS` — a positive integer, env-only. A
+ * config.yaml key would go through Harper's models-config persist path,
+ * which is exactly the class of state embeddings-boot exists to avoid
+ * (flair#694). Invalid values fall through to the host-aware default.
+ */
+export function resolveEmbedThreads(env = process.env, cores = availableParallelism()) {
+    const parsed = parsePositiveInt(env.FLAIR_EMBED_THREADS);
+    if (parsed !== undefined)
+        return parsed;
+    const safeCores = Number.isFinite(cores) && cores >= 1 ? Math.floor(cores) : 1;
+    return Math.max(1, safeCores - 1);
+}
+function parsePositiveInt(raw) {
+    if (raw == null)
+        return undefined;
+    const trimmed = raw.trim();
+    if (trimmed === "")
+        return undefined;
+    const n = Number(trimmed);
+    if (!Number.isInteger(n) || n < 1)
+        return undefined;
+    return n;
+}
 let registered = false;
 /**
  * Register the embedding backend. Idempotent within a process (mirrors
@@ -161,12 +199,16 @@ export async function registerEmbeddingsBackend() {
     try {
         const { register } = await import("harper-fabric-embeddings");
         const modelPath = benchModelPathOverride();
+        const threads = resolveEmbedThreads();
         await register({
             logicalName: LOGICAL_NAME,
             kind: "embedding",
-            config: modelPath
-                ? { modelPath, pooling: EMBEDDING_POOLING }
-                : { modelName: MODEL_NAME, modelsDir: resolveModelsDir(), pooling: EMBEDDING_POOLING },
+            config: {
+                ...(modelPath
+                    ? { modelPath, pooling: EMBEDDING_POOLING }
+                    : { modelName: MODEL_NAME, modelsDir: resolveModelsDir(), pooling: EMBEDDING_POOLING }),
+                threads,
+            },
         });
     }
     catch (err) {

package/dist/resources/health.js

@@ -1,4 +1,4 @@
-import { Resource, databases } from "harper";
+import { Resource, databases, server, logger } from "harper";
 import { promises as fsp, existsSync, readFileSync } from "node:fs";
 import { homedir, platform } from "node:os";
 import { join, dirname } from "node:path";
@@ -8,6 +8,9 @@ import { resolveBuildInfo } from "./build-info.js";
 import { getMigrationStatusSnapshot } from "./migrations/status.js";
 import { resolveMigrationDataDirForRead } from "./migrations/data-dir.js";
 import { REM_DEDUP_STATS_PATH } from "./dedup-cluster.js";
+import { hybridEnabled } from "./bm25.js";
+import { bm25IndexEnabled, bm25IndexStatus } from "./bm25-index-service.js";
+import { buildPublicHealthBody, resolveSearchReadiness } from "./search-readiness.js";
 const db = databases;
 const redactHome = (p) => {
     const home = homedir();
@@ -52,7 +55,7 @@ function resolveVersion() {
     return process.env.npm_package_version ?? "dev";
 }
 /**
- * Health endpoint — truly public, returns only { ok: true }.
+ * Health endpoint — truly public. Identity-free; no auth, no role check.
  *
  * `allowRead() { return true }` opens Harper's role gate for anonymous GETs,
  * which is what makes /Health work for callers outside `authorizeLocal`'s
@@ -66,8 +69,15 @@ function resolveVersion() {
  *
  * Same pattern as `FederationPair.allowCreate(){ return true }` (PR #299):
  * declare the Resource anonymously-accessible at the Harper layer; let the
- * handler itself enforce whatever it needs (in this case, nothing — /Health
- * is intentionally and always public).
+ * handler itself enforce whatever it needs.
+ *
+ * flair#1326: `ok: true` used to mean only "this resource answered." That
+ * is a green light that lies when search routes are not mounted yet, or
+ * when the hybrid BM25 index is still cold (first search after restart
+ * scans the corpus; the lag grows with store size). `searchReady` is
+ * always present. When search cannot be served at all, this endpoint
+ * returns HTTP 503 and `ok: false`. When the process is live but recall
+ * is still cold, it stays 200 and names the lag on `searchReadyReason`.
  *
  * Rich stats (memory counts, agent names, etc.) are behind /HealthDetail
  * which requires authentication. This prevents information leakage on
@@ -96,13 +106,39 @@ export class Health extends Resource {
         // a 40-hex sha when the build ran in a git work tree, an honest null
         // otherwise (tarball builds) — never omitted, never fabricated (Sherlock).
         const build = resolveBuildInfo();
-        return {
-            ok: true,
+        const readiness = currentSearchReadiness();
+        const body = buildPublicHealthBody(readiness, {
             version: build?.version ?? resolveVersion(),
             buildCommit: build?.commit ?? null,
-        };
+        });
+        if (readiness.status !== 200) {
+            return new Response(JSON.stringify(body), {
+                status: readiness.status,
+                headers: { "content-type": "application/json" },
+            });
+        }
+        return body;
     }
 }
+/** Same sources /Health and /HealthDetail consult so they cannot disagree. */
+let _warnedMissingRegistry = false;
+function currentSearchReadiness() {
+    // Shipped Harper launch: this Resource is already registered, so
+    // server.resources is populated. The null skip is a stated fail-open
+    // (Sherlock on #1406) for the injectable/test path — not an accident.
+    const resources = server.resources ?? null;
+    if (!resources && !_warnedMissingRegistry) {
+        _warnedMissingRegistry = true;
+        logger.warn?.("Health: server.resources is absent — skipping the search-route mount check (table-only fail-open). Shipped Harper launch always exposes the registry.");
+    }
+    return resolveSearchReadiness({
+        resources,
+        memoryTable: db.flair?.Memory,
+        bm25: bm25IndexStatus(),
+        hybridEnabled: hybridEnabled(),
+        bm25IndexEnabled: bm25IndexEnabled(),
+    });
+}
 /**
  * Authenticated health detail — returns memory/agent/soul stats + process info.
  * Requires Ed25519 agent auth or admin basic auth.
@@ -127,6 +163,15 @@ export class HealthDetail extends Resource {
         const stats = { ok: true };
         const nowMs = Date.now();
         const warnings = [];
+        // flair#1326: same search-ready signal as public /Health. HealthDetail
+        // stays HTTP 200 (it is a stats dump, not a traffic gate); the field
+        // and a warning name the lag so `flair status` / operators can see it.
+        const readiness = currentSearchReadiness();
+        stats.searchReady = readiness.searchReady;
+        if (readiness.searchReadyReason) {
+            stats.searchReadyReason = readiness.searchReadyReason;
+            warnings.push({ level: "warn", message: readiness.searchReadyReason });
+        }
         const ctx = this.getContext?.();
         // #614 fix: resolve identity via the shared three-way verdict
         // (internal/agent/anonymous — agent-auth.ts) instead of reading

package/dist/resources/mcp-tools.js

@@ -811,6 +811,7 @@ export const TOOLS = {
                     { count: "memoriesIncluded", containers: ["memories", "predicted"] }, // #1199
                     { count: "teammateFindingsIncluded", containers: ["teammateFindings"] }, // #1199
                     { count: "sections.events", containers: ["events"] }, // #1206
+                    { count: "sections.soul", containers: ["soul"] }, // #1371
                 ],
                 // present + typed even when empty — never a bare {} / missing key (#1182).
                 selfDescribingEmpty: [

package/dist/resources/memory-read-scope.js

@@ -95,5 +95,7 @@ export async function resolveReadScope(authAgentId) {
             return false;
         return record.agentId === authAgentId || !isPrivateVisibility(record.visibility);
     };
+    // See ReadScope.isAllowed's doc: agentId + visibility, nothing else.
+    isAllowed.scopableOnly = true;
     return { allowedOwners: [authAgentId], condition, isAllowed };
 }

package/dist/resources/search-readiness.js

@@ -0,0 +1,100 @@
+/**
+ * search-readiness.ts — the honest /Health search-ready signal (flair#1326).
+ *
+ * Harper's process can answer /health (and Flair's /Health resource can
+ * answer {ok:true}) while search is still unusable:
+ *
+ *   1. Boot window — jsResources register incrementally. /Health can be up
+ *      while /Memory and /SemanticSearch still 404 from Harper's catch-all
+ *      (documented in packages/adk-flair-js/test/helpers/boot-harper.mjs).
+ *   2. Cold BM25 index — hybrid retrieval's persistent index is lazy-built
+ *      on the first search, not at component start (bm25-index-service.ts).
+ *      That first-query corpus scan grows with store size. /Health answering
+ *      is not "recall is warm."
+ *
+ * This module is Harper-free so the decision is unit-testable against the
+ * shipped function. Callers inject the registry / table / index status they
+ * already have.
+ *
+ * Two layers, on purpose:
+ *   - Routes/table not mounted → not healthy (ok:false, HTTP 503). A
+ *     traffic-gating probe that only looks at status must not get a green
+ *     light for a node whose search routes are not serving.
+ *   - Routes up but index still cold → process is live (ok:true, HTTP 200)
+ *     and searchReady:false names the lag. We do NOT 503 on a cold index:
+ *     the index builds on the first search, and a health check must not
+ *     trigger that scan (bm25-index-service.ts: "Eager building would add a
+ *     full corpus scan to every boot including … health checks"). 503-until-
+ *     warm would deadlock — health waits for the index, the index waits for
+ *     a search that never comes.
+ */
+function routeMounted(resources, name) {
+    const entry = resources.get?.(name) ?? resources.getMatch?.(name);
+    return Boolean(entry?.Resource);
+}
+/**
+ * Decide whether search is actually usable, and whether /Health should claim
+ * the process is healthy.
+ *
+ * `resources` is optional. Stated fail-open (Sherlock on #1406): when the
+ * registry is missing we skip the route-mount check rather than 503 forever.
+ * A table handle can exist while `/Memory` and `/SemanticSearch` still 404,
+ * so this is weaker than the primary defense. In the shipped launch path
+ * `/Health` is served by this Resource after Harper has registered us, so
+ * `server.resources` is populated; the skip is the injectable/test path
+ * (and a theoretical export without a registry). Health.ts logs once if
+ * the live call site actually takes it.
+ */
+export function resolveSearchReadiness(opts) {
+    if (opts.resources) {
+        const memoryMounted = routeMounted(opts.resources, "Memory");
+        const searchMounted = routeMounted(opts.resources, "SemanticSearch");
+        if (!memoryMounted || !searchMounted) {
+            const missing = [
+                !memoryMounted ? "Memory" : null,
+                !searchMounted ? "SemanticSearch" : null,
+            ].filter(Boolean).join(", ");
+            return notServing(`search routes not mounted (${missing})`);
+        }
+    }
+    if (!opts.memoryTable || typeof opts.memoryTable.search !== "function") {
+        return notServing("memory table not queryable");
+    }
+    // Hybrid + the persistent index are default-on. A cold or in-flight BM25
+    // index means the first search will pay a full corpus scan — the #1326
+    // lag. Name it; do not fail liveness.
+    //
+    // `disabled` (feed/build failure) and `FLAIR_BM25_INDEX=false` both fall
+    // back to the per-query scan. The kill switch never calls ensureReady, so
+    // status stays `empty` for the life of the process — that is serving, not
+    // cold. Treating it as lag would make searchReady false forever and refuse
+    // a node that is already answering recall.
+    const indexInPath = opts.hybridEnabled !== false && opts.bm25IndexEnabled !== false;
+    if (indexInPath && opts.bm25) {
+        if (opts.bm25.state === "building") {
+            return namesLag("bm25 index building — first search is still scanning the corpus");
+        }
+        if (opts.bm25.state === "empty") {
+            return namesLag("bm25 index not built (cold boot; first search scans the corpus)");
+        }
+    }
+    return { searchReady: true, ok: true, status: 200 };
+}
+function notServing(searchReadyReason) {
+    return { searchReady: false, ok: false, status: 503, searchReadyReason };
+}
+function namesLag(searchReadyReason) {
+    return { searchReady: false, ok: true, status: 200, searchReadyReason };
+}
+/** Public /Health JSON body. `searchReady` is always present (never omitted). */
+export function buildPublicHealthBody(readiness, identity) {
+    const body = {
+        ok: readiness.ok,
+        version: identity.version,
+        buildCommit: identity.buildCommit,
+        searchReady: readiness.searchReady,
+    };
+    if (readiness.searchReadyReason)
+        body.searchReadyReason = readiness.searchReadyReason;
+    return body;
+}

package/dist/resources/semantic-retrieval-core.js

@@ -59,6 +59,8 @@ import { cosineSimilarity } from "./dedup.js";
 import { compositeScore } from "./scoring.js";
 import { buildBM25, fuseRrfNormalized, SEM_LIMIT } from "./bm25.js";
 import { isAllowedBm25Candidate } from "./bm25-filter.js";
+import { indexedBm25Ids } from "./bm25-index-service.js";
+import { byRecencyThenId } from "./sort-comparators.js";
 // Convert HNSW cosine distance (1 - similarity) to similarity score.
 function distanceToSimilarity(distance) {
     return 1 - distance;
@@ -154,23 +156,60 @@ export async function retrieveCandidates(params) {
                 semIds.push(record.id);
             }
         }
-        // ── (b) BM25 candidate records over the SCOPED corpus ────────────────
-        const corpusQuery = conditions.length > 0
-            ? { conditions, select }
-            : { select };
-        const corpusResults = withDetachedTxn(ctx, () => databases.flair.Memory.search(corpusQuery));
+        // ── (b) The BM25 lexical leg ─────────────────────────────────────────
+        //
+        // flair#1357. This used to be unconditional: fetch the WHOLE scoped corpus
+        // out of Harper, then `buildBM25()` it — per query. That made retrieval
+        // latency linear in store size (5.6s p50 at 60k rows, 28.7s at 180k). The
+        // lexical leg is now served from a persistent, incrementally-maintained
+        // index (resources/bm25-index.ts) whose contract is RANKING-IDENTICAL:
+        // same ids, same order, byte for byte. The legacy scan below is still the
+        // reference implementation AND the fallback — the index returns null for
+        // any query it cannot reproduce exactly, and for a query with no text
+        // there is no lexical leg to serve at all.
         const allowedById = new Map();
-        const bm25Docs = [];
-        for await (const record of corpusResults) {
-            // Defense-in-depth: re-check the SAME conditions[] + temporal filters
-            // in-process. Even if a Harper query change ever let an out-of-scope
-            // record through, it is dropped here BEFORE it can be BM25-scored/fused.
-            if (!isAllowedBm25Candidate(record, conditions, { sinceDate, asOf }))
-                continue;
-            if (!passesAllowed(record))
-                continue;
-            allowedById.set(record.id, record);
-            bm25Docs.push({ id: record.id, content: record.content });
+        let bm25Ids = [];
+        // Only the no-signal listing branch (d) still needs the full scoped
+        // corpus materialised; every other branch resolves records by id.
+        const needCorpusListing = !q && !qEmb;
+        let servedFromIndex = false;
+        if (q) {
+            const fromIndex = await indexedBm25Ids({
+                q: String(q),
+                conditions: conditions,
+                timeFilters: { sinceDate, asOf },
+                isAllowed,
+                limit: SEM_LIMIT,
+                ctx,
+            });
+            if (fromIndex) {
+                bm25Ids = fromIndex;
+                servedFromIndex = true;
+            }
+        }
+        if (!servedFromIndex && (q || needCorpusListing)) {
+            // ── Legacy path: scoped corpus scan + per-query buildBM25() ──────────
+            const corpusQuery = conditions.length > 0
+                ? { conditions, select }
+                : { select };
+            const corpusResults = withDetachedTxn(ctx, () => databases.flair.Memory.search(corpusQuery));
+            const bm25Docs = [];
+            for await (const record of corpusResults) {
+                // Defense-in-depth: re-check the SAME conditions[] + temporal filters
+                // in-process. Even if a Harper query change ever let an out-of-scope
+                // record through, it is dropped here BEFORE it can be BM25-scored/fused.
+                if (!isAllowedBm25Candidate(record, conditions, { sinceDate, asOf }))
+                    continue;
+                if (!passesAllowed(record))
+                    continue;
+                allowedById.set(record.id, record);
+                bm25Docs.push({ id: record.id, content: record.content });
+            }
+            if (q) {
+                const bm25 = buildBM25(bm25Docs);
+                const ranked = bm25.rank(String(q));
+                bm25Ids = ranked.filter(r => r.score > 0).slice(0, SEM_LIMIT).map(r => r.id);
+            }
         }
         // Carry semantic candidates that survived their temporal gate into the
         // allowed map too (so a fused id always resolves to a record). Semantic
@@ -181,12 +220,45 @@ export async function retrieveCandidates(params) {
                 allowedById.set(r.id, rest);
             }
         }
-        // ── (c) BM25 lexical ranking → top SEM_LIMIT (only when q present) ────
-        let bm25Ids = [];
-        if (q) {
-            const bm25 = buildBM25(bm25Docs);
-            const ranked = bm25.rank(String(q));
-            bm25Ids = ranked.filter(r => r.score > 0).slice(0, SEM_LIMIT).map(r => r.id);
+        // ── (b2) Resolve the index-served BM25 candidates ─────────────────────
+        // On the indexed path there is no corpus map to read from, so a BM25-only
+        // rescue is point-looked-up and projected down to `select`. The projection
+        // reproduces Harper's own `search({select})` shape exactly — the keys of
+        // `select` that the row actually carries, in `select` declaration order
+        // (measured; pinned by test/integration/bm25-index-scan-order-1357.test.ts).
+        //
+        // The freshly-read row is then re-checked against the SAME conditions[] +
+        // temporal filters + scope predicate before it is allowed into the fusion.
+        // The index already applied all three to its own copy of the row; this is
+        // the Sherlock gate applied to the row we are actually about to return, so
+        // a stale index entry can only ever REMOVE a candidate, never smuggle an
+        // out-of-scope record into the union.
+        if (servedFromIndex) {
+            const resolved = [];
+            for (const id of bm25Ids) {
+                if (allowedById.has(id)) {
+                    resolved.push(id);
+                    continue;
+                }
+                const full = await withDetachedTxn(ctx, () => databases.flair.Memory.get(id));
+                if (!full)
+                    continue;
+                if (!isAllowedBm25Candidate(full, conditions, { sinceDate, asOf }))
+                    continue;
+                if (!passesAllowed(full))
+                    continue;
+                const projected = {};
+                for (const key of select)
+                    if (key in full)
+                        projected[key] = full[key];
+                allowedById.set(id, projected);
+                if (qEmb) {
+                    const storedEmbedding = Array.isArray(full.embedding) ? full.embedding : [];
+                    semSimById.set(id, cosineSimilarity(qEmb, storedEmbedding));
+                }
+                resolved.push(id);
+            }
+            bm25Ids = resolved;
         }
         // ── (d) No retrieval signal at all → full scoped listing ────────────
         if (!q && !qEmb) {
@@ -413,7 +485,14 @@ export async function retrieveCandidates(params) {
     // retrieval lives in the fused ORDER (a BM25 rank-1 rescue outranks weak
     // semantic hits), while `_score` carries the honest absolute evidence for
     // each result — the two can disagree, and that is correct.
-    filteredResults.sort((a, b) => b._rank - a._rank);
+    //
+    // Ties at this sort are cross-leg RRF identities (flair#1412): no
+    // within-leg tie-break can prevent `1/(k+3)+1/(k+5) === 1/(k+5)+1/(k+3)`.
+    // Break them by createdAt DESC (null → oldest, never NaN) then id ASC.
+    // Determinism is the id ASC total order. createdAt is best-effort recency
+    // within an exact `_rank` tie — federated writer clocks can skew, and
+    // that cannot reintroduce nondeterminism (see byRecencyThenId).
+    filteredResults.sort((a, b) => (b._rank - a._rank) || byRecencyThenId(a, b));
     for (const r of filteredResults)
         delete r._rank;
     return filteredResults;

package/dist/resources/sort-comparators.js

@@ -0,0 +1,45 @@
+/**
+ * Deterministic sort comparators (flair#1412).
+ *
+ * Retrieval and selection sorts used to key on one number (score, rank,
+ * priority) and then inherit `Array.prototype.sort`'s input order on ties.
+ * That input order is Harper scan order, which is free to change across
+ * restarts. The family below makes the tie-break the default thing to
+ * reach for: each helper has its own primary key, and they share the
+ * `compareKey` tail — not one comparator forced onto every site.
+ *
+ * Do not collapse this into a single `byScoreThenId`. cosine.ts keys on
+ * corpus `.index`; MemoryBootstrap (flair#1409, not this PR) keys on the
+ * soul `key`. The shared part is the tail.
+ *
+ * Locale-independent: `compareKey` is code-unit / numeric `<`/`>`, never
+ * `localeCompare`. ISO-8601 UTC strings compare chronologically that way.
+ */
+/** Total-order tail. Works for ids, corpus indexes, soul keys. */
+export function compareKey(a, b) {
+    return a < b ? -1 : a > b ? 1 : 0;
+}
+/** Score / rank descending, then `id` ascending. */
+export function byNumberDescThenId(getNumber) {
+    return (a, b) => (getNumber(b) - getNumber(a)) || compareKey(a.id, b.id);
+}
+/**
+ * `createdAt` DESC (null → oldest, never NaN) then `id` ASC.
+ *
+ * Determinism comes from `id` ASC — ids are unique, so this is a total
+ * order no matter what `createdAt` does. Do not date-arithmetic a
+ * possibly-null field: `new Date(null).getTime()` is 0, but subtracting
+ * NaN (unparseable / missing-after-coercion) is undefined behaviour for
+ * `Array.prototype.sort`. Empty-string fallback keeps the comparison in
+ * string space; an empty value is less than any ISO-8601 UTC timestamp,
+ * so it sorts last under DESC.
+ *
+ * Clock-skew caveat: `createdAt` is writer-stamped. Across federated
+ * writers this is best-effort recency within an exact `_rank` tie, not a
+ * correctness claim. Skew can misorder rows the ranker already called
+ * equivalent; it cannot reintroduce nondeterminism, because `id` ASC
+ * still resolves every remaining tie.
+ */
+export function byRecencyThenId(a, b) {
+    return compareKey(b.createdAt ?? "", a.createdAt ?? "") || compareKey(a.id, b.id);
+}