@tpsdev-ai/flair 0.27.1 → 0.29.0

package/dist/resources/migrations/graph-heal.js

@@ -0,0 +1,250 @@
+/**
+ * graph-heal.ts — the recall graph-heal OBSERVABILITY migration.
+ *
+ * WHAT ACTUALLY HEALS: nothing in this file. The heal is entirely
+ * schema-driven — `schemas/memory.graphql`'s `embedding` field now declares
+ * `@indexed(type: "HNSW", M: 16)` (M:16 = Harper's own default, a zero
+ * behavior change). On the first boot after upgrade, Harper structurally
+ * diffs the persisted per-attribute HNSW descriptor (built with NO options on
+ * every pre-this-change install) against the schema and, because
+ * `canonicalizeIndexOptions` does NOT inject defaults (`{type:"HNSW"}` and
+ * `{type:"HNSW",M:16}` are DIFFERENT canonical keys — verified against the
+ * installed @harperfast/harper source, resources/databases.js:
+ * canonicalizeIndexOptions + the `indexOptionsChanged` reset of
+ * `lastIndexedKey` to undefined), clears the graph store and rebuilds it
+ * CLEANLY from the already-correct stored vectors. That rebuild heals the
+ * stale/asymmetric reverse-edge corruption that an OLDER (pre-fix) Harper's
+ * INCREMENTAL HNSW update left on disk during the July bulk embedding-stamp
+ * re-embed — a fresh index of the same vectors finds the true neighbors, the
+ * mangled live graph did not. The corruption is HISTORICAL: the installed
+ * Harper (5.1.22) already fixes that incremental-update path —
+ * HierarchicalNavigableSmallWorld reconstructs the prior vector from the
+ * stored node and carries its real connections into the reverse-edge cleanup
+ * the old build skipped — so current Harper no longer produces it; it just
+ * still persists on disk in stores re-embedded under the old engine. The heal
+ * is therefore version-agnostic: it repairs any such store regardless of the
+ * Harper now running. No re-embed happens — only the graph is rebuilt from the
+ * vectors already on disk.
+ *
+ * WHY THIS MIGRATION EXISTS ANYWAY: the heal is invisible — Harper triggers
+ * it, this code cannot force it, and there is no Harper API to observe it
+ * directly. So this migration's ONLY job is to VERIFY + LEDGER that recall is
+ * healthy after the boot, so the heal is auditable and version-gated (runs
+ * once per upgrade, then the runner's state-file short-circuit — state.ts —
+ * skips it). It touches ZERO rows: riskClass 'derived-only', countPending()
+ * is always 0, run() writes one structural-only OrgEvent and returns
+ * `processed: 0`. The runner (resources/migrations/runner.ts) then records
+ * success in the state file, short-circuiting every subsequent boot at this
+ * version.
+ *
+ * NON-THROWING throughout (invariant II — the runner halts-not-bricks): a
+ * canary that can't run, an index still mid-rebuild, or a ledger-write
+ * failure all degrade to "not verified this boot, retry next boot" or a
+ * best-effort skip — never an exception into the boot path.
+ *
+ * DEFENSE-IN-DEPTH RULE (a prudent, version-independent default — NOT a
+ * workaround for a live bug): any BULK re-embed of the corpus should be paired
+ * with a structural graph REBUILD (a full runIndexing pass) rather than
+ * relying on the engine's incremental `index()` updates to converge. The
+ * installed Harper (5.1.22) DOES converge — its update path is fixed — but the
+ * rebuild keeps recall correct regardless of engine version, closes the rare
+ * decode-failure edge case, and is what heals a store already corrupted on
+ * disk by an older engine. The lever flair uses for that rebuild is a
+ * structural-schema change (the M:16 descriptor bump that ships with this
+ * migration) — see the guard comments in
+ * resources/migrations/embedding-stamp.ts and the `flair reembed` path in
+ * src/cli.ts. Do NOT use resources/MemoryReindex.ts's `_reindex` for graph
+ * correctness — it re-PUTs the same vector through the incremental path and
+ * does not rebuild the graph.
+ */
+import { databases } from "@harperfast/harper";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+export const GRAPH_HEAL_ID = "graph-heal";
+/** How many rows to scan for a canary that carries a real embedding. Bounded — detect() must stay cheap. */
+const CANARY_SCAN_LIMIT = 8;
+function defaultMemoryTable() {
+    return databases.flair.Memory;
+}
+function defaultOrgEventTable() {
+    return databases.flair.OrgEvent;
+}
+/**
+ * Same "resolve the running package's own version" idiom as
+ * resources/health.ts / resources/migration-boot.ts. Inlined (not imported
+ * from migration-boot) to avoid an import cycle: migration-boot → registry →
+ * graph-heal. Purely informational — it only stamps the ledger detail blob;
+ * a "dev" fallback is harmless.
+ */
+function resolveRunningVersionLocal() {
+    try {
+        const here = dirname(fileURLToPath(import.meta.url));
+        const candidates = [
+            join(here, "..", "..", "..", "package.json"), // dist/resources/migrations → root
+            join(here, "..", "..", "package.json"),
+        ];
+        for (const p of candidates) {
+            if (existsSync(p)) {
+                const pkg = JSON.parse(readFileSync(p, "utf-8"));
+                if (pkg.version)
+                    return String(pkg.version);
+            }
+        }
+    }
+    catch {
+        /* fall through */
+    }
+    return process.env.npm_package_version ?? "dev";
+}
+/**
+ * Pick a canary row that carries a real embedding vector: scan a bounded
+ * handful of rows and `.get()` the first one whose stored `embedding` is a
+ * non-empty array. `.get()` (a primary-key read) always returns the full
+ * record including `embedding`, independent of any `select`/projection
+ * quirk. Returns null when the corpus has no embedded rows at all — a fresh
+ * or empty instance, where there is simply nothing to heal or verify.
+ */
+async function pickCanary(table) {
+    for await (const row of table.search({ select: ["id"], limit: CANARY_SCAN_LIMIT })) {
+        const id = String(row.id ?? "");
+        if (!id)
+            continue;
+        const full = await table.get(id);
+        const emb = full?.embedding;
+        if (Array.isArray(emb) && emb.length > 0)
+            return { id, embedding: emb };
+    }
+    return null;
+}
+/**
+ * The id of the HNSW nearest neighbor of `target` — the exact cosine-sort
+ * query the retrieval path (resources/semantic-retrieval-core.ts) uses,
+ * bounded to rank 1. Throws if the index is unavailable / mid-rebuild; the
+ * caller treats a throw as "not verified this boot".
+ */
+async function topNeighborId(table, target) {
+    for await (const row of table.search({
+        sort: { attribute: "embedding", target, distance: "cosine" },
+        select: ["id"],
+        limit: 1,
+    })) {
+        const id = row.id;
+        return id ? String(id) : null;
+    }
+    return null;
+}
+/**
+ * Canary self-recall: a healthy HNSW graph returns the canary row itself as
+ * the rank-1 neighbor of its OWN stored vector. Cheap, read-only, bounded —
+ * and the cleanest black-box proof the graph rebuilt and is serving (a
+ * mid-rebuild index throws or returns nothing; a store with no embedded rows
+ * has no canary). Never throws (invariant II).
+ */
+async function recallHealthy(table) {
+    try {
+        const canary = await pickCanary(table);
+        if (!canary)
+            return { verified: false, hasData: false };
+        const top = await topNeighborId(table, canary.embedding);
+        return { verified: top === canary.id, hasData: true };
+    }
+    catch {
+        return { verified: false, hasData: true };
+    }
+}
+/** Count rows carrying a real (non-hash-fallback) embedding — the vectors that live in the HNSW graph. Best-effort. */
+async function countEmbeddedVectors(table) {
+    try {
+        let n = 0;
+        for await (const row of table.search({ select: ["embeddingModel"] })) {
+            const m = row.embeddingModel;
+            if (typeof m === "string" && m.length > 0 && m !== "hash-512d")
+                n++;
+        }
+        return n;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Structural-only ledger OrgEvent (Sherlock discipline, same as
+ * resources/migrations/ledger.ts): counts, outcome, version — NEVER a memory
+ * id or content (the canary row's id is deliberately omitted). Written via an
+ * internal (no-HTTP-context) put — resolveAgentAuth(undefined) → internal, the
+ * same trusted server-internal write path the migration ledger already uses.
+ */
+async function writeGraphHealEvent(obs, table) {
+    const countNote = obs.embeddedVectorCount != null
+        ? ` (${obs.embeddedVectorCount} embedded vector${obs.embeddedVectorCount === 1 ? "" : "s"})`
+        : "";
+    await table.put({
+        id: `migration-graph-heal-verified-${obs.at}`,
+        authorId: "flair-migrations",
+        kind: "migration",
+        scope: "full",
+        summary: `HNSW graph-heal: recall ${obs.verified ? "verified healthy" : "unconfirmed"}${countNote}`,
+        detail: JSON.stringify({
+            migrationId: GRAPH_HEAL_ID,
+            verified: obs.verified,
+            canaryRank1: obs.verified,
+            embeddedVectorCount: obs.embeddedVectorCount,
+            runningVersion: obs.runningVersion,
+            verifiedAt: obs.at,
+        }),
+        refId: GRAPH_HEAL_ID,
+        createdAt: obs.at,
+    });
+}
+/**
+ * `getTable`/`getOrgEventTable`/`getVersion`/`now` are injectable so tests can
+ * exercise detect/countPending/run against fakes (matching the mocking style
+ * of embedding-stamp.ts / the unit tests). The real HNSW self-recall + ledger
+ * path is exercised end-to-end in
+ * test/integration/hnsw-graph-heal-e2e.test.ts.
+ */
+export function createGraphHealMigration(getTable = defaultMemoryTable, getOrgEventTable = defaultOrgEventTable, getVersion = resolveRunningVersionLocal, now = () => new Date()) {
+    return {
+        id: GRAPH_HEAL_ID,
+        riskClass: "derived-only",
+        affectsTables: ["Memory"],
+        /**
+         * True only when recall is confirmed healthy AND there is embedded data
+         * to verify — i.e. the graph rebuilt and is serving. Returns false (the
+         * runner marks the migration completed for this boot WITHOUT writing a
+         * success state entry, so detect() runs again next boot) when the index
+         * is still mid-rebuild or the store has nothing embedded yet. This is the
+         * enrollment gate: only a confirmed-healthy boot proceeds to run() +
+         * ledger + the version short-circuit.
+         */
+        async detect() {
+            const { verified } = await recallHealthy(getTable());
+            return verified;
+        },
+        /** Observe-only — never any pending row work. */
+        async countPending() {
+            return 0;
+        },
+        /**
+         * Called exactly once (countPending() is 0 → the runner's batch loop runs
+         * once and breaks). Re-verifies recall for the ledger snapshot, counts
+         * the embedded vectors, writes ONE structural-only OrgEvent, and returns
+         * `processed: 0` so the completion gate (count+marker, derived-only)
+         * passes and the runner records success → short-circuit next boot.
+         */
+        async run(_batchSize) {
+            const table = getTable();
+            const { verified } = await recallHealthy(table);
+            const embeddedVectorCount = await countEmbeddedVectors(table);
+            try {
+                await writeGraphHealEvent({ verified, embeddedVectorCount, runningVersion: getVersion(), at: now().toISOString() }, getOrgEventTable());
+            }
+            catch {
+                // Observability must never brick the runner — a ledger-write failure
+                // is swallowed; the heal itself already happened schema-side.
+            }
+            return { processed: 0, touchedIds: [] };
+        },
+    };
+}

package/dist/resources/migrations/registry.js

@@ -1,10 +1,11 @@
 /**
  * registry.ts — the MigrationRegistry: an ordered list of registered
- * migrations. embedding-stamp always registers; the synthetic CI-only
- * variant registers ONLY when shouldRegisterSyntheticMigration() is true
- * (see synthetic-test-migration.ts's doc for the exact gating rule).
+ * migrations. embedding-stamp and graph-heal always register; the synthetic
+ * CI-only variant registers ONLY when shouldRegisterSyntheticMigration() is
+ * true (see synthetic-test-migration.ts's doc for the exact gating rule).
  */
 import { createEmbeddingStampMigration } from "./embedding-stamp.js";
+import { createGraphHealMigration } from "./graph-heal.js";
 import { createSyntheticTestMigration, shouldRegisterSyntheticMigration } from "./synthetic-test-migration.js";
 export class MigrationRegistry {
     migrations = [];
@@ -23,12 +24,17 @@ export class MigrationRegistry {
     }
 }
 /**
- * Builds the production registry. Always includes embedding-stamp;
- * conditionally includes the CI-only synthetic migration.
+ * Builds the production registry. Always includes embedding-stamp and
+ * graph-heal; conditionally includes the CI-only synthetic migration.
+ *
+ * Order note: graph-heal registers AFTER embedding-stamp so that if a boot
+ * ever runs both, the (verify-only) recall check ledgers the state AFTER any
+ * re-embed work that cycle already applied — never a stale pre-embed reading.
  */
 export function buildRegistry(env = process.env) {
     const registry = new MigrationRegistry();
     registry.register(createEmbeddingStampMigration());
+    registry.register(createGraphHealMigration());
     if (shouldRegisterSyntheticMigration(env)) {
         registry.register(createSyntheticTestMigration());
     }

package/dist/resources/models-dir.js

@@ -0,0 +1,55 @@
+/**
+ * models-dir.ts — the ONE place that decides where model GGUFs live.
+ *
+ * Extracted from embeddings-provider.ts (flair#815) so the reranker
+ * (rerank-provider.ts) can share the exact same resolution WITHOUT importing
+ * embeddings-provider: several unit-isolated tests `mock.module()` the whole
+ * embeddings-provider module (with only the named exports THEY consume), and
+ * bun's module cache is process-wide — so any new cross-module import of
+ * embeddings-provider from another resource makes those mocks incomplete and
+ * kills unrelated test files at module load. This module is tiny, pure
+ * node-stdlib, and never mocked, so both providers (and any future model
+ * consumer) can depend on it safely.
+ */
+import { existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+/**
+ * Resolve the directory model GGUFs live in / download into — shared by the
+ * embedding engine (embeddings-boot.ts's `register()` options) and the
+ * cross-encoder reranker (rerank-provider.ts, flair#815 — it used to hardcode
+ * `<cwd>/models`, silently failing open wherever Harper's cwd wasn't the
+ * models location).
+ *
+ * Resolution order (everything writable, never the read-only package dir):
+ *   1. FLAIR_MODELS_DIR        — explicit operator/docker override.
+ *   2. <ROOTPATH>/models       — Harper's data dir (Flair passes ROOTPATH =
+ *                                ~/.flair/data when it spawns Harper). User-
+ *                                owned and writable even on sudo-global installs.
+ *   3. <cwd>/models            — ONLY if a model already lives there. Backward
+ *                                compat for existing writable installs that
+ *                                downloaded into the package dir before this fix;
+ *                                never used as a download target on fresh installs.
+ *   4. ~/.flair/data/models    — last-resort default when ROOTPATH is unset.
+ *
+ * The chosen dir is always writable, so the embeddings engine can download the
+ * model on first use without hitting EACCES on a root-owned package dir.
+ *
+ * Tested independently (test/unit/embeddings-models-dir.test.ts, via
+ * embeddings-provider's re-export) as the single documented source of truth
+ * for this default.
+ */
+export function resolveModelsDir() {
+    const override = process.env.FLAIR_MODELS_DIR;
+    if (override)
+        return override;
+    const rootPath = process.env.ROOTPATH;
+    if (rootPath)
+        return join(rootPath, "models");
+    // Backward compat: a prior (writable) install may have the model cached in
+    // the package dir. Reuse it rather than re-downloading — but only if present.
+    const cwdModels = join(process.cwd(), "models");
+    if (existsSync(cwdModels))
+        return cwdModels;
+    return join(homedir(), ".flair", "data", "models");
+}

package/dist/resources/record-types.js

@@ -127,8 +127,8 @@
  *
  * `remEligible` — MUST be `false` for every entry in v1 (typed as the
  * literal `false`, not `boolean`, so a stray `true` is a compile error, not
- * just a runtime policy mistake). REM's nightly gather step
- * (specs/FLAIR-NIGHTLY-REM.md §11) is hardcoded to `GET /Memory?agentId=…`
+ * just a runtime policy mistake). REM's nightly gather step (see docs/rem.md)
+ * is hardcoded to `GET /Memory?agentId=…`
  * — single table, single agent, by construction. There is no multi-table
  * distillation input path to opt into yet; the field exists so a future
  * REM generalization doesn't require a breaking schema change to every