@tpsdev-ai/flair 0.53.0 → 0.54.2

package/dist/resources/migrations/embedding-stamp.js

@@ -19,6 +19,14 @@
  * migrations-embedding-stamp-e2e.test.ts`) before it ever had real work to
  * do was the point.
  *
+ * flair#1073: that self-heal sat idle for days on a Fabric instance. The
+ * runner is boot-keyed; a first detect() against a not-yet-visible corpus
+ * (or a version-keyed short-circuit after a no-op complete) marked this
+ * migration done while 554 pre-flip rows stayed stale. `alwaysDetect`
+ * plus delayed follow-up cycles close that; /HealthDetail names this
+ * migration when the corpus is still split. run() has no batch cap — the
+ * runner loops until processed=0 — and a re-run is idempotent.
+ *
  * Reuses Memory's OWN regen branch — never duplicates embedding logic —
  * via the SAME mechanism `flair reembed` (src/cli.ts) already uses in
  * production: a genuine `PUT /Memory/:id` HTTP request (admin-authenticated
@@ -128,10 +136,11 @@
 import { databases } from "harper";
 import { getModelId } from "../embeddings-provider.js";
 import { currentSpaceRawForms, isCurrentSpaceStamp } from "../embedding-space-guard.js";
+import { EMBEDDING_STAMP_ID } from "./stamp-outstanding.js";
+export { EMBEDDING_STAMP_ID };
 function defaultMemoryTable() {
     return databases.flair.Memory;
 }
-export const EMBEDDING_STAMP_ID = "embedding-stamp";
 const REGEN_HTTP_TIMEOUT_MS = 20_000; // a real embedding compute can be slow on constrained hardware
 /** Same admin-password resolution as resources/auth-middleware.ts's getAdminPass(). */
 function resolveAdminAuthHeader() {
@@ -224,6 +233,16 @@ export function createEmbeddingStampMigration(getTable = defaultMemoryTable, get
         id: EMBEDDING_STAMP_ID,
         riskClass: "derived-only",
         affectsTables: ["Memory"],
+        // flair#1073: pending work is "does any row's stamp differ from
+        // getModelId()", not "did this version already record success." A
+        // Fabric boot that detect()'d empty (tables ready, corpus not yet
+        // visible) or a no-op complete at the searchprefix-flip version wrote
+        // success and then skipped every later boot at that version — 554
+        // pre-flip rows sat stale for days. alwaysDetect keeps the cheap
+        // limit=1 corpus read on every cycle so a false complete cannot hide
+        // remaining work. run() is idempotent; a re-run over already-current
+        // rows is a no-op.
+        alwaysDetect: true,
         async detect() {
             const table = getTable();
             for await (const _row of table.search({ conditions: staleCondition(), limit: 1 })) {

package/dist/resources/migrations/recheck.js

@@ -0,0 +1,43 @@
+/**
+ * recheck.ts — follow-up migration cycles after the boot-keyed first pass
+ * (flair#1073).
+ *
+ * The boot runner fires once via `setImmediate` after tables exist. On a
+ * Fabric component deploy that is not enough: Memory.search can be a live
+ * accessor before the replica's rows are visible, so embedding-stamp's
+ * detect() returns false, the cycle records "nothing pending" / completed,
+ * and a long-lived process never looks again. These delayed rechecks are
+ * the live self-heal — the same runMigrationCycle, single-flight lock
+ * included, so a no-op is cheap (alwaysDetect detect() is limit=1) and a
+ * mid-flight first cycle just gets `single-flight` from the later one.
+ *
+ * Delays are spaced to cover the Fabric boot race (tens of seconds) and a
+ * slow replica catch-up (minutes), not an infinite poll. A process that
+ * is still split after the last delay will be picked up on the next
+ * restart (alwaysDetect) or the next `upgrade --target` stamp-convergence
+ * verify.
+ */
+export const DEFAULT_STAMP_RECHECK_DELAYS_MS = [
+    30_000,
+    120_000,
+    600_000,
+];
+export function scheduleFollowUpCycles(opts) {
+    const delaysMs = opts.delaysMs ?? DEFAULT_STAMP_RECHECK_DELAYS_MS;
+    const setT = opts.setTimeoutFn ?? setTimeout;
+    const clearT = opts.clearTimeoutFn ?? clearTimeout;
+    const timers = [];
+    for (const ms of delaysMs) {
+        timers.push(setT(() => {
+            void opts.run();
+        }, ms));
+    }
+    return {
+        delaysMs,
+        cancel() {
+            for (const t of timers)
+                clearT(t);
+            timers.length = 0;
+        },
+    };
+}

package/dist/resources/migrations/runner.js

@@ -150,7 +150,7 @@ async function runCycleLocked(deps, lock) {
     const state = readMigrationState(deps.statePath);
     const candidates = [];
     for (const migration of deps.registry.list()) {
-        if (isShortCircuited(state, migration.id, deps.runningVersion)) {
+        if (isShortCircuited(state, migration.id, deps.runningVersion) && !migration.alwaysDetect) {
             // flair#812: a short-circuit is an ASSERTION READ OFF A FILE, not an
             // observation of the corpus — and that file is hand-editable (the
             // documented remediation for a stuck migration is to correct it by
@@ -159,6 +159,11 @@ async function runCycleLocked(deps, lock) {
             // The skip itself is unchanged (it is the documented cheap path); it
             // now just says what it is, so `flair doctor` never presents an
             // unverified claim as a verified one.
+            //
+            // flair#1073: `alwaysDetect` migrations skip this path — their
+            // pending work is a corpus property, so a version-keyed "already
+            // done" marker can lie (embedding-stamp completed at 0.30.0 while
+            // 554 pre-flip rows were still stale). detect() is still cheap.
             setMigrationProgress({
                 id: migration.id,
                 rowsDone: 0,

package/dist/resources/migrations/stamp-outstanding.js

@@ -0,0 +1,171 @@
+/**
+ * stamp-outstanding.ts — corpus-derived "is embedding-stamp still pending?"
+ * (flair#1073). Harper-free, no embedding-space-guard import: that module
+ * schedules a boot pre-warm at load. This file is the Harper/HealthDetail
+ * canonical. The CLI ships a copy at src/stamp-outstanding.ts because
+ * tsconfig.cli.json's rootDir is `src` and the published tree cannot
+ * resolve `../resources/...` from dist/cli.js. Stay in sync via
+ * test/unit/stamp-outstanding-cli-parity.test.ts.
+ *
+ * THE point of this module: runner bookkeeping (`state: completed`,
+ * `rowsDone: 0`) is not the signal. A Fabric boot that detect()'d empty,
+ * or a version-keyed short-circuit, can mark embedding-stamp done while
+ * the corpus is still split. Outstanding is derived from modelCounts vs
+ * the current space, then annotated with whatever the runner last said
+ * so the warning can name the migration AND the consequences (search
+ * unreliable, dedup inactive) instead of a generic mixed-models symptom
+ * with a manual `flair reembed` remedy.
+ */
+/** Stable id — kept here so Harper resources can import this file without
+ *  loading embedding-stamp.ts (which statically imports `harper`).
+ *  Re-exported from embedding-stamp.ts so existing imports keep working.
+ *  The CLI copy at src/stamp-outstanding.ts exports the same constant. */
+export const EMBEDDING_STAMP_ID = "embedding-stamp";
+/** Mirrors embedding-space-guard's normalizeStamp / NON_SPACE_STAMPS —
+ *  inlined so this file never loads that module's boot side effect. */
+const NON_SPACE_STAMPS = new Set(["hash-512d"]);
+const DEFAULT_ENGINE = "gguf";
+export function normalizeStampForOutstanding(stamp) {
+    if (stamp == null)
+        return null;
+    const s = String(stamp).trim();
+    if (s === "" || NON_SPACE_STAMPS.has(s))
+        return null;
+    return s.includes(":") ? s : `${DEFAULT_ENGINE}:${s}`;
+}
+/**
+ * Current-space id for a client that only has `modelCounts` (no live
+ * `getModelId()`). Prefers an existing `+searchprefix` stamp so a healthy
+ * bare+`gguf:` pair of the SAME space does not invent a foreign current.
+ * A uniformly pre-flip corpus has no suffix — append `+searchprefix` so
+ * those rows read as stale (the HealthDetail / quality-report gap Bugbot
+ * flagged on flair#1606).
+ */
+export function resolveCurrentModelId(modelCounts, explicit) {
+    const trimmed = explicit?.trim();
+    if (trimmed)
+        return trimmed;
+    let prefixed;
+    let anyReal;
+    for (const [stamp, n] of Object.entries(modelCounts)) {
+        if (NON_SPACE_STAMPS.has(stamp) || typeof n !== "number" || n <= 0)
+            continue;
+        anyReal ??= stamp;
+        if (stamp.includes("+searchprefix")) {
+            if (!prefixed || stamp.includes(":"))
+                prefixed = stamp;
+        }
+    }
+    if (prefixed)
+        return prefixed;
+    if (anyReal) {
+        const bare = anyReal.includes(":") ? anyReal.slice(anyReal.indexOf(":") + 1) : anyReal;
+        return `${DEFAULT_ENGINE}:${bare}+searchprefix`;
+    }
+    return `${DEFAULT_ENGINE}:nomic-embed-text-v1.5-Q4_K_M+searchprefix`;
+}
+/**
+ * Split the corpus into current-space vs foreign-space counts. A bare
+ * stamp and its `gguf:` equivalent are the SAME space; `+searchprefix`
+ * vs bare-without-suffix are NOT (that split is this migration's payload).
+ */
+export function countStampSpaces(modelCounts, currentModelId) {
+    const currentSpace = normalizeStampForOutstanding(currentModelId);
+    let currentCount = 0;
+    let staleCount = 0;
+    const staleStamps = [];
+    for (const [raw, n] of Object.entries(modelCounts)) {
+        if (typeof n !== "number" || n <= 0)
+            continue;
+        const space = normalizeStampForOutstanding(raw);
+        if (space === null)
+            continue;
+        if (currentSpace !== null && space === currentSpace) {
+            currentCount += n;
+        }
+        else {
+            staleCount += n;
+            staleStamps.push(`${raw}:${n}`);
+        }
+    }
+    return { currentCount, staleCount, staleStamps, currentSpace };
+}
+function runnerAnnotation(input) {
+    const mig = input.migration;
+    const phase = input.cyclePhase;
+    if (input.lastCycleError) {
+        return ` last boot cycle failed (${input.lastCycleError})`;
+    }
+    if (phase === "idle") {
+        return " migration boot cycle never fired on this instance — no migration will run until this is resolved";
+    }
+    if (mig?.state === "running" || mig?.state === "checking" || mig?.state === "preflight" || mig?.state === "snapshotting" || mig?.state === "completing") {
+        const rem = typeof mig.rowsRemaining === "number" ? `, ${mig.rowsRemaining} remaining` : "";
+        return ` in progress (${mig.state}${rem})`;
+    }
+    if (mig?.state === "halted" || mig?.state === "failed") {
+        return ` ${mig.state}${mig.reason ? `: ${mig.reason}` : ""} — see \`flair doctor\``;
+    }
+    if (mig?.state === "completed" || phase === "done") {
+        return " the last cycle marked it complete without converging; it will retry automatically";
+    }
+    if (input.audience === "client") {
+        return " the Harper process applies it on its next migration cycle — this CLI does not";
+    }
+    return " it will apply automatically on this process's next migration cycle";
+}
+/**
+ * Corpus-derived outstanding check. Returns `outstanding: false` when every
+ * real-space stamp matches the current space (or the store is empty of
+ * real embeddings). Runner bookkeeping cannot override a split corpus.
+ */
+export function describeStampOutstanding(input) {
+    const { currentCount, staleCount, staleStamps } = countStampSpaces(input.modelCounts, input.currentModelId);
+    if (staleCount === 0) {
+        return { outstanding: false, staleCount: 0, currentCount };
+    }
+    const list = staleStamps.join(", ");
+    const warning = `migration '${EMBEDDING_STAMP_ID}' is outstanding ` +
+        `(${staleCount} row${staleCount === 1 ? "" : "s"} still on ${list}` +
+        `${currentCount > 0 ? `, ${currentCount} already current` : ""}) — ` +
+        `cross-model search is unreliable and duplicate detection is inactive ` +
+        `until the re-embed completes;` +
+        runnerAnnotation(input);
+    return {
+        outstanding: true,
+        migrationId: EMBEDDING_STAMP_ID,
+        staleCount,
+        currentCount,
+        staleStamps,
+        warning,
+    };
+}
+/**
+ * Convergence predicate for `upgrade --target` / `deploy` post-verify
+ * (flair#1073). A split corpus is not converged. A halted/failed
+ * embedding-stamp is not converged. A still-running cycle is not
+ * converged. Missing migration state on a split corpus is not converged.
+ * An empty / already-current corpus is converged.
+ */
+export function stampMigrationConverged(input) {
+    const described = describeStampOutstanding(input);
+    if (described.outstanding) {
+        return { converged: false, detail: described.warning };
+    }
+    const mig = input.migration;
+    if (mig?.state === "halted" || mig?.state === "failed") {
+        return {
+            converged: false,
+            detail: `migration '${EMBEDDING_STAMP_ID}' ${mig.state}${mig.reason ? `: ${mig.reason}` : ""}`,
+        };
+    }
+    if (mig &&
+        (mig.state === "running" ||
+            mig.state === "checking" ||
+            mig.state === "preflight" ||
+            mig.state === "snapshotting" ||
+            mig.state === "completing")) {
+        return { converged: false, detail: `migration '${EMBEDDING_STAMP_ID}' still ${mig.state}` };
+    }
+    return { converged: true, detail: "embedding-stamp converged (corpus is a single current space)" };
+}

package/dist/resources/migrations/visibility-backfill.js

@@ -12,7 +12,7 @@
  * WHY THIS EXISTS: `resources/Memory.ts`'s `defaultVisibilityForDurability`
  * (flair#509) only stamps `visibility` on a WRITE — a row that has sat
  * untouched since before that slice shipped has no `visibility` field at
- * all. `src/cli.ts`'s federation push filter (`isFederationPrivateVisibility`)
+ * all. `src/commands/federation.ts`'s federation push filter (`isFederationPrivateVisibility`)
  * excludes only `visibility === "private"` — by design, per
  * `resources/memory-visibility.ts`'s migration invariant, an ABSENT
  * `visibility` field is NOT private and must keep syncing exactly as
@@ -38,7 +38,7 @@
  * EXTENDS that only resolves once a live Harper has already registered the
  * base `Memory` resource, which unit tests (no real Harper; see
  * `test/unit/migrations-visibility-backfill.test.ts`) cannot satisfy. Same
- * reasoning, same shape, as `src/cli.ts`'s `isFederationPrivateVisibility`
+ * reasoning, same shape, as `src/commands/federation.ts`'s `isFederationPrivateVisibility`
  * duplicating `resources/memory-visibility.ts`'s `isPrivateVisibility`
  * rather than importing it (see that file's comment) — both are one-line,
  * pure, and load-bearing enough that a comment pointing at the canonical

package/dist/resources/org-event-catchup-lib.js

@@ -0,0 +1,47 @@
+/**
+ * org-event-catchup-lib.ts — Harper-free OrgEvent catch-up filters + paging.
+ *
+ * Targeting / expiry / position-order live here so OrgEventCatchup and
+ * MemoryBootstrap share one definition (and unit tests can pin it without
+ * spinning Harper). Watermark storage is resources/agent-read-position.ts.
+ */
+import { comparePosition, createdAtFloorFromPosition, laterTimestamp, pageAfter, recordPosition, } from "./agent-read-position-lib.js";
+export function eventTargetsParticipant(event, participantId) {
+    const targets = event.targetIds;
+    return !targets || targets.length === 0 || targets.includes(participantId);
+}
+export function eventIsExpired(event, now = new Date()) {
+    return Boolean(event.expiresAt && new Date(event.expiresAt) < now);
+}
+export function withEventPosition(event) {
+    return { ...event, position: recordPosition(event) };
+}
+export function catchupSeekTimestamp(after, since) {
+    return laterTimestamp(createdAtFloorFromPosition(after), since ?? null);
+}
+export function isCatchupEligible(event, opts) {
+    if (!event.createdAt)
+        return false;
+    if (opts.since && event.createdAt < opts.since)
+        return false;
+    if (comparePosition(recordPosition(event), opts.after) <= 0)
+        return false;
+    if (!eventTargetsParticipant(event, opts.participantId))
+        return false;
+    if (eventIsExpired(event, opts.now ?? new Date()))
+        return false;
+    return true;
+}
+export async function collectCatchupEvents(rows, opts) {
+    const results = [];
+    for await (const event of rows) {
+        if (!isCatchupEligible(event, opts))
+            continue;
+        results.push(withEventPosition(event));
+    }
+    results.sort((a, b) => comparePosition(a.position, b.position));
+    return results;
+}
+export function pageCatchupEvents(ordered, after, pageSize) {
+    return pageAfter(ordered, after, pageSize);
+}

package/dist/resources/record-owner-guard.js

@@ -84,6 +84,7 @@
  * being absent on the day someone adds one.
  */
 export const OWNER_FIELDS = Object.freeze({
+    AgentReadPosition: "agentId",
     Asset: "agentId",
     Credential: "principalId",
     Integration: "agentId",