@tpsdev-ai/flair 0.53.0 → 0.54.1

package/dist/resources/federation-peer-liveness.js

@@ -0,0 +1,73 @@
+/**
+ * Pure classifier for federation peer liveness — no Harper imports.
+ *
+ * flair#1499: HealthDetail used to count `status === "connected"` (a value
+ * pairing never writes — spokes stay `paired` even after a successful
+ * lastSyncAt advance) and then take the oldest lastSyncAt across ALL peers,
+ * including revoked. A healthy hub synced a minute ago plus a revoked row
+ * from June therefore rendered as "0 connected" and fired
+ * "federation peers all disconnected >24h".
+ *
+ * Three contact states, plus revoked (which never drives the staleness
+ * warning). Same class as #988: missing/unreadable evidence is UNKNOWN,
+ * never DISCONNECTED.
+ *
+ *   (a) real lastSyncAt within the window → connected
+ *   (b) real lastSyncAt older than the window → disconnected
+ *   (c) no parseable lastSyncAt → unknown (never rendered as (b))
+ *   revoked stored status → revoked (excluded from the >24h warning)
+ */
+export const PEER_STALE_MS = 24 * 3600 * 1000;
+export const FEDERATION_PEERS_ALL_DISCONNECTED_WARNING = "federation peers all disconnected >24h";
+/** Parse a written last-contact stamp. Missing / unparseable → null, never 0/epoch. */
+export function parseLastContactMs(lastSyncAt) {
+    if (typeof lastSyncAt !== "string")
+        return null;
+    const trimmed = lastSyncAt.trim();
+    if (!trimmed)
+        return null;
+    const t = Date.parse(trimmed);
+    return Number.isFinite(t) ? t : null;
+}
+export function classifyPeerLiveness(peer, nowMs, windowMs = PEER_STALE_MS) {
+    if (peer.status === "revoked")
+        return "revoked";
+    const ts = parseLastContactMs(peer.lastSyncAt);
+    if (ts === null)
+        return "unknown";
+    // Future / equal-now stamps are contact within the window (clock skew).
+    if (nowMs - ts > windowMs)
+        return "disconnected";
+    return "connected";
+}
+export function summarizePeerLiveness(peers, nowMs, windowMs = PEER_STALE_MS) {
+    let connected = 0;
+    let disconnected = 0;
+    let revoked = 0;
+    let unknown = 0;
+    for (const peer of peers) {
+        const liveness = classifyPeerLiveness(peer, nowMs, windowMs);
+        if (liveness === "connected")
+            connected++;
+        else if (liveness === "disconnected")
+            disconnected++;
+        else if (liveness === "revoked")
+            revoked++;
+        else
+            unknown++;
+    }
+    const nonRevoked = peers.length - revoked;
+    return {
+        total: peers.length,
+        connected,
+        disconnected,
+        revoked,
+        unknown,
+        allNonRevokedDisconnected: nonRevoked > 0 && disconnected === nonRevoked,
+    };
+}
+export function federationPeersAllDisconnectedWarning(summary) {
+    if (!summary.allNonRevokedDisconnected)
+        return null;
+    return { level: "warn", message: FEDERATION_PEERS_ALL_DISCONNECTED_WARNING };
+}

package/dist/resources/health.js

@@ -11,7 +11,10 @@ import { REM_DEDUP_STATS_PATH } from "./dedup-cluster.js";
 import { hybridEnabled } from "./bm25.js";
 import { bm25IndexEnabled, bm25IndexStatus } from "./bm25-index-service.js";
 import { normalizeStamp } from "./embedding-space-guard.js";
+import { getModelId } from "./embeddings-provider.js";
+import { describeStampOutstanding, EMBEDDING_STAMP_ID } from "./migrations/stamp-outstanding.js";
 import { buildPublicHealthBody, resolveSearchReadiness } from "./search-readiness.js";
+import { classifyPeerLiveness, federationPeersAllDisconnectedWarning, summarizePeerLiveness, } from "./federation-peer-liveness.js";
 const db = databases;
 const redactHome = (p) => {
     const home = homedir();
@@ -160,6 +163,14 @@ export class HealthDetail extends Resource {
         const stats = { ok: true };
         const nowMs = Date.now();
         const warnings = [];
+        // flair#1073: set while walking memories, consumed after the migrations
+        // snapshot so the outstanding-migration warning can name runner state.
+        // Never copied onto `stats` — it is not part of the /HealthDetail shape.
+        // `memoryScanOk` is the gate (not "mixed spaces"): a uniformly pre-flip
+        // corpus is one space and still outstanding against getModelId().
+        let mixedEmbeddingSpaces = false;
+        let memoryScanOk = false;
+        let memoryModelCounts = {};
         // flair#1326: same search-ready signal as public /Health. HealthDetail
         // stays HTTP 200 (it is a stats dump, not a traffic gate); the field
         // and a warning name the lag so `flair status` / operators can see it.
@@ -258,13 +269,12 @@ export class HealthDetail extends Resource {
             // re-embed) still trips it, and that is exactly when the query-time guard
             // degrades recall to keyword-only.
             const distinctSpaces = new Set(realModels.map((k) => normalizeStamp(k)).filter((s) => s !== null));
-            if (distinctSpaces.size > 1) {
-                const list = realModels.map((k) => `${k}:${modelCounts[k]}`).join(", ");
-                warnings.push({
-                    level: "warn",
-                    message: `multiple embedding models in use (${list}) — cross-model search unreliable; run: flair reembed against one model`,
-                });
-            }
+            // Always keep counts — a uniformly stale pre-flip corpus is one space
+            // and must still name embedding-stamp (Bugbot High on flair#1606).
+            memoryScanOk = true;
+            memoryModelCounts = modelCounts;
+            if (distinctSpaces.size > 1)
+                mixedEmbeddingSpaces = true;
         }
         catch {
             stats.memories = null;
@@ -378,11 +388,16 @@ export class HealthDetail extends Resource {
             }
             else {
                 const inst = instances[0];
+                // flair#1499: derive connected/down from lastSyncAt, not stored
+                // status. Pairing writes `paired`; a recent lastSyncAt is connected.
+                // Revoked rows are counted separately and never drive the >24h warning.
+                const liveness = summarizePeerLiveness(peers, nowMs);
                 const peersBlock = {
-                    total: peers.length,
-                    connected: peers.filter((p) => p.status === "connected").length,
-                    disconnected: peers.filter((p) => p.status === "disconnected").length,
-                    revoked: peers.filter((p) => p.status === "revoked").length,
+                    total: liveness.total,
+                    connected: liveness.connected,
+                    disconnected: liveness.disconnected,
+                    revoked: liveness.revoked,
+                    unknown: liveness.unknown,
                 };
                 const pendingTokens = tokens.filter((t) => !t.consumedBy && t.expiresAt && new Date(t.expiresAt).getTime() > nowMs).length;
                 stats.federation = {
@@ -395,17 +410,13 @@ export class HealthDetail extends Resource {
                             role: p.role,
                             status: p.status,
                             lastSyncAt: p.lastSyncAt ?? null,
+                            liveness: classifyPeerLiveness(p, nowMs),
                         }))
                         : undefined,
                 };
-                if (peers.length > 0 && peersBlock.connected === 0) {
-                    const oldest = peers
-                        .map((p) => (p.lastSyncAt ? new Date(p.lastSyncAt).getTime() : 0))
-                        .reduce((a, b) => (a === 0 ? b : b === 0 ? a : Math.min(a, b)), 0);
-                    if (oldest > 0 && nowMs - oldest > 24 * 3600 * 1000) {
-                        warnings.push({ level: "warn", message: "federation peers all disconnected >24h" });
-                    }
-                }
+                const disconnectWarn = federationPeersAllDisconnectedWarning(liveness);
+                if (disconnectWarn)
+                    warnings.push(disconnectWarn);
                 if (pendingTokens > 0) {
                     warnings.push({ level: "info", message: `${pendingTokens} pairing token(s) unconsumed` });
                 }
@@ -660,9 +671,47 @@ export class HealthDetail extends Resource {
                     message: "migration boot cycle never fired on this instance (cyclePhase=idle) — no migration will run until this is resolved; see `flair doctor`",
                 });
             }
+            if (memoryScanOk) {
+                const stamp = snapshot.migrations.find((m) => m.id === EMBEDDING_STAMP_ID);
+                const outstanding = describeStampOutstanding({
+                    modelCounts: memoryModelCounts,
+                    currentModelId: getModelId(),
+                    migration: stamp,
+                    cyclePhase: snapshot.cyclePhase,
+                    lastCycleError: snapshot.lastCycleError,
+                });
+                if (outstanding.outstanding) {
+                    warnings.push({ level: "warn", message: outstanding.warning });
+                }
+                else if (mixedEmbeddingSpaces) {
+                    const list = Object.entries(memoryModelCounts)
+                        .filter(([k, n]) => k !== "hash-512d" && n > 0)
+                        .map(([k, n]) => `${k}:${n}`)
+                        .join(", ");
+                    warnings.push({
+                        level: "warn",
+                        message: `multiple embedding models in use (${list}) — cross-model search unreliable; run: flair reembed against one model`,
+                    });
+                }
+            }
         }
         catch {
             stats.migrations = null;
+            if (memoryScanOk) {
+                const outstanding = describeStampOutstanding({
+                    modelCounts: memoryModelCounts,
+                    currentModelId: getModelId(),
+                });
+                if (outstanding.outstanding) {
+                    warnings.push({ level: "warn", message: outstanding.warning });
+                }
+                else if (mixedEmbeddingSpaces) {
+                    warnings.push({
+                        level: "warn",
+                        message: `multiple embedding models in use — cross-model search unreliable; run: flair reembed against one model`,
+                    });
+                }
+            }
         }
         // ── Disk ──
         // flair#812: same shared read-only resolution as the migrations section