@tpsdev-ai/flair 0.53.0 → 0.54.1

package/dist/federation-verify.js

@@ -0,0 +1,498 @@
+/**
+ * federation-verify.ts — `flair federation verify` (flair#823)
+ *
+ * End-to-end canary: write a tagged memory locally, push it (bring-up has
+ * no sync daemon yet), then probe each peer. Same class as fleet-verify /
+ * flair#988: separate "couldn't check" from "verified wrong."
+ *
+ *   UNVERIFIABLE (warning, exit 0): HTTP 401/403, unreachable, revoked,
+ *     no endpoint, or we could not inject the canary while lastSyncAt is
+ *     still fresh.
+ *   FAIL (exit 1): a reachable, authenticated peer is missing the canary
+ *     after a successful push — or lastSyncAt is stale when we could not
+ *     inject and the peer answered 200 without the tag.
+ *   OK (exit 0): the canary was found.
+ *
+ * Revoked peers are UNVERIFIABLE (listed, not probed, not FAIL) — Cos
+ * mid-flight: same couldn't-check bucket as 401 / unreachable. Authenticating
+ * the probe may later use #822's publicKey; this module does not wait on it.
+ * Do NOT always exit 0 — a verified-wrong reachable peer still fails.
+ */
+import { validatePeerEndpoint } from "./fleet-verify.js";
+import { DEFAULT_INTERVAL_SECONDS, freshnessWindowMs, } from "./federation/scheduler.js";
+export const FED_VERIFY_EXIT_OK = 0;
+export const FED_VERIFY_EXIT_DIVERGED = 1;
+export function defaultFreshnessMs() {
+    return freshnessWindowMs(DEFAULT_INTERVAL_SECONDS);
+}
+export function lastSyncFreshness(lastSyncAt, nowMs, windowMs) {
+    if (!lastSyncAt)
+        return { fresh: false, ageMs: null };
+    const t = Date.parse(lastSyncAt);
+    if (!Number.isFinite(t))
+        return { fresh: false, ageMs: null };
+    const ageMs = nowMs - t;
+    return { fresh: ageMs <= windowMs, ageMs };
+}
+/**
+ * After the wait window, a peer answered 200 but never showed the canary.
+ * A successful push means we verified the write is absent → FAIL.
+ * A failed/skipped push plus a fresh lastSyncAt is "couldn't inject,"
+ * not "verified diverged." A stale reachable peer still FAILs.
+ */
+export function classifyMissingAfterWindow(input) {
+    const { fresh, ageMs } = lastSyncFreshness(input.lastSyncAt, input.nowMs, input.freshnessMs);
+    if (input.pushed) {
+        return {
+            status: "fail",
+            detail: `timeout — reachable peer did not have the canary after push within ${input.waitSeconds}s`,
+        };
+    }
+    if (fresh) {
+        const age = ageMs == null ? "unknown" : `${Math.floor(ageMs / 1000)}s ago`;
+        return {
+            status: "unverifiable",
+            detail: `could not push canary; lastSyncAt is fresh (${age}) — could not confirm this write propagated`,
+        };
+    }
+    const age = ageMs == null ? "never" : `${Math.floor(ageMs / 1000)}s ago`;
+    return {
+        status: "fail",
+        detail: `timeout — reachable peer missing canary; lastSyncAt stale/absent (${age})`,
+    };
+}
+/** Immediate classification of a single HTTP status from the read-back probe. */
+export function classifyProbeHttpStatus(status) {
+    if (status === 401 || status === 403)
+        return "auth";
+    if (status >= 200 && status < 300)
+        return "found-ok";
+    return "other";
+}
+export function authUnverifiableDetail(status) {
+    return `HTTP ${status} — could not authenticate to peer (couldn't check, not a sync failure)`;
+}
+/**
+ * Revoked rows are decommissioned. Do not HTTP-probe them, and do not FAIL
+ * them — Cos / flair#823: revoked = UNVERIFIABLE (warn), same bucket as
+ * 401 / unreachable.
+ */
+export function selectPeersToProbe(peers, onlyId) {
+    const skippedRevoked = peers.filter((p) => p.status === "revoked" && (!onlyId || p.id === onlyId));
+    let probe = peers.filter((p) => p.status !== "revoked");
+    if (onlyId)
+        probe = probe.filter((p) => p.id === onlyId);
+    return { probe, skippedRevoked };
+}
+export function revokedAsUnverifiable(p) {
+    return {
+        id: p.id,
+        status: "unverifiable",
+        detail: "revoked — could not check (intentional, not a sync failure)",
+        lastSyncAt: p.lastSyncAt ?? null,
+        authenticated: false,
+    };
+}
+function formatUnverifiableWarning(unverifiableCount, checkedOk) {
+    return (`${unverifiableCount} peer(s) unverifiable — could not check ` +
+        `(auth refused, unreachable, revoked, or no endpoint); ` +
+        `${checkedOk} checked peer(s) have the memory.`);
+}
+/**
+ * Separate "couldn't check" from "verified wrong" (flair#823 / #988).
+ * Unverifiable peers never fail the run; a reachable missing-canary still does.
+ */
+export function describeFederationVerify(results) {
+    const failed = results.filter((r) => r.status === "fail");
+    const unverifiable = results.filter((r) => r.status === "unverifiable");
+    const ok = results.filter((r) => r.status === "ok");
+    const warning = unverifiable.length > 0
+        ? formatUnverifiableWarning(unverifiable.length, ok.length)
+        : null;
+    if (results.length === 0) {
+        return {
+            kind: "empty",
+            exitCode: FED_VERIFY_EXIT_OK,
+            failedCount: 0,
+            unverifiableCount: 0,
+            okCount: 0,
+            summary: "no peers to probe.",
+            warning: null,
+        };
+    }
+    if (failed.length > 0) {
+        return {
+            kind: "diverged",
+            exitCode: FED_VERIFY_EXIT_DIVERGED,
+            failedCount: failed.length,
+            unverifiableCount: unverifiable.length,
+            okCount: ok.length,
+            summary: `FAIL: ${failed.length}/${results.length} peer(s) did not see the memory.`,
+            warning,
+        };
+    }
+    if (ok.length === 0 && unverifiable.length > 0) {
+        return {
+            kind: "unverifiable-only",
+            exitCode: FED_VERIFY_EXIT_OK,
+            failedCount: 0,
+            unverifiableCount: unverifiable.length,
+            okCount: 0,
+            summary: "no reachable peer was verified wrong.",
+            warning,
+        };
+    }
+    return {
+        kind: "ok",
+        exitCode: FED_VERIFY_EXIT_OK,
+        failedCount: 0,
+        unverifiableCount: unverifiable.length,
+        okCount: ok.length,
+        summary: `PASS: memory propagated to all ${ok.length} checked peer(s).`,
+        warning,
+    };
+}
+export function decideFederationVerifyExitCode(results) {
+    return describeFederationVerify(results).exitCode;
+}
+export function renderFederationVerifyVerdict(verdict) {
+    const lines = [];
+    if (verdict.warning) {
+        lines.push(`── WARNING: ${verdict.warning} ──`);
+    }
+    if (verdict.kind === "diverged") {
+        lines.push(`── ${verdict.summary} ──`);
+        lines.push("Diagnostics to run next:");
+        lines.push("  flair federation status     # confirm peers are paired + lastSyncAt is recent");
+        lines.push("  flair federation reachability  # confirm peers are HTTP-reachable");
+        lines.push("  flair federation sync       # push now (verify also does this itself)");
+        lines.push("  curl <peer-endpoint>/Health  # raw probe");
+    }
+    else if (verdict.kind !== "empty") {
+        lines.push(`── ${verdict.summary} ──`);
+    }
+    return lines.join("\n");
+}
+function defaultClock() {
+    return {
+        now: () => Date.now(),
+        sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+    };
+}
+function canaryContainsTag(data, tag) {
+    const results = data?.results ?? [];
+    return results.some((r) => (r.content ?? "").includes(tag));
+}
+/** Credentials must travel even when there is no --target (local flag-only). */
+export function verifyApiOpts(opts) {
+    const apiOpts = {};
+    if (opts.baseUrl)
+        apiOpts.baseUrl = opts.baseUrl;
+    if (opts.explicitAdminPass)
+        apiOpts.explicitAdminPass = opts.explicitAdminPass;
+    if (opts.adminUser)
+        apiOpts.adminUser = opts.adminUser;
+    return (apiOpts.baseUrl || apiOpts.explicitAdminPass || apiOpts.adminUser) ? apiOpts : undefined;
+}
+export async function runFederationVerify(opts, deps) {
+    const log = deps.log ?? (() => { });
+    const err = deps.error ?? (() => { });
+    const clock = deps.clock ?? defaultClock();
+    const freshnessMs = opts.freshnessMs ?? defaultFreshnessMs();
+    const probeTimeoutMs = opts.probeTimeoutMs ?? 5_000;
+    const pollIntervalMs = opts.pollIntervalMs ?? 5_000;
+    const apiOpts = verifyApiOpts(opts);
+    log(`── flair federation verify — ${new Date(clock.now()).toISOString()} ──`);
+    log(`Tag: ${opts.tag}, wait window: ${opts.waitSeconds}s`);
+    const memId = `${opts.agentId}-${clock.now()}-fed-verify`;
+    const writtenAt = new Date(clock.now()).toISOString();
+    try {
+        await deps.api("PUT", `/Memory/${encodeURIComponent(memId)}`, {
+            id: memId,
+            agentId: opts.agentId,
+            content: `${opts.tag} — federation verify probe written at ${writtenAt}`,
+            // #1257: ephemeral is private-only. Private never federates. standard
+            // + shared is the legal federable pair. type:session lets rem light
+            // archive leftovers after 30d if the post-probe archive push fails.
+            // Federation does not propagate DELETE, so cleanup archives + syncs
+            // before the local delete (SemanticSearch excludes archived:true).
+            type: "session",
+            durability: "standard",
+            visibility: "shared",
+            tags: ["federation-verify", opts.tag],
+            createdAt: writtenAt,
+        }, apiOpts);
+        log(`1. Wrote local memory: ${memId}`);
+    }
+    catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        err(`1. Local write FAILED: ${message}`);
+        return {
+            exitCode: FED_VERIFY_EXIT_DIVERGED,
+            verdict: {
+                kind: "diverged",
+                exitCode: FED_VERIFY_EXIT_DIVERGED,
+                failedCount: 0,
+                unverifiableCount: 0,
+                okCount: 0,
+                summary: `local write failed: ${message}`,
+                warning: null,
+            },
+            peers: [],
+            skippedRevoked: [],
+            memId: null,
+            cleanedUp: true,
+            pushed: false,
+            tag: opts.tag,
+        };
+    }
+    let pushed = false;
+    let cleanedUp = false;
+    let result = {
+        exitCode: FED_VERIFY_EXIT_OK,
+        verdict: describeFederationVerify([]),
+        peers: [],
+        skippedRevoked: [],
+        memId,
+        cleanedUp: false,
+        pushed: false,
+        tag: opts.tag,
+    };
+    try {
+        log("   Pushing canary via federation sync…");
+        const syncResult = await deps.syncOnce(opts.syncOpts);
+        if (syncResult.error) {
+            log(`   Push: FAILED (${syncResult.error.message}) — probing with lastSyncAt freshness`);
+        }
+        else if (syncResult.pushed > 0) {
+            // A no-change sync ({ pushed: 0 }) did not inject the canary — do not
+            // treat that as a successful push or a missing-canary probe becomes
+            // a false FAIL.
+            pushed = true;
+            log(`   Push: synced ${syncResult.pushed} record(s) (${syncResult.skipped} skipped)`);
+        }
+        else {
+            log(`   Push: synced 0 records — canary was not injected; probing with lastSyncAt freshness`);
+        }
+        let listed = [];
+        try {
+            const r = await deps.api("GET", "/FederationPeers", undefined, apiOpts);
+            listed = r.peers ?? [];
+        }
+        catch (e) {
+            const message = e instanceof Error ? e.message : String(e);
+            err(`Failed to list peers: ${message}`);
+            const peers = [{
+                    id: "(peer enumeration)",
+                    status: "unverifiable",
+                    detail: `GET /FederationPeers failed: ${message} — could not check any peer`,
+                    lastSyncAt: null,
+                    authenticated: false,
+                }];
+            const verdict = describeFederationVerify(peers);
+            log(renderFederationVerifyVerdict(verdict));
+            result = {
+                exitCode: verdict.exitCode,
+                verdict,
+                peers,
+                skippedRevoked: [],
+                memId,
+                cleanedUp: false,
+                pushed,
+                tag: opts.tag,
+            };
+            return result;
+        }
+        const selected = selectPeersToProbe(listed, opts.peerId);
+        const skippedRevoked = selected.skippedRevoked;
+        const toProbe = selected.probe;
+        const revokedRows = skippedRevoked.map(revokedAsUnverifiable);
+        for (const row of revokedRows) {
+            log(`   ${row.id}  UNVERIFIABLE (${row.detail})`);
+        }
+        if (toProbe.length === 0) {
+            if (revokedRows.length === 0)
+                log("(no peers to probe)");
+            const verdict = describeFederationVerify(revokedRows);
+            log(renderFederationVerifyVerdict(verdict));
+            result = {
+                exitCode: verdict.exitCode,
+                verdict,
+                peers: revokedRows,
+                skippedRevoked,
+                memId,
+                cleanedUp: false,
+                pushed,
+                tag: opts.tag,
+            };
+            return result;
+        }
+        log(`2. Probing ${toProbe.length} peer(s) over ${opts.waitSeconds}s window…`);
+        const started = clock.now();
+        const settled = new Map();
+        const sawAuthenticatedMissing = new Set();
+        const settle = (row) => {
+            settled.set(row.id, row);
+            const label = row.status === "ok" ? "OK" : row.status === "fail" ? "FAIL" : "UNVERIFIABLE";
+            log(`   ${row.id}  ${label} (${row.detail})`);
+        };
+        while (clock.now() - started < opts.waitMs && settled.size < toProbe.length) {
+            for (const p of toProbe) {
+                if (settled.has(p.id))
+                    continue;
+                const endpointCheck = validatePeerEndpoint(p.endpoint);
+                if ("error" in endpointCheck) {
+                    settle({
+                        id: p.id,
+                        status: "unverifiable",
+                        detail: endpointCheck.error,
+                        lastSyncAt: p.lastSyncAt ?? null,
+                        authenticated: false,
+                    });
+                    continue;
+                }
+                let probeUrl;
+                try {
+                    probeUrl = new URL("/SemanticSearch", endpointCheck.url);
+                }
+                catch {
+                    settle({
+                        id: p.id,
+                        status: "unverifiable",
+                        detail: `invalid endpoint URL: ${p.endpoint}`,
+                        lastSyncAt: p.lastSyncAt ?? null,
+                        authenticated: false,
+                    });
+                    continue;
+                }
+                try {
+                    const res = await deps.fetch(probeUrl, {
+                        method: "POST",
+                        headers: { "Content-Type": "application/json" },
+                        body: JSON.stringify({ q: opts.tag, limit: 5 }),
+                        signal: AbortSignal.timeout(probeTimeoutMs),
+                    });
+                    const kind = classifyProbeHttpStatus(res.status);
+                    if (kind === "auth") {
+                        settle({
+                            id: p.id,
+                            status: "unverifiable",
+                            detail: authUnverifiableDetail(res.status),
+                            lastSyncAt: p.lastSyncAt ?? null,
+                            authenticated: false,
+                        });
+                        continue;
+                    }
+                    if (kind !== "found-ok") {
+                        // Retry 4xx/5xx other than 401/403 — couldn't complete this tick.
+                        continue;
+                    }
+                    const data = await res.json().catch(() => ({}));
+                    if (canaryContainsTag(data, opts.tag)) {
+                        const elapsed = Math.floor((clock.now() - started) / 1000);
+                        settle({
+                            id: p.id,
+                            status: "ok",
+                            detail: `memory found after ${elapsed}s`,
+                            lastSyncAt: p.lastSyncAt ?? null,
+                            authenticated: true,
+                        });
+                    }
+                    else {
+                        sawAuthenticatedMissing.add(p.id);
+                    }
+                }
+                catch {
+                    // Unreachable this tick — retry until the window elapses.
+                }
+            }
+            if (settled.size < toProbe.length) {
+                await clock.sleep(pollIntervalMs);
+            }
+        }
+        for (const p of toProbe) {
+            if (settled.has(p.id))
+                continue;
+            if (sawAuthenticatedMissing.has(p.id)) {
+                const classified = classifyMissingAfterWindow({
+                    pushed,
+                    lastSyncAt: p.lastSyncAt,
+                    nowMs: clock.now(),
+                    freshnessMs,
+                    waitSeconds: opts.waitSeconds,
+                });
+                settle({
+                    id: p.id,
+                    status: classified.status,
+                    detail: classified.detail,
+                    lastSyncAt: p.lastSyncAt ?? null,
+                    authenticated: true,
+                });
+            }
+            else {
+                settle({
+                    id: p.id,
+                    status: "unverifiable",
+                    detail: `unreachable — no successful probe within ${opts.waitSeconds}s (couldn't check, not a sync failure)`,
+                    lastSyncAt: p.lastSyncAt ?? null,
+                    authenticated: false,
+                });
+            }
+        }
+        const peers = [...revokedRows, ...toProbe.map((p) => settled.get(p.id))];
+        const verdict = describeFederationVerify(peers);
+        log(renderFederationVerifyVerdict(verdict));
+        result = {
+            exitCode: verdict.exitCode,
+            verdict,
+            peers,
+            skippedRevoked,
+            memId,
+            cleanedUp: false,
+            pushed,
+            tag: opts.tag,
+        };
+        return result;
+    }
+    finally {
+        // Federation merge is LWW put-by-id — local DELETE never reaches the
+        // hub. If we injected the canary, archive it and push that update so
+        // peer SemanticSearch (archived not_equal true) stops returning it.
+        if (pushed) {
+            try {
+                await deps.api("PUT", `/Memory/${encodeURIComponent(memId)}`, {
+                    id: memId,
+                    agentId: opts.agentId,
+                    content: `${opts.tag} — federation verify probe written at ${writtenAt}`,
+                    type: "session",
+                    durability: "standard",
+                    visibility: "shared",
+                    tags: ["federation-verify", opts.tag],
+                    createdAt: writtenAt,
+                    archived: true,
+                    archivedAt: new Date(clock.now()).toISOString(),
+                }, apiOpts);
+                const archiveSync = await deps.syncOnce(opts.syncOpts);
+                if (archiveSync.error) {
+                    log(`4. Cleanup: archive push FAILED (${archiveSync.error.message}) — canary may linger on peers`);
+                }
+                else {
+                    log(`4. Cleanup: pushed archive of ${memId} (peers drop it from search)`);
+                }
+            }
+            catch (e) {
+                const message = e instanceof Error ? e.message : String(e);
+                log(`4. Cleanup: could not archive canary for peer cleanup (${message})`);
+            }
+        }
+        try {
+            await deps.api("DELETE", `/Memory/${encodeURIComponent(memId)}`, undefined, apiOpts);
+            cleanedUp = true;
+            log(`4. Cleanup: deleted local memory ${memId}`);
+        }
+        catch {
+            log(`4. Cleanup: could NOT delete local memory ${memId} (manual cleanup needed)`);
+        }
+        result.cleanedUp = cleanedUp;
+    }
+}