@tpsdev-ai/flair 0.39.0 → 0.41.0

package/README.md

@@ -404,7 +404,7 @@ Shipped: Ed25519 identity and auth · memory CRUD with durability enforcement an
 
 Next: git-backed memory sync · opt-in encryption at rest (AES-256-GCM per memory).
 
-> **Note:** Flair runs on [Harper v5](https://harper.fast), currently in beta. We run it in production daily and track upstream closely. Pin your Harper version.
+> **Note:** Flair runs on [Harper v5](https://harper.fast). We run it in production daily and track upstream closely. Pin your Harper version.
 
 ## License
 

package/dist/cli.js

@@ -774,6 +774,14 @@ export function buildDirectSpawnEnv(opts) {
         HTTP_PORT: String(opts.httpPort),
         OPERATIONSAPI_NETWORK_PORT: opsNetworkPortValue(opts.opsBindHost, opts.opsPort),
         LOCAL_STUDIO: "false",
+        // flair#905 / lrf5: Harper's forceDowngradePrompt reads CONFIRM_DOWNGRADE
+        // from the environment (via the `prompt` npm package's assignCmdEnvVariables
+        // override). Under launchd/systemd stdin is not a TTY, so the prompt gets
+        // EOF and Harper exits 0 without starting — leaving the instance DOWN with
+        // no error. Setting this to "yes" makes the prompt non-interactive: Harper
+        // proceeds without blocking, which is the correct default for a managed
+        // restart where the operator already chose to proceed.
+        CONFIRM_DOWNGRADE: "yes",
     };
     if (opts.adminPass)
         env.HDB_ADMIN_PASSWORD = opts.adminPass;
@@ -9688,6 +9696,50 @@ program
             console.error(`   Recover by hand: npm install -g @tpsdev-ai/flair@${toVersion} && flair start`);
             process.exit(1);
         }
+        // flair#1053: when the engine (Harper) version changed, the pre-upgrade
+        // snapshot is the ONLY way back — the old Harper cannot read data written
+        // by the new one (e.g. 5.2 LZ4-compressed storage is unreadable by 5.1).
+        // Restore it before restarting, or refuse loudly when none exists.
+        if (engineVersionChanging) {
+            if (snapshotPath) {
+                console.log(`\nEngine version changed — restoring pre-upgrade snapshot before rollback...`);
+                console.log(`  snapshot: ${snapshotPath}`);
+                console.log(`  target:   ${upgradeDataDir}`);
+                try {
+                    await validateSnapshotArchive({ file: snapshotPath, targetDir: upgradeDataDir });
+                    rmSync(upgradeDataDir, { recursive: true, force: true });
+                    mkdirSync(upgradeDataDir, { recursive: true, mode: 0o700 });
+                    await extractSnapshotSafely({ file: snapshotPath, targetDir: upgradeDataDir });
+                    console.log(`  ✅ snapshot restored`);
+                }
+                catch (err) {
+                    console.error(`❌ snapshot restore failed: ${err.message}`);
+                    console.error(`   @tpsdev-ai/flair@${toVersion} is installed but the data directory could not be restored.`);
+                    console.error(`   The snapshot itself is intact at ${snapshotPath} — restore it by hand:`);
+                    console.error(`   flair snapshot restore "${snapshotPath}"`);
+                    console.error(`   Then: flair start`);
+                    process.exit(1);
+                }
+            }
+            else {
+                // No snapshot exists — the old Harper WILL NOT BOOT against the new
+                // data. Refuse loudly rather than attempting a guaranteed failure.
+                console.error(`\n❌ Cannot roll back: the Harper engine version changed (${currentEngineVersion ?? "?"} → ${targetEngineVersion ?? "?"}) and no pre-upgrade snapshot exists.`);
+                console.error(`   The old Harper cannot read data written by the new engine — restarting without a snapshot restore would fail.`);
+                console.error(`   @tpsdev-ai/flair@${toVersion} is installed but NOT running.`);
+                if (snapshotDecision === "nudge") {
+                    console.error(`   A snapshot was skipped because --no-engine-snapshot was passed.`);
+                    console.error(`   Recovery options:`);
+                    console.error(`   1. Re-upgrade to the version that wrote this data: npm install -g @tpsdev-ai/flair@${expectedFlairVersion ?? "latest"} && flair start`);
+                    console.error(`   2. Restore from a ` + "`flair backup`" + ` JSON export on a fresh data directory.`);
+                }
+                else {
+                    console.error(`   No snapshot was taken (data directory may not have existed, or the snapshot step was skipped).`);
+                    console.error(`   Recovery: re-upgrade to the version that wrote this data, or restore from a ` + "`flair backup`" + ` JSON export.`);
+                }
+                process.exit(1);
+            }
+        }
         // Same post-swap rule as the upgrade restart above: the rolled-back
         // version's own CLI is the thing that knows how to start it.
         const rolledBackCli = resolveInstalledFlairCli(flairPackageDir(), toVersion);
@@ -10321,8 +10373,18 @@ async function stopFlairProcess(port, dataDir) {
         }
         catch { }
     }
-    // Wait briefly for shutdown
-    await new Promise((r) => setTimeout(r, 2000));
+    // flair#905 / lrf5: wait for every signalled process to actually exit.
+    // A blind 2-second sleep is not a guarantee — Harper may be flushing
+    // RocksDB WAL/MANIFEST, and the next start will fail with a locked data
+    // directory if the old process hasn't released it yet. The launchd path
+    // above already does this via waitForProcessExit; the port-based path
+    // must match that guarantee.
+    for (const target of targets) {
+        try {
+            await waitForProcessExit(target, STARTUP_TIMEOUT_MS);
+        }
+        catch { /* best-effort — the next start will surface the real problem */ }
+    }
 }
 /**
  * Start the local Flair (Harper) process — launchd `start` on darwin when a

package/dist/lib/mcp-enable.js

@@ -748,6 +748,83 @@ export function buildClaudePasteBlock(resource) {
         "  (no client ID to enter — Claude presents its own Client ID Metadata Document URL automatically)",
     ].join("\n");
 }
+/** How long to wait for the ops API / PID change after a restart. */
+const RESTART_WAIT_TIMEOUT_MS = 30_000;
+/** Poll interval while waiting for the ops API after restart. */
+const RESTART_WAIT_POLL_MS = 1000;
+/**
+ * Wait for the ops API to respond after a restart, then poll until the PID
+ * changes (proving a genuine restart, not the old process still answering).
+ *
+ * After a real restart, the old process can briefly still respond to the first
+ * ops request. We capture the PID on every poll until it changes — if the
+ * window expires with the PID still the same we report thread-bounce failure
+ * so the operator knows the restart was a no-op rather than a timing quirk.
+ *
+ * `timeoutMs` and `pollMs` are injectable via `deps` so tests can run fast.
+ */
+async function waitForOpsApi(opsUrl, authHeader, prePid, deps = {}) {
+    const fetchImpl = deps.fetchImpl ?? fetch;
+    const timeoutMs = deps.timeoutMs ?? RESTART_WAIT_TIMEOUT_MS;
+    const pollMs = deps.pollMs ?? RESTART_WAIT_POLL_MS;
+    const deadline = Date.now() + timeoutMs;
+    let attempt = 0;
+    while (Date.now() < deadline) {
+        attempt++;
+        try {
+            const res = await fetchImpl(opsUrl, {
+                method: "POST",
+                headers: { "Content-Type": "application/json", Authorization: authHeader },
+                body: JSON.stringify({ operation: "system_information", attributes: ["harperdb_processes"] }),
+                signal: AbortSignal.timeout(3000),
+            });
+            if (!res.ok) {
+                await new Promise((r) => setTimeout(r, pollMs));
+                continue;
+            }
+            // Ops API answered — extract the PID from this response
+            const data = await res.json();
+            const pid = data?.harperdb_processes?.core?.[0]?.pid;
+            if (!pid || typeof pid !== "number") {
+                await new Promise((r) => setTimeout(r, pollMs));
+                continue;
+            }
+            // If PID changed from pre-restart value, the restart is confirmed
+            if (pid !== prePid)
+                return { pid };
+            // PID still the same (old process may still be answering), keep polling
+        }
+        catch { /* not ready yet — connection refused, timeout, etc. */ }
+        await new Promise((r) => setTimeout(r, pollMs));
+    }
+    throw new Error(`ops API at ${opsUrl} did not confirm a new process within ${timeoutMs}ms (${attempt} attempts) ` +
+        `— the restart may have failed or the process bounced on the same thread (pid ${prePid} unchanged). ` +
+        `Restart the instance manually, then re-run: flair mcp enable`);
+}
+/**
+ * Capture the Harper core process PID from the ops API via `system_information`.
+ * This PID is the boot discriminator: it changes on every real restart.
+ */
+export async function captureBootDiscriminator(opsPortOrUrl, adminUser, adminPass, deps = {}) {
+    const fetchImpl = deps.fetchImpl ?? fetch;
+    const opsUrl = resolveOpsUrl(opsPortOrUrl);
+    const authHeader = basicAuthHeader(adminUser, adminPass);
+    const res = await fetchImpl(opsUrl, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", Authorization: authHeader },
+        body: JSON.stringify({ operation: "system_information", attributes: ["harperdb_processes"] }),
+    });
+    if (!res.ok) {
+        const text = await res.text().catch(() => "");
+        throw new Error(`system_information failed (HTTP ${res.status}): ${text}`);
+    }
+    const data = await res.json();
+    const pid = data?.harperdb_processes?.core?.[0]?.pid;
+    if (!pid || typeof pid !== "number") {
+        throw new Error("system_information returned no harperdb_processes.core entry with a PID");
+    }
+    return { pid };
+}
 /**
  * Full `flair mcp enable` orchestration. No `process.exit`, no console
  * output — directly unit-testable with a mocked fetch and temp dirs, same
@@ -898,10 +975,23 @@ export async function enableMcp(params, deps = {}) {
             push(false, `not applied: pass --confirm-secrets-applied once the staged secrets are live on ${params.instance}, then re-run \`flair mcp enable\` (earlier steps are idempotent and will reuse what's already provisioned).`);
             return { ok: false, dryRun, steps, failedStep: "apply-config-and-restart", secretsMechanism: secretsResult.mechanism, secretsPath: secretsResult.path };
         }
-        // ── Apply config + restart ────────────────────────────────────────────────
         currentStep = "apply-config-and-restart";
+        // ── Capture boot discriminator BEFORE restart (flair#1120) ─────────────
+        const preDiscriminator = await captureBootDiscriminator(params.instance, params.adminUser, params.adminPass, { fetchImpl: deps.fetchImpl });
         await applyRemoteConfigAndRestart({ opsPortOrUrl: params.instance, adminUser: params.adminUser, adminPass: params.adminPass, configBlock }, { fetchImpl: deps.fetchImpl });
         push(true, `set_configuration + restart succeeded against ${params.instance}`);
+        // ── Verify the process actually restarted (flair#1120) ──────────────────
+        // Poll the ops API until the PID changes — the old process can briefly
+        // still answer after a real restart, so a single post-capture is unreliable.
+        // waitForOpsApi guarantees PID change (or throws on timeout), so the restart
+        // is confirmed when this call returns.
+        currentStep = "verify-restart";
+        const postDiscriminator = await waitForOpsApi(resolveOpsUrl(params.instance), basicAuthHeader(params.adminUser, params.adminPass), preDiscriminator.pid, {
+            fetchImpl: deps.fetchImpl,
+            timeoutMs: deps.waitForOpsApiTimeoutMs,
+            pollMs: deps.waitForOpsApiPollMs,
+        });
+        push(true, `process restarted: pid changed ${preDiscriminator.pid} -> ${postDiscriminator.pid}`);
         // ── Self-verify from the operator's machine, public origin, CIMD-inclusive
         currentStep = "self-verify";
         const verify = await selfVerifyMcpMetadata(issuer, { fetchImpl: deps.fetchImpl });

package/dist/resources/SemanticSearch.js

@@ -1,7 +1,7 @@
 import { Resource, databases } from "harper";
 import { resolveAgentAuth, allowVerified } from "./agent-auth.js";
 import { getEmbedding, getMode } from "./embeddings-provider.js";
-import { patchRecord } from "./table-helpers.js";
+import { patchRecord, withDetachedTxn } from "./table-helpers.js";
 import { checkRateLimit, rateLimitResponse } from "./rate-limiter.js";
 import { resolveReadScope } from "./memory-read-scope.js";
 // The BM25 + union-RRF hybrid path is feature-flagged via hybridEnabled()
@@ -61,7 +61,7 @@ export class SemanticSearch extends Resource {
         // recall-harness (test/bench/recall-harness/run.ts) and `recall-eval.mjs`
         // before reconsidering this default if the compositeScore formula or
         // corpus changes.
-        const { agentId: bodyAgentId, q, queryEmbedding, tag, subject, subjects, limit = 10, includeSuperseded = false, scoring = "raw", minScore = 0, since, asOf, includeTrust = false, abstain = false } = data || {};
+        const { agentId: bodyAgentId, q, queryEmbedding, tag, subject, subjects, limit = 10, includeSuperseded = false, scoring = "raw", minScore = 0, since, asOf, includeTrust = false, abstain = false, explain = false } = data || {};
         // Authenticated identity lives on the Harper Resource context (getContext().request).
         // `this.request` is NOT populated on Harper v5 Resources — prior reads here
         // silently returned undefined and the defense-in-depth scope check below
@@ -179,6 +179,33 @@ export class SemanticSearch extends Resource {
                 });
             }
         }
+        // ─── Explain mode: return Harper's ENGINE-LEVEL query plan ────────────
+        // When explain=true, construct the same search query that the HNSW leg
+        // would use and pass explain:true through to Harper's Table.search().
+        // Harper's cost-based planner re-sorts conditions by estimated count at
+        // execution (the scope OR-group estimates Infinity and can never drive;
+        // a selective tags-equals wins the seek). The returned plan shows the
+        // ENGINE's chosen order — the proof the spec requires.
+        //
+        // No search is executed; no side effects (rate-limit, hit-tracking).
+        if (explain) {
+            const ctx = this.getContext?.();
+            const explainQuery = {
+                sort: qEmb ? { attribute: "embedding", target: qEmb, distance: "cosine" } : undefined,
+                select: DEFAULT_SELECT,
+                limit,
+                explain: true,
+            };
+            if (conditions.length > 0)
+                explainQuery.conditions = conditions;
+            const plan = withDetachedTxn(ctx, () => databases.flair.Memory.search(explainQuery));
+            return {
+                explain: true,
+                plan,
+                tag,
+                scoring,
+            };
+        }
         const hybrid = hybridEnabled();
         // The overfetch policy (how many raw candidates to pull from the
         // HNSW/BM25 legs relative to what the caller ultimately wants) is THIS

package/docs/upgrade.md

@@ -350,6 +350,14 @@ if you want to see it scripted end-to-end.
   between Flair versions.
 - **Config:** `~/.flair/config.yaml` format is additive — new options fall back to
   defaults when absent, old options aren't removed out from under you.
+- **Harper 5.2:** Starting with the release that pins Harper 5.2.0 (see CHANGELOG). This upgrade is **forward-only** — Harper 5.2 writes LZ4-compressed storage
+  that Harper 5.1.x cannot read. If you need to roll back, restore the pre-upgrade snapshot: `flair snapshot restore <path>` (use `flair snapshot list` to find available snapshots). Note this is the physical engine snapshot restore, distinct from `flair restore` which replays a logical JSON export and cannot recover a 5.1-incompatible data directory. Installing an older Harper version will not boot against a
+  5.2-written data directory. Flair automatically takes a snapshot when the Harper engine
+  minor version changes, so a valid snapshot will exist if you
+  upgraded via `flair upgrade`. Real snapshots for production datasets run hundreds of
+  megabytes — plan disk capacity accordingly. Cleanup of old engine snapshots is manual;
+  `flair snapshot list` shows available snapshots, and you can delete entries you no longer
+  need (e.g. `rm ~/.flair/upgrade-snapshots/flair-data-<old>.tar.gz`).
 
 ## Rollback
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tpsdev-ai/flair",
-  "version": "0.39.0",
+  "version": "0.41.0",
   "packageManager": "bun@1.3.10",
   "description": "Identity, memory, and soul for AI agents. Cryptographic identity (Ed25519), semantic memory with local embeddings, and persistent personality — all in a single process.",
   "type": "module",
@@ -64,11 +64,11 @@
     "@harperfast/oauth": "2.4.0",
     "@types/js-yaml": "4.0.9",
     "commander": "14.0.3",
-    "harper": "5.1.22",
+    "harper": "5.2.0",
     "harper-fabric-embeddings": "^0.5.0",
     "jose": "6.2.2",
-    "js-yaml": "4.3.0",
-    "tar": "7.5.20",
+    "js-yaml": "^4.3.1",
+    "tar": "^7.5.22",
     "tweetnacl": "1.0.3"
   },
   "overrides": {
@@ -76,7 +76,13 @@
     "brace-expansion": "^5.0.9",
     "undici": "^8.9.0",
     "fast-uri": "^4.1.2",
-    "hono": "^4.12.34"
+    "hono": "^4.12.34",
+    "js-yaml": "^4.3.1",
+    "adm-zip": "^0.6.0",
+    "@opentelemetry/core": "^2.8.0",
+    "uuid": "^11.1.1",
+    "tar": "^7.5.22",
+    "@tootallnate/once": "^2.0.1"
   },
   "devDependencies": {
     "@playwright/test": "1.59.1",