@tpsdev-ai/flair 0.25.4 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/cli.js +998 -50
- package/dist/doctor-client.js +47 -0
- package/dist/rem/runner.js +60 -3
- package/dist/resources/MemoryDedupStats.js +161 -0
- package/dist/resources/dedup-cluster.js +145 -0
- package/dist/resources/health.js +33 -1
- package/dist/resources/mcp-oauth.js +4 -2
- package/docs/assets/flair-cross-orchestrator.cast +33 -0
- package/docs/assets/flair-cross-orchestrator.gif +0 -0
- package/docs/assets/flair-demo.cast +18 -0
- package/docs/assets/flair-demo.gif +0 -0
- package/docs/auth.md +178 -0
- package/docs/bridges.md +291 -0
- package/docs/claude-code.md +202 -0
- package/docs/deployment.md +204 -0
- package/docs/entity-vocabulary.md +109 -0
- package/docs/federation.md +179 -0
- package/docs/integrations.md +197 -0
- package/docs/mcp-clients.md +240 -0
- package/docs/n8n-management.md +142 -0
- package/docs/n8n.md +122 -0
- package/docs/notes/adk-spike-findings-2026-05-05.md +331 -0
- package/docs/notes/mcp-agent-auth-consumer.md +229 -0
- package/docs/notes/mcp-oauth-model2.md +132 -0
- package/docs/notes/rem-ux.md +39 -0
- package/docs/openclaw.md +131 -0
- package/docs/quickstart.md +153 -0
- package/docs/releasing.md +173 -0
- package/docs/rem.md +60 -0
- package/docs/rerank-provisioning.md +67 -0
- package/docs/secrets-and-keys.md +173 -0
- package/docs/spoke-bringup.md +303 -0
- package/docs/supply-chain-policy.md +156 -0
- package/docs/system-requirements.md +42 -0
- package/docs/the-team.md +129 -0
- package/docs/troubleshooting.md +187 -0
- package/docs/upgrade.md +416 -0
- package/package.json +2 -1
package/dist/cli.js
CHANGED
|
@@ -20,7 +20,7 @@ import { markStale, sortOldestVersionFirst } from "./fleet-presence.js";
|
|
|
20
20
|
import { detectClients, wireClaudeCode, wireCodex, wireGemini, wireCursor } from "./install/clients.js";
|
|
21
21
|
import { resolveAgentKeyPath, loadEd25519PrivateKeyFromFile, signClientAssertion, buildTokenRequestForm, getMcpAccessToken, McpTokenRequestError, defaultMcpClientId, defaultMcpTokenEndpoint, defaultMcpResource, defaultMcpIssuer, MAX_ASSERTION_LIFETIME_SECONDS, } from "./mcp-client-assertion.js";
|
|
22
22
|
import { enableMcp, disableMcp, mcpStatus, checkLocalOriginRefusal, selfVerifyMcpMetadata, } from "./lib/mcp-enable.js";
|
|
23
|
-
import { readClientMcpBlock, checkClaudeMdBootstrap, checkSessionStartHook, fixClaudeMdBootstrap, fixSessionStartHook, applyOrReportClaudeMdBootstrap, applyOrReportSessionStartHook, resolveWireFlairUrl, planAgentIterations, describeAgentGateFinding, classifyKeyFile, resolveCollisionSafeName, pruneDateStamp, PRUNED_DIR_NAME, } from "./doctor-client.js";
|
|
23
|
+
import { readClientMcpBlock, checkClaudeMdBootstrap, checkSessionStartHook, fixClaudeMdBootstrap, fixSessionStartHook, applyOrReportClaudeMdBootstrap, applyOrReportSessionStartHook, resolveWireFlairUrl, planAgentIterations, inferSoleAgentId, fixCommandAgentHint, describeAgentGateFinding, classifyKeyFile, resolveCollisionSafeName, pruneDateStamp, PRUNED_DIR_NAME, } from "./doctor-client.js";
|
|
24
24
|
import { installHook, uninstallHook, hookStatus, isSupportedHarness, SUPPORTED_HARNESSES, } from "./hook-install.js";
|
|
25
25
|
import { readSecretFileSecure, readAdminPassFileSecure, defaultKeysDir, resolveLocalAdminPass, resolveKeyPath, buildEd25519Auth, authFetch, isLocalBase, authedRequest, } from "./lib/auth-resolve.js";
|
|
26
26
|
// Federation crypto helpers — inlined to avoid cross-boundary imports from
|
|
@@ -2725,6 +2725,17 @@ program
|
|
|
2725
2725
|
console.log(` └─────────────────────────────────────────────────┘`);
|
|
2726
2726
|
}
|
|
2727
2727
|
console.log(`\n Export: FLAIR_URL=${httpUrl}`);
|
|
2728
|
+
// flair#802a: a non-interactive shell (CI, Docker, an unattended setup
|
|
2729
|
+
// script) that omits --agent lands here with NO indication that agent
|
|
2730
|
+
// registration, MCP client wiring, and the smoke test were all skipped
|
|
2731
|
+
// — the run exits 0 and looks complete. In a real TTY the missing
|
|
2732
|
+
// --agent is usually obvious from the command the user just typed;
|
|
2733
|
+
// non-interactively it's easy to never notice until something that
|
|
2734
|
+
// needed the agent (recall, an MCP client) mysteriously doesn't work.
|
|
2735
|
+
if (!process.stdin.isTTY) {
|
|
2736
|
+
console.log(`\n ℹ Non-interactive shell: skipped agent registration, MCP client wiring, and the smoke test.`);
|
|
2737
|
+
console.log(` Complete setup with: flair init --agent <id> --client all`);
|
|
2738
|
+
}
|
|
2728
2739
|
}
|
|
2729
2740
|
// All init work is genuinely done at this point: Harper is installed +
|
|
2730
2741
|
// running (detached, unref'd — survives this process exiting), the agent is
|
|
@@ -6432,6 +6443,12 @@ remNightly
|
|
|
6432
6443
|
if (row.candidates) {
|
|
6433
6444
|
console.log(`Staged: ${row.candidates.length} candidate${row.candidates.length === 1 ? "" : "s"}`);
|
|
6434
6445
|
}
|
|
6446
|
+
// row.dedup populates when step 6 (instance-wide dedup-cluster stat,
|
|
6447
|
+
// flair-quality Slice 1c) succeeded this cycle. Absent on dry-run skip
|
|
6448
|
+
// or a non-fatal failure (see Errors below — e.g. non-admin caller).
|
|
6449
|
+
if (row.dedup) {
|
|
6450
|
+
console.log(`Dedup: ${row.dedup.clusterCount} cluster${row.dedup.clusterCount === 1 ? "" : "s"} (${row.dedup.totalMemoriesInClusters} memories, largest ${row.dedup.largestClusterSize})`);
|
|
6451
|
+
}
|
|
6435
6452
|
console.log(`Duration: ${row.durationMs}ms`);
|
|
6436
6453
|
if (row.errors.length > 0) {
|
|
6437
6454
|
console.log(`Errors:`);
|
|
@@ -8650,11 +8667,22 @@ async function stopFlairProcess(port) {
|
|
|
8650
8667
|
console.log("Stopping...");
|
|
8651
8668
|
try {
|
|
8652
8669
|
const { execSync } = await import("node:child_process");
|
|
8653
|
-
|
|
8670
|
+
// -sTCP:LISTEN: match the LISTENING server only — a bare `lsof -ti :port`
|
|
8671
|
+
// also matches CLIENT sockets referencing the port, including THIS CLI's
|
|
8672
|
+
// own keep-alive connections left by the credential pre-flight's
|
|
8673
|
+
// probeInstance() HTTP calls (flair#741). Without the filter, the upgrade
|
|
8674
|
+
// path SIGTERM'd its own process mid-restart — "Stopping..." then death
|
|
8675
|
+
// (exit 143) before "Starting..." ever ran, leaving the server down
|
|
8676
|
+
// (flair#800, deterministic on the Linux/non-launchd default path).
|
|
8677
|
+
const lsof = execSync(`lsof -ti :${port} -sTCP:LISTEN`, { encoding: "utf-8" }).trim();
|
|
8654
8678
|
if (lsof) {
|
|
8655
8679
|
for (const pid of lsof.split("\n")) {
|
|
8680
|
+
const target = Number(pid.trim());
|
|
8681
|
+
// Belt-and-suspenders: never SIGTERM ourselves, whatever lsof says.
|
|
8682
|
+
if (!Number.isFinite(target) || target === process.pid)
|
|
8683
|
+
continue;
|
|
8656
8684
|
try {
|
|
8657
|
-
process.kill(
|
|
8685
|
+
process.kill(target, "SIGTERM");
|
|
8658
8686
|
}
|
|
8659
8687
|
catch { }
|
|
8660
8688
|
}
|
|
@@ -9815,9 +9843,18 @@ program
|
|
|
9815
9843
|
console.log(` Skipped.`);
|
|
9816
9844
|
}
|
|
9817
9845
|
else {
|
|
9818
|
-
|
|
9846
|
+
// flair#802b: fall back to the sole locally-keyed agent when
|
|
9847
|
+
// nothing else identifies one — the only case doctor can
|
|
9848
|
+
// infer without being told (see inferSoleAgentId's doc
|
|
9849
|
+
// comment in doctor-client.ts for why 0/2+ keys don't guess).
|
|
9850
|
+
const fixAgentId = opts.agent || process.env.FLAIR_AGENT_ID || anyKnownAgentId || inferSoleAgentId(keyAgentIds);
|
|
9819
9851
|
if (!fixAgentId) {
|
|
9820
|
-
|
|
9852
|
+
if (keyAgentIds.length > 1) {
|
|
9853
|
+
console.log(` ${render.icons.warn} Cannot auto-wire ${client.label}: multiple agents found (${[...keyAgentIds].sort().join(", ")}) — pass --agent <id> to choose which one`);
|
|
9854
|
+
}
|
|
9855
|
+
else {
|
|
9856
|
+
console.log(` ${render.icons.warn} Cannot auto-wire ${client.label}: no agent registered — run \`flair agent add <id>\` first, then re-run \`flair doctor --fix\``);
|
|
9857
|
+
}
|
|
9821
9858
|
}
|
|
9822
9859
|
else {
|
|
9823
9860
|
const wireEnv = { FLAIR_AGENT_ID: fixAgentId, FLAIR_URL: resolveWireFlairUrl(block.flairUrl, baseUrl) };
|
|
@@ -9833,7 +9870,13 @@ program
|
|
|
9833
9870
|
}
|
|
9834
9871
|
}
|
|
9835
9872
|
else {
|
|
9836
|
-
|
|
9873
|
+
// flair#802b: only splice in a concrete --agent if the id isn't
|
|
9874
|
+
// already resolvable some other way — an explicit --agent /
|
|
9875
|
+
// FLAIR_AGENT_ID / an already-wired client's agent id means bare
|
|
9876
|
+
// `--fix` already works, so don't clutter the suggestion.
|
|
9877
|
+
const knownAgentId = opts.agent || process.env.FLAIR_AGENT_ID || anyKnownAgentId;
|
|
9878
|
+
const agentHint = knownAgentId ? "" : fixCommandAgentHint(keyAgentIds);
|
|
9879
|
+
console.log(` ${render.wrap(render.c.dim, "Fix:")} flair doctor --fix${agentHint} ${render.wrap(render.c.dim, `(wires ${client.label} automatically)`)}`);
|
|
9837
9880
|
}
|
|
9838
9881
|
issues++;
|
|
9839
9882
|
continue;
|
|
@@ -10148,6 +10191,892 @@ program
|
|
|
10148
10191
|
if (summary.exitCode !== 0)
|
|
10149
10192
|
process.exit(summary.exitCode);
|
|
10150
10193
|
});
|
|
10194
|
+
// ─── flair quality — pure metric computation ─────────────────────────────────
|
|
10195
|
+
// Slice 1a of the memory-quality-observability arc (ops/proposals/
|
|
10196
|
+
// flair-quality-slice1-spec.md, K&S design-approved). Same "extract the pure
|
|
10197
|
+
// decision logic" idiom as summarizeDoctorRun above (flair#721) and
|
|
10198
|
+
// derivePresenceStatus (resources/Presence.ts): the CLI action spawns a
|
|
10199
|
+
// network fetch + a long console.log sequence, which is high-effort/
|
|
10200
|
+
// low-value to drive directly in a test — this is the actual metric math,
|
|
10201
|
+
// fed a fixture-shaped /HealthDetail body.
|
|
10202
|
+
//
|
|
10203
|
+
// ZERO new server surface. Every field this reads already exists in
|
|
10204
|
+
// resources/health.ts's response (the same one `flair status`/`flair doctor`
|
|
10205
|
+
// consume) — no new query pattern, no new endpoint. That's what makes the
|
|
10206
|
+
// Sherlock "read-scope holds by construction" argument hold: quality can
|
|
10207
|
+
// never see anything status/doctor couldn't already see, because it reads
|
|
10208
|
+
// the identical payload.
|
|
10209
|
+
//
|
|
10210
|
+
// One metric from the design doc is deliberately NOT computed at full
|
|
10211
|
+
// fidelity here — it degrades gracefully into a `gaps` entry instead of
|
|
10212
|
+
// failing:
|
|
10213
|
+
// - staleness: /HealthDetail's `memories.expired` count is instance-wide
|
|
10214
|
+
// only (resources/health.ts never buckets it per agent), so --agent
|
|
10215
|
+
// does not scope this metric. The "old + never-recalled" variant isn't
|
|
10216
|
+
// computed at all — no read API exposes per-memory last-recalled data.
|
|
10217
|
+
//
|
|
10218
|
+
// Slice 1b (flair-quality-slice1b): signal density now ALSO carries citation
|
|
10219
|
+
// rate. Kern's design nod: extend /HealthDetail's per-agent aggregation
|
|
10220
|
+
// server-side (resources/health.ts sums Memory.usageCount — a field already
|
|
10221
|
+
// loaded by the existing memory loop, no new query) rather than have the CLI
|
|
10222
|
+
// join against `GET /Memory` itself — keeps quality's read footprint
|
|
10223
|
+
// identical to status/doctor's by construction. `citationRate` here is
|
|
10224
|
+
// computed CLI-side from the two numbers the server hands back
|
|
10225
|
+
// (usageCount/memoryCount), same "server aggregates, CLI computes" split as
|
|
10226
|
+
// every other metric in this file.
|
|
10227
|
+
//
|
|
10228
|
+
// Backward compatibility: an OLDER server's /HealthDetail predates
|
|
10229
|
+
// per-agent `usageCount` entirely (field is `undefined` on the row, not
|
|
10230
|
+
// `0` — Slice 1a's payload literally didn't have the key). That's detected
|
|
10231
|
+
// per-row and the whole report degrades to the Slice-1a write-volume-only
|
|
10232
|
+
// shape + a gap note, rather than silently reporting citationRate 0 as if
|
|
10233
|
+
// it were real data from a server that never sent it.
|
|
10234
|
+
//
|
|
10235
|
+
// Naming (Sherlock security finding on the design round, reaffirmed for
|
|
10236
|
+
// citation rate): "signal density" / "citation rate" describes a USAGE
|
|
10237
|
+
// PATTERN, never a trust/quality verdict. A low citation rate means "writes
|
|
10238
|
+
// exploratory content that's rarely cited", not "noisy" or "untrustworthy"
|
|
10239
|
+
// — same for "quiet agents": an ops fact (days since last write), not a
|
|
10240
|
+
// trust signal. Keep that framing in any copy touching this code.
|
|
10241
|
+
//
|
|
10242
|
+
// Slice 1c (flair-quality-slice1c-dedup-spec.md, K&S-resolved to "Option C,
|
|
10243
|
+
// server-side"): dedup-cluster count — how many near-duplicate memory
|
|
10244
|
+
// CLUSTERS exist instance-wide. Sherlock's hard security line: embeddings
|
|
10245
|
+
// are the most sensitive data in the system and must never leave the
|
|
10246
|
+
// server, so — unlike every other metric in this file — the computation
|
|
10247
|
+
// does NOT happen here. A nightly server-side REM step
|
|
10248
|
+
// (resources/MemoryDedupStats.ts, wired into src/rem/runner.ts) computes it
|
|
10249
|
+
// server-side via a bounded-k ANN sweep (reusing the HNSW-backed retrieval
|
|
10250
|
+
// core) and persists ONLY the aggregate `{clusterCount, largestClusterSize,
|
|
10251
|
+
// totalMemoriesInClusters, computedAt}` to a small server-side stat file.
|
|
10252
|
+
// /HealthDetail does a CHEAP read of that file (resources/health.ts) —
|
|
10253
|
+
// still zero new query pattern from the CLI's perspective, same "quality
|
|
10254
|
+
// reads a precomputed number from /HealthDetail" contract as every other
|
|
10255
|
+
// metric here. Nightly-stale by construction (only as fresh as the last
|
|
10256
|
+
// REM cycle) — that's an accepted trade-off, not a bug: it's a "silting up"
|
|
10257
|
+
// trend signal, not a real-time alert. Absent (fresh instance, REM never
|
|
10258
|
+
// run, or an older server) degrades to `null` + a `gaps` entry, never a
|
|
10259
|
+
// false zero.
|
|
10260
|
+
//
|
|
10261
|
+
// Slice 1d (memory-quality arc, self-referential design — no hardcoded canned
|
|
10262
|
+
// queries): recall SPOT-CHECK. Unlike every metric above, this ISN'T read
|
|
10263
|
+
// from /HealthDetail at all — it's the one metric in this file that requires
|
|
10264
|
+
// live QUERIES, because it's checking whether querying itself still works.
|
|
10265
|
+
// For a sample of the querying agent's OWN memories (fetchRecallSpotCheckData
|
|
10266
|
+
// below, GET /Memory?agentId=<self>), a CUE is derived from each memory
|
|
10267
|
+
// (deriveRecallCue — its `subject` if present, else the leading ~8 words /
|
|
10268
|
+
// first sentence of `content`; a PARTIAL cue, never the full content) and
|
|
10269
|
+
// searched for through the EXACT SAME authenticated read path `flair memory
|
|
10270
|
+
// search` uses: `api("POST", "/SemanticSearch", ...)`, which resolves auth
|
|
10271
|
+
// via the shared authedRequest() 5-tier resolver (src/lib/auth-resolve.ts) —
|
|
10272
|
+
// identical code path, so read-scope holds by construction (no new
|
|
10273
|
+
// endpoint, no cross-agent/private data, scoped to the querying agent's own
|
|
10274
|
+
// memories same as `flair memory search` always was). computeRecallSpotCheck
|
|
10275
|
+
// (below) is the pure scorer: recall@k = fraction of sampled memories whose
|
|
10276
|
+
// own id appears in its search's top-k; MRR = mean reciprocal rank (0 if
|
|
10277
|
+
// not found within k).
|
|
10278
|
+
//
|
|
10279
|
+
// Framing — this is a HEALTH SPOT-CHECK, not a benchmark and not a trust
|
|
10280
|
+
// judgment. Querying by a cue derived FROM the target memory is easier than
|
|
10281
|
+
// a real user query, so a high score means "recall is functioning", not
|
|
10282
|
+
// "recall is optimal". Its job is to catch recall CRATERING (embeddings
|
|
10283
|
+
// down, index busted) — the score collapsing toward 0 is the signal, not
|
|
10284
|
+
// fine-grained precision grading. Requires an actual agent identity to
|
|
10285
|
+
// query AS (semantic search is agent-scoped) — no identity, fewer than the
|
|
10286
|
+
// sample-size memories to sample, or a search error all degrade to `null` +
|
|
10287
|
+
// a `gaps` entry, same graceful-degradation contract as every metric here —
|
|
10288
|
+
// NEVER a false 0.0 masquerading as a real (broken) score.
|
|
10289
|
+
/** First-pass default, same "documented heuristic, not derived from data we
|
|
10290
|
+
* don't have" spirit as health.ts's own 10%-hash-fallback threshold below.
|
|
10291
|
+
* Tunable later if a real fleet shows this is too loud/quiet. */
|
|
10292
|
+
export const QUALITY_QUIET_THRESHOLD_DAYS = 7;
|
|
10293
|
+
/** Mirrors resources/health.ts's own hash-fallback warning threshold (kept as
|
|
10294
|
+
* a literal constant here rather than imported — health.ts computes its
|
|
10295
|
+
* warning string server-side, this recomputes the same judgment CLI-side
|
|
10296
|
+
* from the raw counts so quality doesn't depend on parsing warning text). */
|
|
10297
|
+
export const QUALITY_HASH_FALLBACK_DEGRADED_PCT = 10;
|
|
10298
|
+
/** Recall spot-check (Slice 1d) defaults — how many of the querying agent's
|
|
10299
|
+
* own memories to sample, and the top-k depth each is searched at. Same
|
|
10300
|
+
* "first-pass default, tunable later" spirit as the thresholds above. */
|
|
10301
|
+
export const QUALITY_RECALL_SAMPLE_SIZE = 10;
|
|
10302
|
+
export const QUALITY_RECALL_K = 5;
|
|
10303
|
+
/**
|
|
10304
|
+
* Derive a PARTIAL search cue from a memory — used by the recall spot-check
|
|
10305
|
+
* (Slice 1d) to query for a memory without handing back its full content.
|
|
10306
|
+
* Prefers `subject` when present and non-trivial (a real word or phrase, not
|
|
10307
|
+
* empty/whitespace-only padding); otherwise falls back to the first sentence
|
|
10308
|
+
* of `content`, capped to the leading ~8 words so the cue stays a genuine
|
|
10309
|
+
* partial cue rather than the whole memory. Pure — no I/O.
|
|
10310
|
+
*/
|
|
10311
|
+
export function deriveRecallCue(memory) {
|
|
10312
|
+
const subject = (memory.subject ?? "").trim();
|
|
10313
|
+
if (subject.length >= 3)
|
|
10314
|
+
return subject;
|
|
10315
|
+
const content = (memory.content ?? "").trim();
|
|
10316
|
+
if (!content)
|
|
10317
|
+
return "";
|
|
10318
|
+
const sentenceMatch = content.match(/^[^.!?\n]+[.!?]?/);
|
|
10319
|
+
const firstSentence = (sentenceMatch ? sentenceMatch[0] : content).trim();
|
|
10320
|
+
const words = firstSentence.split(/\s+/).filter(Boolean);
|
|
10321
|
+
const cueWordLimit = 8;
|
|
10322
|
+
return words.length <= cueWordLimit ? firstSentence : words.slice(0, cueWordLimit).join(" ");
|
|
10323
|
+
}
|
|
10324
|
+
/**
|
|
10325
|
+
* Pure scorer for the recall spot-check (Slice 1d): given the ids of the
|
|
10326
|
+
* sampled memories and, for each, the list of memory ids its derived-cue
|
|
10327
|
+
* search returned (already agent-scoped via the same read path `flair
|
|
10328
|
+
* memory search` uses), compute recall@k + MRR. `perQueryResultIds[i]` is
|
|
10329
|
+
* truncated to the first `k` entries here (not assumed pre-truncated by the
|
|
10330
|
+
* caller) so a caller that over-fetches still gets a correct top-k score.
|
|
10331
|
+
* Never throws; an empty sample scores 0/0 rather than dividing by zero —
|
|
10332
|
+
* callers are expected to treat an empty sample as a `gaps` case, not a
|
|
10333
|
+
* real 0.0 score (see fetchRecallSpotCheckData / computeQualityReport).
|
|
10334
|
+
*/
|
|
10335
|
+
export function computeRecallSpotCheck(sampledIds, perQueryResultIds, k) {
|
|
10336
|
+
const sampleSize = sampledIds.length;
|
|
10337
|
+
if (sampleSize === 0) {
|
|
10338
|
+
return { recallAtK: 0, mrr: 0, sampleSize: 0, k };
|
|
10339
|
+
}
|
|
10340
|
+
let hits = 0;
|
|
10341
|
+
let reciprocalSum = 0;
|
|
10342
|
+
for (let i = 0; i < sampleSize; i++) {
|
|
10343
|
+
const targetId = sampledIds[i];
|
|
10344
|
+
const topK = (perQueryResultIds[i] ?? []).slice(0, k);
|
|
10345
|
+
const rank = topK.indexOf(targetId);
|
|
10346
|
+
if (rank !== -1) {
|
|
10347
|
+
hits += 1;
|
|
10348
|
+
reciprocalSum += 1 / (rank + 1);
|
|
10349
|
+
}
|
|
10350
|
+
}
|
|
10351
|
+
return {
|
|
10352
|
+
recallAtK: Math.round((hits / sampleSize) * 100) / 100,
|
|
10353
|
+
mrr: Math.round((reciprocalSum / sampleSize) * 100) / 100,
|
|
10354
|
+
sampleSize,
|
|
10355
|
+
k,
|
|
10356
|
+
};
|
|
10357
|
+
}
|
|
10358
|
+
/**
|
|
10359
|
+
* Pure computation: /HealthDetail response (+ reachability) → quality report.
|
|
10360
|
+
* Never throws — every missing data source degrades to a null section + a
|
|
10361
|
+
* `gaps` entry (same graceful-degradation contract as `flair doctor`), so a
|
|
10362
|
+
* partially-populated instance still gets a partial, honest report instead
|
|
10363
|
+
* of a crash.
|
|
10364
|
+
*
|
|
10365
|
+
* `opts.recallSpotCheckData` is the ONE exception to "fed purely from
|
|
10366
|
+
* /HealthDetail": the recall spot-check (Slice 1d) requires live queries
|
|
10367
|
+
* (fetchRecallSpotCheckData, run by the `quality` command BEFORE calling
|
|
10368
|
+
* here, same "I/O happens outside, this function only computes" split as
|
|
10369
|
+
* fetchHealthDetail/computeQualityReport itself). Passing nothing degrades
|
|
10370
|
+
* to a `gaps` entry, same as every other metric.
|
|
10371
|
+
*/
|
|
10372
|
+
export function computeQualityReport(healthy, healthData, opts = {}) {
|
|
10373
|
+
const now = opts.now ?? Date.now();
|
|
10374
|
+
const agentFilter = opts.agentId ?? null;
|
|
10375
|
+
const gaps = [];
|
|
10376
|
+
// ── Instance health: migrations ──
|
|
10377
|
+
let migrationsClean = null;
|
|
10378
|
+
let haltedMigrations = [];
|
|
10379
|
+
const migList = healthData?.migrations?.migrations;
|
|
10380
|
+
if (Array.isArray(migList)) {
|
|
10381
|
+
haltedMigrations = migList
|
|
10382
|
+
.filter((m) => m?.state === "halted" || m?.state === "failed")
|
|
10383
|
+
.map((m) => ({ id: m.id, state: m.state, reason: m.reason }));
|
|
10384
|
+
migrationsClean = haltedMigrations.length === 0;
|
|
10385
|
+
}
|
|
10386
|
+
else {
|
|
10387
|
+
gaps.push({ metric: "instance.migrationsClean", reason: "no migrations block in /HealthDetail response" });
|
|
10388
|
+
}
|
|
10389
|
+
// ── Instance health: embeddings operational ──
|
|
10390
|
+
// Inferred from stored coverage stats (hash-fallback %, mixed embedding
|
|
10391
|
+
// models) — NOT a live semantic round-trip like `flair doctor` runs
|
|
10392
|
+
// (verifySemanticSearch writes a probe memory to verify recall-by-meaning,
|
|
10393
|
+
// which this read-only command must not do).
|
|
10394
|
+
let embeddingsStatus = "unknown";
|
|
10395
|
+
let embeddingsDetail = "no memory stats available";
|
|
10396
|
+
const memories = healthData?.memories;
|
|
10397
|
+
if (memories && typeof memories.total === "number") {
|
|
10398
|
+
if (memories.total === 0) {
|
|
10399
|
+
embeddingsDetail = "no memories written yet";
|
|
10400
|
+
}
|
|
10401
|
+
else {
|
|
10402
|
+
const hashFallback = memories.hashFallback ?? 0;
|
|
10403
|
+
const pct = Math.round((hashFallback / memories.total) * 100);
|
|
10404
|
+
const modelCounts = (memories.modelCounts ?? {});
|
|
10405
|
+
const realModels = Object.keys(modelCounts).filter((k) => k !== "hash-512d" && modelCounts[k] > 0);
|
|
10406
|
+
if (pct >= QUALITY_HASH_FALLBACK_DEGRADED_PCT) {
|
|
10407
|
+
embeddingsStatus = "degraded";
|
|
10408
|
+
embeddingsDetail = `${hashFallback}/${memories.total} (${pct}%) memories are hash-fallback`;
|
|
10409
|
+
}
|
|
10410
|
+
else if (realModels.length > 1) {
|
|
10411
|
+
embeddingsStatus = "degraded";
|
|
10412
|
+
embeddingsDetail = `multiple embedding models in use (${realModels.join(", ")}) — cross-model search unreliable`;
|
|
10413
|
+
}
|
|
10414
|
+
else {
|
|
10415
|
+
embeddingsStatus = "ok";
|
|
10416
|
+
embeddingsDetail = `${memories.total - hashFallback}/${memories.total} memories have real embeddings`;
|
|
10417
|
+
}
|
|
10418
|
+
}
|
|
10419
|
+
}
|
|
10420
|
+
// ── Embedding coverage ──
|
|
10421
|
+
let embeddingCoverage = null;
|
|
10422
|
+
if (memories && typeof memories.total === "number") {
|
|
10423
|
+
const total = memories.total;
|
|
10424
|
+
const hashFallback = memories.hashFallback ?? 0;
|
|
10425
|
+
const withEmbeddings = typeof memories.withEmbeddings === "number" ? memories.withEmbeddings : Math.max(0, total - hashFallback);
|
|
10426
|
+
const coveragePct = total > 0 ? Math.round((withEmbeddings / total) * 100) : 0;
|
|
10427
|
+
embeddingCoverage = { total, withEmbeddings, hashFallback, coveragePct };
|
|
10428
|
+
}
|
|
10429
|
+
else {
|
|
10430
|
+
gaps.push({ metric: "embeddingCoverage", reason: "no memory stats available in /HealthDetail response" });
|
|
10431
|
+
}
|
|
10432
|
+
// ── Staleness (instance-wide only — see module doc above) ──
|
|
10433
|
+
let staleness = null;
|
|
10434
|
+
if (memories && typeof memories.total === "number" && typeof memories.expired === "number") {
|
|
10435
|
+
const total = memories.total;
|
|
10436
|
+
const expired = memories.expired;
|
|
10437
|
+
const stalePct = total > 0 ? Math.round((expired / total) * 100) : 0;
|
|
10438
|
+
staleness = { scope: "instance", total, expired, stalePct };
|
|
10439
|
+
gaps.push({
|
|
10440
|
+
metric: "staleness",
|
|
10441
|
+
reason: "instance-wide only — /HealthDetail's `expired` count isn't broken down per agent, so --agent doesn't scope this metric; the \"old + never-recalled\" dead-weight variant also isn't computed (no read API exposes per-memory last-recalled data)",
|
|
10442
|
+
});
|
|
10443
|
+
}
|
|
10444
|
+
else {
|
|
10445
|
+
gaps.push({ metric: "staleness", reason: "no expired-memory count available in /HealthDetail response" });
|
|
10446
|
+
}
|
|
10447
|
+
// ── Per-agent rows (shared source for signal density + quiet agents) ──
|
|
10448
|
+
const perAgentAll = Array.isArray(healthData?.agents?.perAgent) ? healthData.agents.perAgent : [];
|
|
10449
|
+
const havePerAgent = Array.isArray(healthData?.agents?.perAgent);
|
|
10450
|
+
const scopedAgents = agentFilter ? perAgentAll.filter((r) => r.id === agentFilter) : perAgentAll;
|
|
10451
|
+
// Detected from the UNFILTERED rows (not scopedAgents) so `--agent` never
|
|
10452
|
+
// masquerades server capability as data-scoping — vacuously true on an
|
|
10453
|
+
// empty perAgent array (nothing to prove otherwise, and both scopes render
|
|
10454
|
+
// identically empty either way).
|
|
10455
|
+
const haveUsageCount = perAgentAll.every((r) => typeof r.usageCount === "number");
|
|
10456
|
+
// ── Signal density (write volume, + citation rate when the server supports it) ──
|
|
10457
|
+
let signalDensity = null;
|
|
10458
|
+
if (havePerAgent && haveUsageCount) {
|
|
10459
|
+
signalDensity = {
|
|
10460
|
+
scope: "write-and-citation",
|
|
10461
|
+
perAgent: scopedAgents.map((r) => {
|
|
10462
|
+
const usageCount = r.usageCount ?? 0;
|
|
10463
|
+
const citationRate = r.memoryCount > 0 ? Math.round((usageCount / r.memoryCount) * 100) / 100 : 0;
|
|
10464
|
+
return { id: r.id, memoryCount: r.memoryCount, writes24h: r.writes24h, lastWriteAt: r.lastWriteAt, usageCount, citationRate };
|
|
10465
|
+
}),
|
|
10466
|
+
};
|
|
10467
|
+
}
|
|
10468
|
+
else if (havePerAgent) {
|
|
10469
|
+
signalDensity = {
|
|
10470
|
+
scope: "write-volume",
|
|
10471
|
+
perAgent: scopedAgents.map((r) => ({ id: r.id, memoryCount: r.memoryCount, writes24h: r.writes24h, lastWriteAt: r.lastWriteAt })),
|
|
10472
|
+
};
|
|
10473
|
+
gaps.push({
|
|
10474
|
+
metric: "signalDensity",
|
|
10475
|
+
reason: "citation rate unavailable — server predates per-agent usageCount in /HealthDetail; upgrade the server",
|
|
10476
|
+
});
|
|
10477
|
+
}
|
|
10478
|
+
else {
|
|
10479
|
+
gaps.push({ metric: "signalDensity", reason: "no per-agent stats available in /HealthDetail response" });
|
|
10480
|
+
}
|
|
10481
|
+
// ── Quiet agents (ops fact — not a trust signal) ──
|
|
10482
|
+
let quietAgents = null;
|
|
10483
|
+
if (havePerAgent) {
|
|
10484
|
+
const rows = scopedAgents.map((r) => {
|
|
10485
|
+
const daysSinceLastWrite = r.lastWriteAt ? Math.floor((now - new Date(r.lastWriteAt).getTime()) / 86_400_000) : null;
|
|
10486
|
+
const quiet = daysSinceLastWrite === null ? true : daysSinceLastWrite >= QUALITY_QUIET_THRESHOLD_DAYS;
|
|
10487
|
+
return { id: r.id, memoryCount: r.memoryCount, writes24h: r.writes24h, lastWriteAt: r.lastWriteAt, daysSinceLastWrite, quiet };
|
|
10488
|
+
});
|
|
10489
|
+
quietAgents = { thresholdDays: QUALITY_QUIET_THRESHOLD_DAYS, perAgent: rows, quietCount: rows.filter((r) => r.quiet).length };
|
|
10490
|
+
}
|
|
10491
|
+
else {
|
|
10492
|
+
gaps.push({ metric: "quietAgents", reason: "no per-agent stats available in /HealthDetail response" });
|
|
10493
|
+
}
|
|
10494
|
+
// ── Dedup clusters (instance-wide near-duplicate count — flair-quality
|
|
10495
|
+
// Slice 1c) — an ops/health signal, not a trust judgment (see module doc
|
|
10496
|
+
// and QualityReport['dedupClusters'] doc above). Always instance-wide;
|
|
10497
|
+
// --agent does not scope it (matches staleness's precedent — the
|
|
10498
|
+
// underlying stat has no per-agent breakdown, by design: per-memory
|
|
10499
|
+
// cluster membership is a disclosure surface Sherlock's review explicitly
|
|
10500
|
+
// ruled out storing at all).
|
|
10501
|
+
let dedupClusters = null;
|
|
10502
|
+
const dedup = healthData?.dedup;
|
|
10503
|
+
if (dedup &&
|
|
10504
|
+
typeof dedup.clusterCount === "number" &&
|
|
10505
|
+
typeof dedup.largestClusterSize === "number" &&
|
|
10506
|
+
typeof dedup.totalMemoriesInClusters === "number" &&
|
|
10507
|
+
typeof dedup.computedAt === "string") {
|
|
10508
|
+
dedupClusters = {
|
|
10509
|
+
clusterCount: dedup.clusterCount,
|
|
10510
|
+
largestClusterSize: dedup.largestClusterSize,
|
|
10511
|
+
totalMemoriesInClusters: dedup.totalMemoriesInClusters,
|
|
10512
|
+
computedAt: dedup.computedAt,
|
|
10513
|
+
};
|
|
10514
|
+
}
|
|
10515
|
+
else {
|
|
10516
|
+
gaps.push({
|
|
10517
|
+
metric: "dedupClusters",
|
|
10518
|
+
reason: "no dedup-cluster stat yet — computed nightly by REM (see `flair rem nightly enable`); run `flair rem nightly run-once` or wait for the first scheduled cycle",
|
|
10519
|
+
});
|
|
10520
|
+
}
|
|
10521
|
+
// ── Recall spot-check (flair-quality Slice 1d) — see module doc above and
|
|
10522
|
+
// QualityReport['recallSpotCheck'] doc for the full framing. Fed by
|
|
10523
|
+
// fetchRecallSpotCheckData's already-fetched raw ids (not by healthData —
|
|
10524
|
+
// this is the one metric here that needed a live query, not a
|
|
10525
|
+
// /HealthDetail read); scored by the pure computeRecallSpotCheck.
|
|
10526
|
+
let recallSpotCheck = null;
|
|
10527
|
+
const rsc = opts.recallSpotCheckData;
|
|
10528
|
+
if (rsc?.ok && rsc.sampledIds && rsc.perQueryResultIds && typeof rsc.k === "number") {
|
|
10529
|
+
const scored = computeRecallSpotCheck(rsc.sampledIds, rsc.perQueryResultIds, rsc.k);
|
|
10530
|
+
recallSpotCheck = { agentId: rsc.agentId ?? agentFilter ?? null, ...scored };
|
|
10531
|
+
}
|
|
10532
|
+
else {
|
|
10533
|
+
gaps.push({
|
|
10534
|
+
metric: "recallSpotCheck",
|
|
10535
|
+
reason: rsc?.skipReason ?? "recall spot-check not attempted — no data passed to computeQualityReport",
|
|
10536
|
+
});
|
|
10537
|
+
}
|
|
10538
|
+
return {
|
|
10539
|
+
agentFilter,
|
|
10540
|
+
instance: { up: healthy, migrationsClean, haltedMigrations, embeddingsStatus, embeddingsDetail },
|
|
10541
|
+
embeddingCoverage,
|
|
10542
|
+
staleness,
|
|
10543
|
+
signalDensity,
|
|
10544
|
+
quietAgents,
|
|
10545
|
+
dedupClusters,
|
|
10546
|
+
recallSpotCheck,
|
|
10547
|
+
gaps,
|
|
10548
|
+
};
|
|
10549
|
+
}
|
|
10550
|
+
/**
|
|
10551
|
+
* The I/O half of the recall spot-check (Slice 1d): fetch a sample of
|
|
10552
|
+
* `agentId`'s own memories and, for each, search for a cue derived from it.
|
|
10553
|
+
* Reuses the EXACT read path `flair memory search` / `flair memory list`
|
|
10554
|
+
* use — `api()` (→ authedRequest's 5-tier resolver) for both the
|
|
10555
|
+
* `GET /Memory?agentId=...` sample fetch and the `POST /SemanticSearch`
|
|
10556
|
+
* queries — so this has zero new endpoint and zero new auth mechanism; it
|
|
10557
|
+
* is scoped to `agentId`'s own memories exactly as those commands already
|
|
10558
|
+
* are. Never throws: every failure mode (no agentId, fewer than
|
|
10559
|
+
* `sampleSize` memories, a fetch/search error) returns `{ ok: false,
|
|
10560
|
+
* skipReason }` for computeQualityReport to turn into a `gaps` entry.
|
|
10561
|
+
*/
|
|
10562
|
+
async function fetchRecallSpotCheckData(agentId, baseUrl, opts = {}) {
|
|
10563
|
+
const sampleSize = opts.sampleSize ?? QUALITY_RECALL_SAMPLE_SIZE;
|
|
10564
|
+
const k = opts.k ?? QUALITY_RECALL_K;
|
|
10565
|
+
if (!agentId) {
|
|
10566
|
+
return { ok: false, skipReason: "no agent identity to query as — pass --agent or set FLAIR_AGENT_ID" };
|
|
10567
|
+
}
|
|
10568
|
+
let all;
|
|
10569
|
+
try {
|
|
10570
|
+
const q = new URLSearchParams({ agentId }).toString();
|
|
10571
|
+
const raw = await api("GET", `/Memory?${q}`, undefined, { baseUrl });
|
|
10572
|
+
all = Array.isArray(raw) ? raw : (raw?.results ?? raw?.items ?? []);
|
|
10573
|
+
}
|
|
10574
|
+
catch (err) {
|
|
10575
|
+
return { ok: false, agentId, skipReason: `could not fetch memories to sample: ${err?.message ?? String(err)}` };
|
|
10576
|
+
}
|
|
10577
|
+
if (all.length < sampleSize) {
|
|
10578
|
+
return {
|
|
10579
|
+
ok: false,
|
|
10580
|
+
agentId,
|
|
10581
|
+
skipReason: `agent '${agentId}' has ${all.length} memories, fewer than the ${sampleSize} needed to sample`,
|
|
10582
|
+
};
|
|
10583
|
+
}
|
|
10584
|
+
// Deterministic sample: the sampleSize most-recently-written memories.
|
|
10585
|
+
const sorted = all.slice().sort((a, b) => {
|
|
10586
|
+
const ta = a.createdAt ? new Date(a.createdAt).getTime() : 0;
|
|
10587
|
+
const tb = b.createdAt ? new Date(b.createdAt).getTime() : 0;
|
|
10588
|
+
return tb - ta;
|
|
10589
|
+
});
|
|
10590
|
+
const sampled = sorted.slice(0, sampleSize);
|
|
10591
|
+
const sampledIds = [];
|
|
10592
|
+
const perQueryResultIds = [];
|
|
10593
|
+
try {
|
|
10594
|
+
for (const m of sampled) {
|
|
10595
|
+
const id = String(m.id);
|
|
10596
|
+
const cue = deriveRecallCue(m);
|
|
10597
|
+
const body = { agentId, q: cue, limit: k };
|
|
10598
|
+
const res = await api("POST", "/SemanticSearch", body, { baseUrl });
|
|
10599
|
+
const results = Array.isArray(res) ? res : (res?.results ?? []);
|
|
10600
|
+
sampledIds.push(id);
|
|
10601
|
+
perQueryResultIds.push(results.map((r) => String(r.id)));
|
|
10602
|
+
}
|
|
10603
|
+
}
|
|
10604
|
+
catch (err) {
|
|
10605
|
+
return { ok: false, agentId, skipReason: `recall spot-check search failed: ${err?.message ?? String(err)}` };
|
|
10606
|
+
}
|
|
10607
|
+
return { ok: true, agentId, sampledIds, perQueryResultIds, k };
|
|
10608
|
+
}
|
|
10609
|
+
// ─── flair quality --emit (Slice 2 of the memory-quality-observability arc:
|
|
10610
|
+
// quality OrgEvents) ─────────────────────────────────────────────────────────
|
|
10611
|
+
//
|
|
10612
|
+
// Design (K&S-approved in the arc round, honored exactly here):
|
|
10613
|
+
//
|
|
10614
|
+
// - NO schema change, NO new table/resource. Events ride the existing
|
|
10615
|
+
// OrgEvent surface (schemas/event.graphql: kind/scope/summary/detail/
|
|
10616
|
+
// targetIds/refId — see `flair orgevent` above) via the exact same
|
|
10617
|
+
// PUT /OrgEvent/{id} write shape, extracted into `publishOrgEvent()` below
|
|
10618
|
+
// so both commands share one call site rather than two hand-rolled fetches.
|
|
10619
|
+
// - The threshold/diff DECISION is CLI-side (diffQualitySnapshots, pure,
|
|
10620
|
+
// fixture-tested below); emission is a thin write via that existing
|
|
10621
|
+
// surface. Kind is one of two: `quality.threshold_crossed` (an absolute
|
|
10622
|
+
// line was crossed since the last snapshot) or `quality.regression` (a
|
|
10623
|
+
// metric moved backward by more than its delta threshold since the last
|
|
10624
|
+
// snapshot) — see QualityEventFinding.
|
|
10625
|
+
// - Snapshots are stored AS Flair memories (durability persistent, subject
|
|
10626
|
+
// `quality-snapshot/<host>` — qualitySnapshotSubject() below) — free
|
|
10627
|
+
// history + search, dogfoods the product, no new storage surface. Content
|
|
10628
|
+
// is the COMPACT numeric core only (QualitySnapshotCore), never the full
|
|
10629
|
+
// human report — see buildQualitySnapshot().
|
|
10630
|
+
// - Sherlock's constraint: events carry BEHAVIORAL FACTS only, never trust
|
|
10631
|
+
// judgments — "embedding coverage dropped to 85% (threshold 90%)", never
|
|
10632
|
+
// "agent X is low quality". Every summary string below is written to that
|
|
10633
|
+
// discipline; keep it that way in any future edit here.
|
|
10634
|
+
// - Edge-triggered, not level-triggered: every threshold/regression check
|
|
10635
|
+
// below fires only on the TRANSITION since the immediately-previous
|
|
10636
|
+
// snapshot (e.g. quietAgents requires "was NOT quiet last snapshot, IS
|
|
10637
|
+
// quiet now" — the spec's own "NEWLY quiet" wording), not on every run
|
|
10638
|
+
// while a condition merely persists. Without this, a metric that stays
|
|
10639
|
+
// below threshold across many `--emit` runs would re-emit an event every
|
|
10640
|
+
// single run — pure noise. First run (no previous snapshot) therefore
|
|
10641
|
+
// always emits nothing (diffQualitySnapshots(current, null) === []) — there
|
|
10642
|
+
// is no prior state to diff against, so nothing can have "crossed" or
|
|
10643
|
+
// "regressed" yet; that run only establishes the baseline.
|
|
10644
|
+
// - Missing data (a null/gap section on either side of the diff) never
|
|
10645
|
+
// produces an event — absence of data is a gap, not a regression. Encoded
|
|
10646
|
+
// by requiring BOTH current and previous to carry a given metric before
|
|
10647
|
+
// diffing it at all.
|
|
10648
|
+
/** First-pass defaults for the Slice 2 diff/thresholds — same "documented
|
|
10649
|
+
* heuristic, tunable later against a real fleet" spirit as
|
|
10650
|
+
* QUALITY_QUIET_THRESHOLD_DAYS / QUALITY_HASH_FALLBACK_DEGRADED_PCT above.
|
|
10651
|
+
* Deliberately NOT exposed as CLI flags (per the arc design: the diff
|
|
10652
|
+
* decision stays CLI-side and legible in one place, not ad-hoc per
|
|
10653
|
+
* invocation) — change these constants and their fixture tests together. */
|
|
10654
|
+
export const QUALITY_EVENT_COVERAGE_ABS_THRESHOLD_PCT = 90;
|
|
10655
|
+
export const QUALITY_EVENT_COVERAGE_DROP_THRESHOLD_PCT = 5;
|
|
10656
|
+
export const QUALITY_EVENT_STALENESS_ABS_THRESHOLD_PCT = 10;
|
|
10657
|
+
export const QUALITY_EVENT_RECALL_DROP_THRESHOLD = 0.2;
|
|
10658
|
+
export const QUALITY_EVENT_DEDUP_GROWTH_PCT_THRESHOLD = 0.5; // >50%
|
|
10659
|
+
export const QUALITY_EVENT_DEDUP_GROWTH_ABS_THRESHOLD = 5; // AND by >= 5 clusters
|
|
10660
|
+
/** Pure: full QualityReport → compact snapshot core. Any section that's
|
|
10661
|
+
* `null` in the report (a gap) stays `null` in the snapshot — the diff step
|
|
10662
|
+
* treats that as "no event", never a false 0. */
|
|
10663
|
+
export function buildQualitySnapshot(report, computedAt) {
|
|
10664
|
+
return {
|
|
10665
|
+
schemaVersion: 1,
|
|
10666
|
+
computedAt: computedAt ?? new Date().toISOString(),
|
|
10667
|
+
agentFilter: report.agentFilter,
|
|
10668
|
+
embeddingCoverage: report.embeddingCoverage ? { coveragePct: report.embeddingCoverage.coveragePct } : null,
|
|
10669
|
+
staleness: report.staleness ? { stalePct: report.staleness.stalePct } : null,
|
|
10670
|
+
recallSpotCheck: report.recallSpotCheck
|
|
10671
|
+
? { recallAtK: report.recallSpotCheck.recallAtK, mrr: report.recallSpotCheck.mrr }
|
|
10672
|
+
: null,
|
|
10673
|
+
quietAgents: report.quietAgents
|
|
10674
|
+
? { perAgent: report.quietAgents.perAgent.map((a) => ({ id: a.id, quiet: a.quiet, daysSinceLastWrite: a.daysSinceLastWrite })) }
|
|
10675
|
+
: null,
|
|
10676
|
+
dedupClusters: report.dedupClusters ? { clusterCount: report.dedupClusters.clusterCount } : null,
|
|
10677
|
+
};
|
|
10678
|
+
}
|
|
10679
|
+
/**
|
|
10680
|
+
* Pure diff: current snapshot + previous snapshot (or null on a first run)
|
|
10681
|
+
* → the list of OrgEvent findings to emit. Never throws. See the module doc
|
|
10682
|
+
* above for the edge-triggered / missing-data-means-no-event rules; fixture
|
|
10683
|
+
* tests live in test/unit/quality-report.test.ts.
|
|
10684
|
+
*/
|
|
10685
|
+
export function diffQualitySnapshots(current, previous) {
|
|
10686
|
+
const findings = [];
|
|
10687
|
+
if (!previous)
|
|
10688
|
+
return findings; // first run — nothing to diff against yet
|
|
10689
|
+
// ── embedding coverage: absolute floor (edge-triggered) + delta drop ──
|
|
10690
|
+
if (current.embeddingCoverage && previous.embeddingCoverage) {
|
|
10691
|
+
const before = previous.embeddingCoverage.coveragePct;
|
|
10692
|
+
const after = current.embeddingCoverage.coveragePct;
|
|
10693
|
+
if (after < QUALITY_EVENT_COVERAGE_ABS_THRESHOLD_PCT && before >= QUALITY_EVENT_COVERAGE_ABS_THRESHOLD_PCT) {
|
|
10694
|
+
findings.push({
|
|
10695
|
+
kind: "quality.threshold_crossed",
|
|
10696
|
+
scope: "quality",
|
|
10697
|
+
summary: `embedding coverage dropped to ${after}% (threshold ${QUALITY_EVENT_COVERAGE_ABS_THRESHOLD_PCT}%)`,
|
|
10698
|
+
detail: { metric: "embeddingCoverage.coveragePct", before, after, threshold: QUALITY_EVENT_COVERAGE_ABS_THRESHOLD_PCT },
|
|
10699
|
+
});
|
|
10700
|
+
}
|
|
10701
|
+
if (before - after > QUALITY_EVENT_COVERAGE_DROP_THRESHOLD_PCT) {
|
|
10702
|
+
findings.push({
|
|
10703
|
+
kind: "quality.regression",
|
|
10704
|
+
scope: "quality",
|
|
10705
|
+
summary: `embedding coverage dropped ${before - after} points since last snapshot (${before}% → ${after}%)`,
|
|
10706
|
+
detail: { metric: "embeddingCoverage.coveragePct", before, after, threshold: QUALITY_EVENT_COVERAGE_DROP_THRESHOLD_PCT },
|
|
10707
|
+
});
|
|
10708
|
+
}
|
|
10709
|
+
}
|
|
10710
|
+
// ── staleness: absolute ceiling only (edge-triggered) ──
|
|
10711
|
+
if (current.staleness && previous.staleness) {
|
|
10712
|
+
const before = previous.staleness.stalePct;
|
|
10713
|
+
const after = current.staleness.stalePct;
|
|
10714
|
+
if (after > QUALITY_EVENT_STALENESS_ABS_THRESHOLD_PCT && before <= QUALITY_EVENT_STALENESS_ABS_THRESHOLD_PCT) {
|
|
10715
|
+
findings.push({
|
|
10716
|
+
kind: "quality.threshold_crossed",
|
|
10717
|
+
scope: "quality",
|
|
10718
|
+
summary: `staleness rose to ${after}% past validTo (threshold ${QUALITY_EVENT_STALENESS_ABS_THRESHOLD_PCT}%)`,
|
|
10719
|
+
detail: { metric: "staleness.stalePct", before, after, threshold: QUALITY_EVENT_STALENESS_ABS_THRESHOLD_PCT },
|
|
10720
|
+
});
|
|
10721
|
+
}
|
|
10722
|
+
}
|
|
10723
|
+
// ── recall spot-check: recall@k and MRR, same delta-drop threshold ──
|
|
10724
|
+
if (current.recallSpotCheck && previous.recallSpotCheck) {
|
|
10725
|
+
const beforeR = previous.recallSpotCheck.recallAtK;
|
|
10726
|
+
const afterR = current.recallSpotCheck.recallAtK;
|
|
10727
|
+
if (beforeR - afterR > QUALITY_EVENT_RECALL_DROP_THRESHOLD) {
|
|
10728
|
+
findings.push({
|
|
10729
|
+
kind: "quality.regression",
|
|
10730
|
+
scope: "quality",
|
|
10731
|
+
summary: `recall spot-check recall@k dropped from ${beforeR} to ${afterR} since last snapshot`,
|
|
10732
|
+
detail: { metric: "recallSpotCheck.recallAtK", before: beforeR, after: afterR, threshold: QUALITY_EVENT_RECALL_DROP_THRESHOLD },
|
|
10733
|
+
});
|
|
10734
|
+
}
|
|
10735
|
+
const beforeM = previous.recallSpotCheck.mrr;
|
|
10736
|
+
const afterM = current.recallSpotCheck.mrr;
|
|
10737
|
+
if (beforeM - afterM > QUALITY_EVENT_RECALL_DROP_THRESHOLD) {
|
|
10738
|
+
findings.push({
|
|
10739
|
+
kind: "quality.regression",
|
|
10740
|
+
scope: "quality",
|
|
10741
|
+
summary: `recall spot-check MRR dropped from ${beforeM} to ${afterM} since last snapshot`,
|
|
10742
|
+
detail: { metric: "recallSpotCheck.mrr", before: beforeM, after: afterM, threshold: QUALITY_EVENT_RECALL_DROP_THRESHOLD },
|
|
10743
|
+
});
|
|
10744
|
+
}
|
|
10745
|
+
}
|
|
10746
|
+
// ── quiet agents: per-agent, NEWLY quiet only (was false last snapshot,
|
|
10747
|
+
// true now) — never re-fires for an agent that was already quiet last
|
|
10748
|
+
// snapshot, and never fires for an agent absent from the previous snapshot
|
|
10749
|
+
// (a brand-new agent can't have "regressed" from a state we never saw). ──
|
|
10750
|
+
if (current.quietAgents && previous.quietAgents) {
|
|
10751
|
+
const prevQuietById = new Map(previous.quietAgents.perAgent.map((a) => [a.id, a.quiet]));
|
|
10752
|
+
for (const a of current.quietAgents.perAgent) {
|
|
10753
|
+
if (a.quiet && prevQuietById.get(a.id) === false) {
|
|
10754
|
+
const days = a.daysSinceLastWrite;
|
|
10755
|
+
findings.push({
|
|
10756
|
+
kind: "quality.threshold_crossed",
|
|
10757
|
+
scope: "quality",
|
|
10758
|
+
summary: days == null ? `agent ${a.id} quiet — no recorded write` : `agent ${a.id} quiet for ${days}d (threshold ${QUALITY_QUIET_THRESHOLD_DAYS}d)`,
|
|
10759
|
+
detail: { metric: "quietAgents", before: false, after: true, threshold: QUALITY_QUIET_THRESHOLD_DAYS },
|
|
10760
|
+
targetIds: [a.id],
|
|
10761
|
+
});
|
|
10762
|
+
}
|
|
10763
|
+
}
|
|
10764
|
+
}
|
|
10765
|
+
// ── dedup clusters: BOTH >50% relative growth AND >=5 absolute growth ──
|
|
10766
|
+
if (current.dedupClusters && previous.dedupClusters) {
|
|
10767
|
+
const before = previous.dedupClusters.clusterCount;
|
|
10768
|
+
const after = current.dedupClusters.clusterCount;
|
|
10769
|
+
const growth = after - before;
|
|
10770
|
+
const growthPct = before > 0 ? growth / before : (after > 0 ? Infinity : 0);
|
|
10771
|
+
if (growthPct > QUALITY_EVENT_DEDUP_GROWTH_PCT_THRESHOLD && growth >= QUALITY_EVENT_DEDUP_GROWTH_ABS_THRESHOLD) {
|
|
10772
|
+
findings.push({
|
|
10773
|
+
kind: "quality.regression",
|
|
10774
|
+
scope: "quality",
|
|
10775
|
+
summary: `dedup cluster count grew from ${before} to ${after} since last snapshot`,
|
|
10776
|
+
detail: { metric: "dedupClusters.clusterCount", before, after, threshold: QUALITY_EVENT_DEDUP_GROWTH_PCT_THRESHOLD },
|
|
10777
|
+
});
|
|
10778
|
+
}
|
|
10779
|
+
}
|
|
10780
|
+
return findings;
|
|
10781
|
+
}
|
|
10782
|
+
/** Subject convention for quality snapshots stored as Flair memories: one
|
|
10783
|
+
* lineage per (agent, Flair instance) pair — an agent that runs
|
|
10784
|
+
* `quality --emit` against more than one target gets independent diff
|
|
10785
|
+
* baselines per target, keyed on HOST (not the full URL, so a bare port
|
|
10786
|
+
* change on the same box doesn't fork the lineage; a different host/instance
|
|
10787
|
+
* correctly starts its own). */
|
|
10788
|
+
export function qualitySnapshotSubject(baseUrl) {
|
|
10789
|
+
let host;
|
|
10790
|
+
try {
|
|
10791
|
+
host = new URL(baseUrl).host;
|
|
10792
|
+
}
|
|
10793
|
+
catch {
|
|
10794
|
+
host = baseUrl.replace(/^[a-zA-Z]+:\/\//, "").replace(/\/.*$/, "");
|
|
10795
|
+
}
|
|
10796
|
+
return `quality-snapshot/${host}`;
|
|
10797
|
+
}
|
|
10798
|
+
/** Fetch the most recent prior quality snapshot for `agentId` at `baseUrl`,
|
|
10799
|
+
* via the exact same read path fetchRecallSpotCheckData uses (`api("GET",
|
|
10800
|
+
* "/Memory?agentId=...")`, self-scoped by the signed request's own agent
|
|
10801
|
+
* identity — no new endpoint). Filters client-side by subject (the server's
|
|
10802
|
+
* `GET /Memory?...` doesn't translate query params into search conditions
|
|
10803
|
+
* beyond the signed agentId scope — see resources/Memory.ts's search()),
|
|
10804
|
+
* same client-side-filter pattern `memory list --hash-fallback` already
|
|
10805
|
+
* uses. Returns null on: no prior snapshot, a fetch error, or a snapshot row
|
|
10806
|
+
* whose content isn't parseable/versioned JSON (never throws — a corrupt or
|
|
10807
|
+
* foreign row degrades to "no snapshot", same as a genuine first run, rather
|
|
10808
|
+
* than crashing `--emit`). */
|
|
10809
|
+
async function fetchPreviousQualitySnapshot(agentId, baseUrl, subject) {
|
|
10810
|
+
let all;
|
|
10811
|
+
try {
|
|
10812
|
+
const q = new URLSearchParams({ agentId }).toString();
|
|
10813
|
+
const raw = await api("GET", `/Memory?${q}`, undefined, { baseUrl });
|
|
10814
|
+
all = Array.isArray(raw) ? raw : (raw?.results ?? raw?.items ?? []);
|
|
10815
|
+
}
|
|
10816
|
+
catch {
|
|
10817
|
+
return null;
|
|
10818
|
+
}
|
|
10819
|
+
const matches = all.filter((m) => m?.subject === subject);
|
|
10820
|
+
if (matches.length === 0)
|
|
10821
|
+
return null;
|
|
10822
|
+
matches.sort((a, b) => {
|
|
10823
|
+
const ta = a.createdAt ? new Date(a.createdAt).getTime() : 0;
|
|
10824
|
+
const tb = b.createdAt ? new Date(b.createdAt).getTime() : 0;
|
|
10825
|
+
return tb - ta;
|
|
10826
|
+
});
|
|
10827
|
+
try {
|
|
10828
|
+
const parsed = JSON.parse(matches[0].content);
|
|
10829
|
+
if (parsed && typeof parsed === "object" && parsed.schemaVersion === 1)
|
|
10830
|
+
return parsed;
|
|
10831
|
+
return null;
|
|
10832
|
+
}
|
|
10833
|
+
catch {
|
|
10834
|
+
return null;
|
|
10835
|
+
}
|
|
10836
|
+
}
|
|
10837
|
+
/** Store `snapshot` as a new persistent-durability memory (subject
|
|
10838
|
+
* `quality-snapshot/<host>`) — the diff baseline the NEXT `--emit` run reads
|
|
10839
|
+
* back via fetchPreviousQualitySnapshot. Content is the compact JSON core
|
|
10840
|
+
* only (buildQualitySnapshot's output), never the full human report. Same
|
|
10841
|
+
* `PUT /Memory/{id}` write shape `memory write-task-summary` already uses.
|
|
10842
|
+
* Throws on a write failure — the CLI action below is responsible for
|
|
10843
|
+
* surfacing that as a clear error, same as every other write path here. */
|
|
10844
|
+
async function storeQualitySnapshot(agentId, baseUrl, subject, snapshot) {
|
|
10845
|
+
const memId = `${agentId}-quality-snapshot-${Date.now()}`;
|
|
10846
|
+
const body = {
|
|
10847
|
+
id: memId,
|
|
10848
|
+
agentId,
|
|
10849
|
+
content: JSON.stringify(snapshot),
|
|
10850
|
+
durability: "persistent",
|
|
10851
|
+
tags: ["quality-snapshot"],
|
|
10852
|
+
subject,
|
|
10853
|
+
type: "quality-snapshot",
|
|
10854
|
+
createdAt: new Date().toISOString(),
|
|
10855
|
+
};
|
|
10856
|
+
const out = await api("PUT", `/Memory/${encodeURIComponent(memId)}`, body, { baseUrl });
|
|
10857
|
+
if (out?.error)
|
|
10858
|
+
throw new Error(String(out.error));
|
|
10859
|
+
return memId;
|
|
10860
|
+
}
|
|
10861
|
+
// ─── flair quality ────────────────────────────────────────────────────────────
|
|
10862
|
+
program
|
|
10863
|
+
.command("quality")
|
|
10864
|
+
.description("Memory-quality report: embedding coverage, staleness, signal density, quiet agents, recall spot-check (read-only)")
|
|
10865
|
+
.option("--port <port>", "Harper HTTP port")
|
|
10866
|
+
.option("--url <url>", "Flair base URL (overrides --port)")
|
|
10867
|
+
.option("--target <url>", "Remote Flair URL (env: FLAIR_TARGET; alias for --url)")
|
|
10868
|
+
.option("--json", "Output as JSON")
|
|
10869
|
+
.option("--agent <id>", "Scope per-agent metrics to one agent id (or set FLAIR_AGENT_ID); default = all agents")
|
|
10870
|
+
.option("--emit", "Slice 2: snapshot this report, diff it against the previous quality-snapshot memory, and emit OrgEvents (quality.threshold_crossed / quality.regression) for any crossings/regressions found. Requires an agent identity (--agent or FLAIR_AGENT_ID) — the opt-in write boundary; without this flag `flair quality` remains fully read-only")
|
|
10871
|
+
.action(async (opts) => {
|
|
10872
|
+
const { healthy, baseUrl, healthData } = await fetchHealthDetail(opts);
|
|
10873
|
+
const agentId = opts.agent || process.env.FLAIR_AGENT_ID || null;
|
|
10874
|
+
if (opts.emit && !agentId) {
|
|
10875
|
+
console.error("Error: --emit requires an agent identity. Pass --agent <id> or set FLAIR_AGENT_ID.");
|
|
10876
|
+
process.exit(1);
|
|
10877
|
+
}
|
|
10878
|
+
// Recall spot-check needs live queries (not just /HealthDetail), so only
|
|
10879
|
+
// attempt it when the instance is actually reachable — no point probing
|
|
10880
|
+
// memory reads against a server fetchHealthDetail already found down.
|
|
10881
|
+
const recallSpotCheckData = healthy
|
|
10882
|
+
? await fetchRecallSpotCheckData(agentId, baseUrl)
|
|
10883
|
+
: { ok: false, skipReason: "instance unreachable" };
|
|
10884
|
+
const report = computeQualityReport(healthy, healthData, { agentId, recallSpotCheckData });
|
|
10885
|
+
// ── Slice 2: --emit is the opt-in write boundary. Everything above this
|
|
10886
|
+
// point is unchanged from pre-Slice-2 behavior; everything in this block
|
|
10887
|
+
// only runs when the flag is passed AND the instance is reachable (an
|
|
10888
|
+
// unreachable instance has nothing to diff against and no live write
|
|
10889
|
+
// target — it falls through to the existing "unreachable" exit(1) below,
|
|
10890
|
+
// same as always). ──
|
|
10891
|
+
let emitResult = null;
|
|
10892
|
+
if (opts.emit && healthy && agentId) {
|
|
10893
|
+
const subject = qualitySnapshotSubject(baseUrl);
|
|
10894
|
+
const previous = await fetchPreviousQualitySnapshot(agentId, baseUrl, subject);
|
|
10895
|
+
const current = buildQualitySnapshot(report);
|
|
10896
|
+
const findings = diffQualitySnapshots(current, previous); // [] on a first run (previous === null)
|
|
10897
|
+
emitResult = { firstRun: previous === null, emittedEvents: [], snapshotId: null, errors: [] };
|
|
10898
|
+
for (const finding of findings) {
|
|
10899
|
+
const published = await publishOrgEvent({
|
|
10900
|
+
agentId,
|
|
10901
|
+
baseUrl,
|
|
10902
|
+
kind: finding.kind,
|
|
10903
|
+
scope: finding.scope,
|
|
10904
|
+
summary: finding.summary,
|
|
10905
|
+
detail: JSON.stringify(finding.detail),
|
|
10906
|
+
targetIds: finding.targetIds,
|
|
10907
|
+
});
|
|
10908
|
+
if (published.ok) {
|
|
10909
|
+
emitResult.emittedEvents.push({ ...finding, orgEventId: published.id });
|
|
10910
|
+
}
|
|
10911
|
+
else {
|
|
10912
|
+
emitResult.errors.push(`${finding.kind} (${finding.detail.metric}): ${published.error}`);
|
|
10913
|
+
}
|
|
10914
|
+
}
|
|
10915
|
+
try {
|
|
10916
|
+
emitResult.snapshotId = await storeQualitySnapshot(agentId, baseUrl, subject, current);
|
|
10917
|
+
}
|
|
10918
|
+
catch (err) {
|
|
10919
|
+
emitResult.errors.push(`snapshot store failed: ${err?.message ?? String(err)}`);
|
|
10920
|
+
}
|
|
10921
|
+
}
|
|
10922
|
+
const mode = render.resolveOutputMode(opts);
|
|
10923
|
+
if (mode === "json") {
|
|
10924
|
+
const out = { healthy, url: baseUrl, flairVersion: __pkgVersion, ...report };
|
|
10925
|
+
if (emitResult) {
|
|
10926
|
+
out.emit = { firstRun: emitResult.firstRun, snapshotId: emitResult.snapshotId, errors: emitResult.errors };
|
|
10927
|
+
out.emittedEvents = emitResult.emittedEvents.map((e) => ({
|
|
10928
|
+
kind: e.kind,
|
|
10929
|
+
scope: e.scope,
|
|
10930
|
+
summary: e.summary,
|
|
10931
|
+
detail: e.detail,
|
|
10932
|
+
targetIds: e.targetIds,
|
|
10933
|
+
orgEventId: e.orgEventId,
|
|
10934
|
+
}));
|
|
10935
|
+
}
|
|
10936
|
+
console.log(render.asJSON(out));
|
|
10937
|
+
if (!healthy)
|
|
10938
|
+
process.exit(1);
|
|
10939
|
+
return;
|
|
10940
|
+
}
|
|
10941
|
+
if (!healthy) {
|
|
10942
|
+
console.log(`Flair v${__pkgVersion} — 🔴 unreachable`);
|
|
10943
|
+
console.log(` URL: ${baseUrl}`);
|
|
10944
|
+
console.log(`\n Run: flair start or flair doctor`);
|
|
10945
|
+
process.exit(1);
|
|
10946
|
+
}
|
|
10947
|
+
const scopeLabel = agentId ? ` ${render.wrap(render.c.dim, `(agent: ${agentId})`)}` : "";
|
|
10948
|
+
console.log(`${render.wrap(render.c.bold, "Flair quality report")}${scopeLabel}`);
|
|
10949
|
+
console.log(render.kv("URL", baseUrl));
|
|
10950
|
+
// Instance health
|
|
10951
|
+
console.log(`\n${render.wrap(render.c.bold, "Instance health")}`);
|
|
10952
|
+
console.log(render.kv("Up", report.instance.up ? `${render.icons.ok} yes` : `${render.icons.error} no`));
|
|
10953
|
+
if (report.instance.migrationsClean === null) {
|
|
10954
|
+
console.log(render.kv("Migrations", `${render.icons.info} unknown ${render.wrap(render.c.dim, "(no data)")}`));
|
|
10955
|
+
}
|
|
10956
|
+
else if (report.instance.migrationsClean) {
|
|
10957
|
+
console.log(render.kv("Migrations", `${render.icons.ok} clean`));
|
|
10958
|
+
}
|
|
10959
|
+
else {
|
|
10960
|
+
console.log(render.kv("Migrations", `${render.icons.error} ${report.instance.haltedMigrations.length} halted/failed`));
|
|
10961
|
+
for (const m of report.instance.haltedMigrations) {
|
|
10962
|
+
console.log(` ${render.icons.error} ${m.id}: ${m.state}${m.reason ? ` — ${m.reason}` : ""}`);
|
|
10963
|
+
}
|
|
10964
|
+
}
|
|
10965
|
+
const embIcon = report.instance.embeddingsStatus === "ok" ? render.icons.ok :
|
|
10966
|
+
report.instance.embeddingsStatus === "degraded" ? render.icons.error :
|
|
10967
|
+
render.icons.info;
|
|
10968
|
+
console.log(render.kv("Embeddings", `${embIcon} ${report.instance.embeddingsStatus} ${render.wrap(render.c.dim, `(${report.instance.embeddingsDetail})`)}`));
|
|
10969
|
+
// Embedding coverage
|
|
10970
|
+
if (report.embeddingCoverage) {
|
|
10971
|
+
const ec = report.embeddingCoverage;
|
|
10972
|
+
console.log(`\n${render.wrap(render.c.bold, "Embedding coverage")}`);
|
|
10973
|
+
console.log(render.kv("Coverage", `${render.wrap(render.c.bold, `${ec.coveragePct}%`)} ${render.wrap(render.c.dim, `(${ec.withEmbeddings}/${ec.total} real, ${ec.hashFallback} hash-fallback)`)}`));
|
|
10974
|
+
}
|
|
10975
|
+
// Staleness
|
|
10976
|
+
if (report.staleness) {
|
|
10977
|
+
const st = report.staleness;
|
|
10978
|
+
console.log(`\n${render.wrap(render.c.bold, "Staleness")}`);
|
|
10979
|
+
console.log(render.kv("Past validTo", `${render.wrap(render.c.bold, `${st.stalePct}%`)} ${render.wrap(render.c.dim, `(${st.expired}/${st.total}, instance-wide)`)}`));
|
|
10980
|
+
}
|
|
10981
|
+
// Signal density
|
|
10982
|
+
if (report.signalDensity) {
|
|
10983
|
+
console.log(`\n${render.wrap(render.c.bold, "Signal density")} ${render.wrap(render.c.dim, "(write + citation activity — a usage pattern, not a trust signal)")}`);
|
|
10984
|
+
if (agentId && report.signalDensity.perAgent.length === 0) {
|
|
10985
|
+
console.log(` ${render.icons.info} no data for agent '${agentId}'`);
|
|
10986
|
+
}
|
|
10987
|
+
else {
|
|
10988
|
+
const showCitation = report.signalDensity.scope === "write-and-citation";
|
|
10989
|
+
const cols = [
|
|
10990
|
+
{ label: "id", key: "id" },
|
|
10991
|
+
{ label: "memories", key: "memoryCount", align: "right" },
|
|
10992
|
+
{ label: "writes_24h", key: "writes24h", align: "right" },
|
|
10993
|
+
...(showCitation
|
|
10994
|
+
? [
|
|
10995
|
+
{ label: "citations", key: "usageCount", align: "right" },
|
|
10996
|
+
{ label: "citation_rate", key: "citationRate", align: "right" },
|
|
10997
|
+
]
|
|
10998
|
+
: []),
|
|
10999
|
+
{ label: "last_write", key: "lastWriteAt", format: (v) => render.relativeTime(v) },
|
|
11000
|
+
];
|
|
11001
|
+
console.log(render.table(cols, report.signalDensity.perAgent));
|
|
11002
|
+
if (showCitation) {
|
|
11003
|
+
console.log(` ${render.wrap(render.c.dim, "citation_rate = avg citations per memory; a low rate means \"writes exploratory content that's rarely cited\", not \"noisy\"")}`);
|
|
11004
|
+
}
|
|
11005
|
+
else {
|
|
11006
|
+
console.log(` ${render.wrap(render.c.dim, "citation rate not shown — server predates per-agent usageCount in /HealthDetail (see Gaps); low write volume means \"writes exploratory content\", not \"noisy\"")}`);
|
|
11007
|
+
}
|
|
11008
|
+
}
|
|
11009
|
+
}
|
|
11010
|
+
// Quiet agents
|
|
11011
|
+
if (report.quietAgents) {
|
|
11012
|
+
console.log(`\n${render.wrap(render.c.bold, "Quiet agents")} ${render.wrap(render.c.dim, `(no write in ${report.quietAgents.thresholdDays}+ days — an ops fact, not a trust signal)`)}`);
|
|
11013
|
+
if (agentId && report.quietAgents.perAgent.length === 0) {
|
|
11014
|
+
console.log(` ${render.icons.info} no data for agent '${agentId}'`);
|
|
11015
|
+
}
|
|
11016
|
+
else {
|
|
11017
|
+
const quiet = report.quietAgents.perAgent.filter((r) => r.quiet);
|
|
11018
|
+
if (quiet.length === 0) {
|
|
11019
|
+
console.log(` ${render.icons.ok} none`);
|
|
11020
|
+
}
|
|
11021
|
+
else {
|
|
11022
|
+
for (const r of quiet) {
|
|
11023
|
+
const label = r.daysSinceLastWrite == null ? "never written" : `quiet for ${r.daysSinceLastWrite}d`;
|
|
11024
|
+
console.log(` ${render.icons.warn} ${r.id} — ${label}`);
|
|
11025
|
+
}
|
|
11026
|
+
}
|
|
11027
|
+
}
|
|
11028
|
+
}
|
|
11029
|
+
// Dedup clusters (flair-quality Slice 1c) — an ops/health signal, not a
|
|
11030
|
+
// trust judgment. Labeled with the nightly REM run that produced it, per
|
|
11031
|
+
// spec, since it's only ever as fresh as the last nightly cycle.
|
|
11032
|
+
if (report.dedupClusters) {
|
|
11033
|
+
const dc = report.dedupClusters;
|
|
11034
|
+
console.log(`\n${render.wrap(render.c.bold, "Dedup clusters")} ${render.wrap(render.c.dim, `(as of last REM run ${render.relativeTime(dc.computedAt)}, ${dc.computedAt})`)}`);
|
|
11035
|
+
console.log(render.kv("Clusters", `${render.wrap(render.c.bold, String(dc.clusterCount))} ${render.wrap(render.c.dim, `(${dc.totalMemoriesInClusters} memories, largest cluster ${dc.largestClusterSize})`)}`));
|
|
11036
|
+
console.log(` ${render.wrap(render.c.dim, "an ops signal — near-duplicate memories piling up, not a trust judgment")}`);
|
|
11037
|
+
}
|
|
11038
|
+
// Recall spot-check (flair-quality Slice 1d) — a health SPOT-CHECK, not
|
|
11039
|
+
// a benchmark or trust judgment: catches recall cratering (embeddings/
|
|
11040
|
+
// index down), not a quality grade. See QualityReport['recallSpotCheck']
|
|
11041
|
+
// doc for the full framing.
|
|
11042
|
+
if (report.recallSpotCheck) {
|
|
11043
|
+
const rc = report.recallSpotCheck;
|
|
11044
|
+
console.log(`\n${render.wrap(render.c.bold, "Recall spot-check")} ${render.wrap(render.c.dim, `(agent ${rc.agentId ?? "—"}, health signal — not a benchmark)`)}`);
|
|
11045
|
+
console.log(render.kv(`recall@${rc.k}`, `${render.wrap(render.c.bold, rc.recallAtK.toFixed(2))} ${render.wrap(render.c.dim, `(MRR ${rc.mrr.toFixed(2)}, ${rc.sampleSize} sampled)`)}`));
|
|
11046
|
+
console.log(` ${render.wrap(render.c.dim, "catches recall cratering (embeddings/index down) — a high score means recall is functioning, not that it's optimal")}`);
|
|
11047
|
+
}
|
|
11048
|
+
// Gaps
|
|
11049
|
+
if (report.gaps.length > 0) {
|
|
11050
|
+
console.log(`\n${render.wrap(render.c.bold, "Gaps")} ${render.wrap(render.c.dim, "(degraded or unavailable from existing read APIs)")}`);
|
|
11051
|
+
for (const g of report.gaps) {
|
|
11052
|
+
console.log(` ${render.icons.info} ${g.metric}: ${g.reason}`);
|
|
11053
|
+
}
|
|
11054
|
+
}
|
|
11055
|
+
// Events (flair-quality Slice 2 — only present when --emit was passed)
|
|
11056
|
+
if (emitResult) {
|
|
11057
|
+
console.log(`\n${render.wrap(render.c.bold, "Events")} ${render.wrap(render.c.dim, "(--emit: snapshot + diff against the previous quality-snapshot memory)")}`);
|
|
11058
|
+
if (emitResult.firstRun) {
|
|
11059
|
+
console.log(` ${render.icons.info} first run — no prior snapshot to diff against; stored a baseline, emitted nothing`);
|
|
11060
|
+
}
|
|
11061
|
+
else if (emitResult.emittedEvents.length === 0) {
|
|
11062
|
+
console.log(` ${render.icons.ok} no threshold crossings or regressions since the last snapshot`);
|
|
11063
|
+
}
|
|
11064
|
+
else {
|
|
11065
|
+
console.log(` ${render.wrap(render.c.bold, String(emitResult.emittedEvents.length))} event(s) emitted:`);
|
|
11066
|
+
for (const e of emitResult.emittedEvents) {
|
|
11067
|
+
const icon = e.kind === "quality.regression" ? render.icons.warn : render.icons.info;
|
|
11068
|
+
console.log(` ${icon} [${e.kind}] ${e.summary}`);
|
|
11069
|
+
}
|
|
11070
|
+
}
|
|
11071
|
+
if (emitResult.snapshotId) {
|
|
11072
|
+
console.log(` ${render.wrap(render.c.dim, `snapshot stored: ${emitResult.snapshotId}`)}`);
|
|
11073
|
+
}
|
|
11074
|
+
for (const err of emitResult.errors) {
|
|
11075
|
+
console.log(` ${render.icons.error} ${err}`);
|
|
11076
|
+
}
|
|
11077
|
+
}
|
|
11078
|
+
console.log("");
|
|
11079
|
+
});
|
|
10151
11080
|
// ─── flair session snapshot ──────────────────────────────────────────────────
|
|
10152
11081
|
// Slice 2 of FLAIR-AGENT-CONTEXT-TIERS-B. Snapshot a
|
|
10153
11082
|
// session jsonl + label metadata into a tar.gz under ~/.flair/snapshots/<agent>/sessions/.
|
|
@@ -12593,6 +13522,58 @@ workspace
|
|
|
12593
13522
|
// Memory.write(): `${agentId}-${randomUUID()}`).
|
|
12594
13523
|
const MAX_ORGEVENT_SUMMARY_LENGTH = 500;
|
|
12595
13524
|
const MAX_ORGEVENT_DETAIL_LENGTH = 8000;
|
|
13525
|
+
/**
|
|
13526
|
+
* The ONE write shape behind `flair orgevent` — signed PUT /OrgEvent/{id} —
|
|
13527
|
+
* extracted so `flair quality --emit` (flair-quality Slice 2) reuses the
|
|
13528
|
+
* exact same call, rather than hand-rolling a second one. Everything below
|
|
13529
|
+
* mirrors what the `orgevent` command's action used to do inline: id
|
|
13530
|
+
* convention (`${agentId}-${randomUUID()}`, matching flair-client's
|
|
13531
|
+
* Memory.write()), the same Ed25519 signing (buildEd25519Auth), the same
|
|
13532
|
+
* length guards (MAX_ORGEVENT_SUMMARY_LENGTH/MAX_ORGEVENT_DETAIL_LENGTH) so
|
|
13533
|
+
* every caller — CLI flag or programmatic — gets them, and the same
|
|
13534
|
+
* authorId self-declaration OrgEvent.put() verifies server-side against the
|
|
13535
|
+
* signature (see the module doc above `orgevent`). Never throws — every
|
|
13536
|
+
* failure mode (missing key, oversized field, non-2xx response) returns
|
|
13537
|
+
* `{ ok: false, error }` for the caller to surface however fits its own UX.
|
|
13538
|
+
*/
|
|
13539
|
+
async function publishOrgEvent(params) {
|
|
13540
|
+
if (params.summary.length > MAX_ORGEVENT_SUMMARY_LENGTH) {
|
|
13541
|
+
return { ok: false, error: `summary exceeds ${MAX_ORGEVENT_SUMMARY_LENGTH} character limit (got ${params.summary.length})` };
|
|
13542
|
+
}
|
|
13543
|
+
if (params.detail && params.detail.length > MAX_ORGEVENT_DETAIL_LENGTH) {
|
|
13544
|
+
return { ok: false, error: `detail exceeds ${MAX_ORGEVENT_DETAIL_LENGTH} character limit (got ${params.detail.length})` };
|
|
13545
|
+
}
|
|
13546
|
+
const keyPath = resolveKeyPath(params.agentId);
|
|
13547
|
+
if (!keyPath) {
|
|
13548
|
+
return { ok: false, error: `private key not found for agent '${params.agentId}'. Check ~/.flair/keys/ or set FLAIR_KEY_DIR.` };
|
|
13549
|
+
}
|
|
13550
|
+
const id = `${params.agentId}-${randomUUID()}`;
|
|
13551
|
+
const auth = buildEd25519Auth(params.agentId, "PUT", `/OrgEvent/${id}`, keyPath);
|
|
13552
|
+
const body = {
|
|
13553
|
+
id,
|
|
13554
|
+
authorId: params.agentId,
|
|
13555
|
+
kind: params.kind,
|
|
13556
|
+
summary: params.summary,
|
|
13557
|
+
createdAt: new Date().toISOString(),
|
|
13558
|
+
};
|
|
13559
|
+
if (params.detail)
|
|
13560
|
+
body.detail = params.detail;
|
|
13561
|
+
if (params.scope)
|
|
13562
|
+
body.scope = params.scope;
|
|
13563
|
+
if (params.targetIds && params.targetIds.length > 0)
|
|
13564
|
+
body.targetIds = params.targetIds;
|
|
13565
|
+
const res = await fetch(`${params.baseUrl}/OrgEvent/${id}`, {
|
|
13566
|
+
method: "PUT",
|
|
13567
|
+
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
13568
|
+
body: JSON.stringify(body),
|
|
13569
|
+
});
|
|
13570
|
+
if (!res.ok) {
|
|
13571
|
+
const text = await res.text().catch(() => "");
|
|
13572
|
+
return { ok: false, error: `PUT /OrgEvent/${id} failed (${res.status}): ${text}` };
|
|
13573
|
+
}
|
|
13574
|
+
const data = await res.json().catch(() => null);
|
|
13575
|
+
return { ok: true, id: data?.id ?? id };
|
|
13576
|
+
}
|
|
12596
13577
|
program
|
|
12597
13578
|
.command("orgevent")
|
|
12598
13579
|
.description("Publish an org-wide coordination event attributed to your agent (PUT /OrgEvent/{id})")
|
|
@@ -12610,59 +13591,26 @@ program
|
|
|
12610
13591
|
console.error("Error: agent ID required. Pass --agent <id> or set FLAIR_AGENT_ID environment variable.");
|
|
12611
13592
|
process.exit(1);
|
|
12612
13593
|
}
|
|
12613
|
-
if (opts.summary && String(opts.summary).length > MAX_ORGEVENT_SUMMARY_LENGTH) {
|
|
12614
|
-
console.error(`Error: --summary exceeds ${MAX_ORGEVENT_SUMMARY_LENGTH} character limit (got ${String(opts.summary).length}).`);
|
|
12615
|
-
process.exit(1);
|
|
12616
|
-
}
|
|
12617
|
-
if (opts.detail && String(opts.detail).length > MAX_ORGEVENT_DETAIL_LENGTH) {
|
|
12618
|
-
console.error(`Error: --detail exceeds ${MAX_ORGEVENT_DETAIL_LENGTH} character limit (got ${String(opts.detail).length}).`);
|
|
12619
|
-
process.exit(1);
|
|
12620
|
-
}
|
|
12621
|
-
const keyPath = resolveKeyPath(agentId);
|
|
12622
|
-
if (!keyPath) {
|
|
12623
|
-
console.error(`Error: private key not found for agent '${agentId}'. Check ~/.flair/keys/ or set FLAIR_KEY_DIR.`);
|
|
12624
|
-
process.exit(1);
|
|
12625
|
-
}
|
|
12626
13594
|
// orgevent reuses --target for recipients, so the remote-URL override is
|
|
12627
13595
|
// --target-url here (env FLAIR_TARGET still honored via resolveBaseUrl).
|
|
12628
13596
|
const baseUrl = resolveBaseUrl({ target: opts.targetUrl, port: opts.port }).replace(/\/$/, "");
|
|
12629
|
-
|
|
12630
|
-
|
|
12631
|
-
|
|
12632
|
-
|
|
12633
|
-
const id = `${agentId}-${randomUUID()}`;
|
|
12634
|
-
const auth = buildEd25519Auth(agentId, "PUT", `/OrgEvent/${id}`, keyPath);
|
|
12635
|
-
// authorId IS included in the body now — OrgEvent.put() (unlike post())
|
|
12636
|
-
// does not auto-attribute from the signature, it 403s any mismatch. This
|
|
12637
|
-
// is a self-declaration the server verifies against the signature, not a
|
|
12638
|
-
// forgeable claim.
|
|
12639
|
-
const body = {
|
|
12640
|
-
id,
|
|
12641
|
-
authorId: agentId,
|
|
13597
|
+
const targetIds = Array.isArray(opts.target) && opts.target.length > 0 ? opts.target : undefined;
|
|
13598
|
+
const result = await publishOrgEvent({
|
|
13599
|
+
agentId,
|
|
13600
|
+
baseUrl,
|
|
12642
13601
|
kind: opts.kind,
|
|
12643
13602
|
summary: opts.summary,
|
|
12644
|
-
|
|
12645
|
-
|
|
12646
|
-
|
|
12647
|
-
body.detail = opts.detail;
|
|
12648
|
-
if (opts.scope)
|
|
12649
|
-
body.scope = opts.scope;
|
|
12650
|
-
if (Array.isArray(opts.target) && opts.target.length > 0)
|
|
12651
|
-
body.targetIds = opts.target;
|
|
12652
|
-
const res = await fetch(`${baseUrl}/OrgEvent/${id}`, {
|
|
12653
|
-
method: "PUT",
|
|
12654
|
-
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
12655
|
-
body: JSON.stringify(body),
|
|
13603
|
+
detail: opts.detail,
|
|
13604
|
+
scope: opts.scope,
|
|
13605
|
+
targetIds,
|
|
12656
13606
|
});
|
|
12657
|
-
if (!
|
|
12658
|
-
|
|
12659
|
-
console.error(`Error: PUT /OrgEvent/${id} failed (${res.status}): ${text}`);
|
|
13607
|
+
if (!result.ok) {
|
|
13608
|
+
console.error(`Error: ${result.error}`);
|
|
12660
13609
|
process.exit(1);
|
|
12661
13610
|
}
|
|
12662
|
-
const
|
|
12663
|
-
const targets = Array.isArray(opts.target) && opts.target.length > 0 ? ` → ${opts.target.join(", ")}` : "";
|
|
13611
|
+
const targets = targetIds ? ` → ${targetIds.join(", ")}` : "";
|
|
12664
13612
|
console.log(`✓ OrgEvent published as '${agentId}': kind=${opts.kind}${targets}`);
|
|
12665
|
-
console.log(` id: ${
|
|
13613
|
+
console.log(` id: ${result.id}`);
|
|
12666
13614
|
});
|
|
12667
13615
|
// ─── flair attention ─────────────────────────────────────────────────────────
|
|
12668
13616
|
//
|