@tpsdev-ai/flair 0.25.4 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/cli.js +998 -50
- package/dist/doctor-client.js +47 -0
- package/dist/rem/runner.js +60 -3
- package/dist/resources/MemoryDedupStats.js +161 -0
- package/dist/resources/dedup-cluster.js +145 -0
- package/dist/resources/health.js +33 -1
- package/dist/resources/mcp-oauth.js +4 -2
- package/docs/assets/flair-cross-orchestrator.cast +33 -0
- package/docs/assets/flair-cross-orchestrator.gif +0 -0
- package/docs/assets/flair-demo.cast +18 -0
- package/docs/assets/flair-demo.gif +0 -0
- package/docs/auth.md +178 -0
- package/docs/bridges.md +291 -0
- package/docs/claude-code.md +202 -0
- package/docs/deployment.md +204 -0
- package/docs/entity-vocabulary.md +109 -0
- package/docs/federation.md +179 -0
- package/docs/integrations.md +197 -0
- package/docs/mcp-clients.md +240 -0
- package/docs/n8n-management.md +142 -0
- package/docs/n8n.md +122 -0
- package/docs/notes/adk-spike-findings-2026-05-05.md +331 -0
- package/docs/notes/mcp-agent-auth-consumer.md +229 -0
- package/docs/notes/mcp-oauth-model2.md +132 -0
- package/docs/notes/rem-ux.md +39 -0
- package/docs/openclaw.md +131 -0
- package/docs/quickstart.md +153 -0
- package/docs/releasing.md +173 -0
- package/docs/rem.md +60 -0
- package/docs/rerank-provisioning.md +67 -0
- package/docs/secrets-and-keys.md +173 -0
- package/docs/spoke-bringup.md +303 -0
- package/docs/supply-chain-policy.md +156 -0
- package/docs/system-requirements.md +42 -0
- package/docs/the-team.md +129 -0
- package/docs/troubleshooting.md +187 -0
- package/docs/upgrade.md +416 -0
- package/package.json +2 -1
package/dist/doctor-client.js
CHANGED
|
@@ -373,6 +373,53 @@ export function planAgentIterations(keyAgentIds, agentFlag) {
|
|
|
373
373
|
return [agentFlag];
|
|
374
374
|
return [...keyAgentIds].sort();
|
|
375
375
|
}
|
|
376
|
+
// ── `doctor --fix` agent-id inference (flair#802b) ─────────────────────────
|
|
377
|
+
//
|
|
378
|
+
// `doctor` suggested `flair doctor --fix` to auto-wire an unconfigured MCP
|
|
379
|
+
// client, but running that exact command failed — "no agent id known — pass
|
|
380
|
+
// --agent <id>" — whenever the client had never been wired before (so there
|
|
381
|
+
// was no existing block to read an agent id from) and neither --agent nor
|
|
382
|
+
// FLAIR_AGENT_ID was set. The suggested fix didn't work as suggested. Two
|
|
383
|
+
// pure decisions fix that without adding any new network/crypto surface:
|
|
384
|
+
//
|
|
385
|
+
// 1. inferSoleAgentId — when exactly one locally-keyed agent exists (the
|
|
386
|
+
// same keyAgentIds pool planAgentIterations already draws from, i.e.
|
|
387
|
+
// doctor's own "Keys found" enumeration), --fix can use it without
|
|
388
|
+
// being told: there's no other candidate it could mean. Two or more
|
|
389
|
+
// keys, or zero, are genuinely ambiguous/unanswerable and still require
|
|
390
|
+
// an explicit --agent (or registering one first) — this never guesses
|
|
391
|
+
// in either of those cases.
|
|
392
|
+
// 2. fixCommandAgentHint — the *printed suggestion* (before --fix ever
|
|
393
|
+
// runs) splices in a concrete `--agent <id>` so the command a user
|
|
394
|
+
// copy-pastes actually works, using the first (sorted) known key id as
|
|
395
|
+
// the example. Only relevant when the id isn't already resolvable some
|
|
396
|
+
// other way (explicit --agent, FLAIR_AGENT_ID, or an id read off an
|
|
397
|
+
// already-wired client) — the caller checks that before calling this.
|
|
398
|
+
/**
|
|
399
|
+
* The one case `doctor --fix` can safely infer an agent id without being
|
|
400
|
+
* told: exactly one locally-keyed agent. Zero keys (nothing to infer) or two
|
|
401
|
+
* or more (genuinely ambiguous — which one?) both return undefined; the
|
|
402
|
+
* caller must fall back to an explicit error telling the user what to do
|
|
403
|
+
* (register one, or pass --agent).
|
|
404
|
+
*/
|
|
405
|
+
export function inferSoleAgentId(keyAgentIds) {
|
|
406
|
+
return keyAgentIds.length === 1 ? keyAgentIds[0] : undefined;
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Build the ` --agent <id>` fragment to splice into a suggested
|
|
410
|
+
* `flair doctor --fix ...` command so the printed suggestion is actually
|
|
411
|
+
* copy-pasteable rather than guaranteed to fail the moment nothing else
|
|
412
|
+
* (explicit --agent, FLAIR_AGENT_ID, an already-wired client's agent id) can
|
|
413
|
+
* supply one. Uses the first (sorted) known local key id as a concrete
|
|
414
|
+
* example. Returns "" when no agent id is known at all — the caller should
|
|
415
|
+
* tell the user to register one first rather than print a `--fix` suggestion
|
|
416
|
+
* that has nothing to work with either way.
|
|
417
|
+
*/
|
|
418
|
+
export function fixCommandAgentHint(keyAgentIds) {
|
|
419
|
+
if (keyAgentIds.length === 0)
|
|
420
|
+
return "";
|
|
421
|
+
return ` --agent ${[...keyAgentIds].sort()[0]}`;
|
|
422
|
+
}
|
|
376
423
|
/**
|
|
377
424
|
* Render decision for one agent's registration-gate outcome, ahead of a
|
|
378
425
|
* verified-read section (Fleet presence / Migrations). Returns null when the
|
package/dist/rem/runner.js
CHANGED
|
@@ -8,7 +8,13 @@
|
|
|
8
8
|
* 4. Trust-tier filter on input memories — permanently deferred (see below).
|
|
9
9
|
* 5. Distillation — call /ReflectMemories with execute:true, persist staged
|
|
10
10
|
* candidate ids to the audit row.
|
|
11
|
-
* 6.
|
|
11
|
+
* 6. Instance-wide dedup-cluster stat — call /MemoryDedupStats (flair-
|
|
12
|
+
* quality Slice 1c). NOT part of FLAIR-NIGHTLY-REM's original per-agent
|
|
13
|
+
* § 4 list — added because a near-duplicate cluster count is inherently
|
|
14
|
+
* instance-wide (dupes across agents matter), so it runs once per cycle
|
|
15
|
+
* after the per-agent passes above, rather than being scoped to
|
|
16
|
+
* opts.agentId like everything else in this file.
|
|
17
|
+
* 7. Append a row to ~/.flair/logs/rem-nightly.jsonl.
|
|
12
18
|
*
|
|
13
19
|
* Status today:
|
|
14
20
|
* - Steps 1, 2 shipped in slice-1 PR-1 (#414).
|
|
@@ -23,7 +29,14 @@
|
|
|
23
29
|
* - Step 5 (distillation) ships in this PR: calls `/ReflectMemories` with
|
|
24
30
|
* `execute: true` after maintenance succeeds. `dryRun` skips the call
|
|
25
31
|
* entirely (staging rows + spending model tokens are side effects).
|
|
26
|
-
* - Step 6
|
|
32
|
+
* - Step 6 (dedup-cluster stat, flair-quality Slice 1c): calls
|
|
33
|
+
* `/MemoryDedupStats` after distillation. `dryRun` skips it (persisting
|
|
34
|
+
* the stat file is a side effect); failure (e.g. the caller isn't admin
|
|
35
|
+
* — the resource is admin-gated) is recorded in `errors` and does not
|
|
36
|
+
* fail the cycle. See resources/MemoryDedupStats.ts for the server-side
|
|
37
|
+
* computation and resources/dedup-cluster.ts for the stat's canonical
|
|
38
|
+
* storage location (NOT this log — see those files' module docs for why).
|
|
39
|
+
* - Step 7 shipped in slice-1 PR-1 (was step 6 before this PR).
|
|
27
40
|
*
|
|
28
41
|
* The audit row's `slice` field tells readers which steps populated which
|
|
29
42
|
* counts: `slice: "1"` rows have `archived`/`expired` undefined; `slice:
|
|
@@ -268,7 +281,50 @@ export async function runNightlyCycle(opts) {
|
|
|
268
281
|
errors.push(`distillation: ${describeApiError(err?.message ?? err)}`);
|
|
269
282
|
}
|
|
270
283
|
}
|
|
271
|
-
// Step 6:
|
|
284
|
+
// Step 6 (flair-quality Slice 1c): instance-wide dedup-cluster stat.
|
|
285
|
+
// Distinct from every step above — NOT scoped to opts.agentId. Runs ONCE
|
|
286
|
+
// per cycle, after the per-agent passes (spec: "a new instance-wide step
|
|
287
|
+
// in the REM nightly runner... runs once per cycle, after the per-agent
|
|
288
|
+
// passes"). Skipped in dry-run for the same reason distillation is:
|
|
289
|
+
// persisting the stat file is a side effect. Non-fatal on failure — e.g.
|
|
290
|
+
// POST /MemoryDedupStats is admin-gated (resources/MemoryDedupStats.ts),
|
|
291
|
+
// so a non-admin agent's nightly cycle records a `dedup:` error here and
|
|
292
|
+
// otherwise completes normally; maintenance + distillation already stand.
|
|
293
|
+
//
|
|
294
|
+
// NOTE (flagged for review): if MULTIPLE agents on the same instance each
|
|
295
|
+
// run their own nightly cycle, each cycle triggers this same instance-wide
|
|
296
|
+
// sweep independently — the stat is recomputed N times a night rather than
|
|
297
|
+
// once. No cross-process guard was added (kept the smallest surface); the
|
|
298
|
+
// recomputation is idempotent (same inputs → same aggregate, just wasted
|
|
299
|
+
// work), not incorrect.
|
|
300
|
+
let dedup;
|
|
301
|
+
if (!opts.dryRun) {
|
|
302
|
+
try {
|
|
303
|
+
const dedupRaw = await opts.apiCall("POST", "/MemoryDedupStats", {});
|
|
304
|
+
const obj = (dedupRaw && typeof dedupRaw === "object") ? dedupRaw : {};
|
|
305
|
+
if (obj.error) {
|
|
306
|
+
errors.push(`dedup: ${describeApiError(obj.error)}`);
|
|
307
|
+
}
|
|
308
|
+
else if (typeof obj.clusterCount === "number" &&
|
|
309
|
+
typeof obj.largestClusterSize === "number" &&
|
|
310
|
+
typeof obj.totalMemoriesInClusters === "number" &&
|
|
311
|
+
typeof obj.computedAt === "string") {
|
|
312
|
+
dedup = {
|
|
313
|
+
clusterCount: obj.clusterCount,
|
|
314
|
+
largestClusterSize: obj.largestClusterSize,
|
|
315
|
+
totalMemoriesInClusters: obj.totalMemoriesInClusters,
|
|
316
|
+
computedAt: obj.computedAt,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
else {
|
|
320
|
+
errors.push("dedup: unexpected /MemoryDedupStats response shape");
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
catch (err) {
|
|
324
|
+
errors.push(`dedup: ${describeApiError(err?.message ?? err)}`);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
// Step 7: log
|
|
272
328
|
const row = {
|
|
273
329
|
...baseRow,
|
|
274
330
|
slice: sliceLabel,
|
|
@@ -281,6 +337,7 @@ export async function runNightlyCycle(opts) {
|
|
|
281
337
|
archived,
|
|
282
338
|
expired,
|
|
283
339
|
candidates,
|
|
340
|
+
dedup,
|
|
284
341
|
durationMs: Date.now() - startedMs,
|
|
285
342
|
errors,
|
|
286
343
|
// `consolidated` remains undefined — this runner has no consolidation
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MemoryDedupStats.ts — instance-wide near-duplicate CLUSTER count.
|
|
3
|
+
*
|
|
4
|
+
* POST /MemoryDedupStats — admin-only. Triggered once per REM nightly cycle
|
|
5
|
+
* (src/rem/runner.ts, after the per-agent maintenance/distillation passes —
|
|
6
|
+
* see that file's module doc for why this step is instance-wide rather than
|
|
7
|
+
* folded into the per-agent runner loop).
|
|
8
|
+
*
|
|
9
|
+
* flair-quality Slice 1c (ops/proposals/flair-quality-slice1c-dedup-spec.md).
|
|
10
|
+
* K&S split resolved to "Option C, server-side": Sherlock's hard security
|
|
11
|
+
* objection to a CLI-side computation was that embeddings are the most
|
|
12
|
+
* sensitive data in the system and must never leave the server; Kern's
|
|
13
|
+
* arch counter was that this doesn't fit REM's PER-AGENT pass (which also
|
|
14
|
+
* strips embeddings before handing memories to the model). Resolution:
|
|
15
|
+
* a SEPARATE server-side nightly job — this resource — that:
|
|
16
|
+
* 1. Reads embeddings DIRECTLY off `databases.flair.Memory` (same access
|
|
17
|
+
* HealthDetail already has — resources/health.ts's `db.flair.Memory.
|
|
18
|
+
* search({})` reads full records instance-wide for its own stats).
|
|
19
|
+
* 2. Uses them ONLY as HNSW query vectors, via `retrieveCandidates()`
|
|
20
|
+
* (resources/semantic-retrieval-core.ts) called BARE — the same
|
|
21
|
+
* in-process, no-HTTP pattern resources/MemoryBootstrap.ts already
|
|
22
|
+
* uses to reuse SemanticSearch's retrieval core without going over the
|
|
23
|
+
* wire. The embedding vector itself is never serialized into an HTTP
|
|
24
|
+
* response or returned to a caller — it lives only in this process's
|
|
25
|
+
* memory for the span of one ANN query, then is discarded.
|
|
26
|
+
* 3. Persists ONLY the aggregate `{ clusterCount, largestClusterSize,
|
|
27
|
+
* totalMemoriesInClusters, computedAt }` to REM_DEDUP_STATS_PATH
|
|
28
|
+
* (resources/dedup-cluster.ts) — never per-memory cluster membership
|
|
29
|
+
* (Sherlock: that would itself be a disclosure surface), never on
|
|
30
|
+
* Memory rows (would pollute the authority/attribution path — #735
|
|
31
|
+
* zero-authority spine).
|
|
32
|
+
*
|
|
33
|
+
* Why the stat file lives on the SERVER's own filesystem (not written by
|
|
34
|
+
* the CLI-side runner, unlike the snapshot/audit-log artifacts in
|
|
35
|
+
* src/rem/snapshot.ts and src/rem/runner.ts): the runner's `apiCall` talks
|
|
36
|
+
* to `flairUrl`, which is not guaranteed to be the same host the CLI runs
|
|
37
|
+
* on (a remote/federated Flair is a real deployment shape — see
|
|
38
|
+
* src/rem/scheduler.ts's `FLAIR_URL` substitution). resources/health.ts's
|
|
39
|
+
* HealthDetail (Part 3 of this slice) is a Resource running INSIDE the same
|
|
40
|
+
* Harper process as this one, so writing here and reading there are
|
|
41
|
+
* guaranteed to be the same file regardless of where the triggering CLI
|
|
42
|
+
* lives. The runner's own audit-log row (src/rem/runner.ts) gets a COPY of
|
|
43
|
+
* the result for convenience/history, but this file is the canonical source
|
|
44
|
+
* `/HealthDetail` reads from.
|
|
45
|
+
*
|
|
46
|
+
* Auth: allowAdmin, matching the codebase's existing convention for
|
|
47
|
+
* INSTANCE-WIDE (not self-scoped) maintenance jobs — see MemoryReindex.ts
|
|
48
|
+
* (also admin-only, also a fleet-wide sweep) and
|
|
49
|
+
* test/unit/resource-allow.test.ts's structural ADMIN_ONLY list, which this
|
|
50
|
+
* resource is added to. Per-agent nightly steps (/MemoryMaintenance,
|
|
51
|
+
* /ReflectMemories) use allowVerified because they're scoped to the calling
|
|
52
|
+
* agent's own memories; this one is never scoped that way — it reads every
|
|
53
|
+
* agent's embeddings — so it follows the admin-gated precedent instead.
|
|
54
|
+
*/
|
|
55
|
+
import { Resource, databases } from "@harperfast/harper";
|
|
56
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
57
|
+
import { dirname } from "node:path";
|
|
58
|
+
import { allowAdmin } from "./agent-auth.js";
|
|
59
|
+
import { retrieveCandidates } from "./semantic-retrieval-core.js";
|
|
60
|
+
import { DEDUP_COSINE_THRESHOLD_DEFAULT } from "./dedup.js";
|
|
61
|
+
import { countDedupClusters, DEDUP_CLUSTER_ANN_K_DEFAULT, DEDUP_CLUSTER_MAX_MEMORIES_DEFAULT, REM_DEDUP_STATS_PATH, } from "./dedup-cluster.js";
|
|
62
|
+
const NOT_ARCHIVED = [{ attribute: "archived", comparator: "not_equal", value: true }];
|
|
63
|
+
function persistStat(stat, pathOverride) {
|
|
64
|
+
const path = pathOverride ?? REM_DEDUP_STATS_PATH;
|
|
65
|
+
const dir = dirname(path);
|
|
66
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
67
|
+
writeFileSync(path, JSON.stringify(stat, null, 2) + "\n", { mode: 0o600 });
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The DB-touching sweep, factored out of post() so it takes an explicit
|
|
71
|
+
* `ctx` and `nowOverride`/`pathOverride` rather than reaching for Resource
|
|
72
|
+
* internals — kept close to post() (not exported broadly) since it needs a
|
|
73
|
+
* live Harper `databases` handle and isn't unit-testable without one; the
|
|
74
|
+
* PURE part (countDedupClusters) is what's unit-tested, in
|
|
75
|
+
* dedup-cluster.ts's own test file.
|
|
76
|
+
*/
|
|
77
|
+
async function computeAndPersist(ctx, opts) {
|
|
78
|
+
const annK = opts?.annK ?? DEDUP_CLUSTER_ANN_K_DEFAULT;
|
|
79
|
+
const maxMemories = opts?.maxMemories ?? DEDUP_CLUSTER_MAX_MEMORIES_DEFAULT;
|
|
80
|
+
// ── 1. Load the sweep set: every non-archived memory's id + embedding,
|
|
81
|
+
// instance-wide (no agentId condition — near-duplicates ACROSS agents are
|
|
82
|
+
// exactly what this stat is for). Bounded to the most-recently-created
|
|
83
|
+
// maxMemories when the instance exceeds the safety cap (see
|
|
84
|
+
// dedup-cluster.ts's doc on why this is a defensive bound, not sampling).
|
|
85
|
+
const all = [];
|
|
86
|
+
for await (const record of databases.flair.Memory.search({
|
|
87
|
+
conditions: NOT_ARCHIVED,
|
|
88
|
+
select: ["id", "embedding", "createdAt"],
|
|
89
|
+
})) {
|
|
90
|
+
if (!record?.id || !Array.isArray(record.embedding) || record.embedding.length === 0)
|
|
91
|
+
continue;
|
|
92
|
+
all.push({ id: record.id, embedding: record.embedding, createdAt: record.createdAt });
|
|
93
|
+
}
|
|
94
|
+
let sweepSet = all;
|
|
95
|
+
if (all.length > maxMemories) {
|
|
96
|
+
sweepSet = [...all]
|
|
97
|
+
.sort((a, b) => (b.createdAt ?? "").localeCompare(a.createdAt ?? ""))
|
|
98
|
+
.slice(0, maxMemories);
|
|
99
|
+
}
|
|
100
|
+
// ── 2. Bounded-k ANN query per memory, via the SAME retrieval core
|
|
101
|
+
// SemanticSearch/MemoryBootstrap use, called bare (in-process, no HTTP —
|
|
102
|
+
// see module doc). Each memory's OWN embedding is the query vector; the
|
|
103
|
+
// vector never leaves this function (select: ["id"] — no embedding field
|
|
104
|
+
// comes back out, and nothing here is returned to an HTTP caller).
|
|
105
|
+
//
|
|
106
|
+
// limit: annK + 1 — a memory's own stored embedding is always its own
|
|
107
|
+
// closest match (cosine 1.0), so the unfiltered result set always
|
|
108
|
+
// contains the memory itself. Fetching one extra slot and filtering self
|
|
109
|
+
// out below (n.id === memory.id) keeps the REAL candidate-neighbor count
|
|
110
|
+
// at the documented annK, rather than silently examining annK-1.
|
|
111
|
+
const edges = [];
|
|
112
|
+
for (const memory of sweepSet) {
|
|
113
|
+
let neighbors;
|
|
114
|
+
try {
|
|
115
|
+
neighbors = await retrieveCandidates({
|
|
116
|
+
queryEmbedding: memory.embedding,
|
|
117
|
+
conditions: NOT_ARCHIVED,
|
|
118
|
+
limit: annK + 1,
|
|
119
|
+
hybrid: false,
|
|
120
|
+
scoring: "raw",
|
|
121
|
+
withSemSimilarity: true,
|
|
122
|
+
select: ["id"],
|
|
123
|
+
ctx,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
// A single memory's ANN query failing (embedding engine hiccup, etc.)
|
|
128
|
+
// must not abort the whole sweep — skip it, the rest still counts.
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
for (const n of neighbors) {
|
|
132
|
+
if (!n?.id || n.id === memory.id)
|
|
133
|
+
continue;
|
|
134
|
+
const cosine = typeof n._semSimilarity === "number" ? n._semSimilarity : 0;
|
|
135
|
+
if (cosine >= DEDUP_COSINE_THRESHOLD_DEFAULT) {
|
|
136
|
+
edges.push({ a: memory.id, b: n.id });
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
// ── 3. Reduce edges → connected components (pure, unit-tested elsewhere).
|
|
141
|
+
const clusters = countDedupClusters(edges);
|
|
142
|
+
const computedAt = (opts?.nowOverride ?? new Date()).toISOString();
|
|
143
|
+
const result = { ...clusters, computedAt };
|
|
144
|
+
// ── 4. Persist server-side ONLY the aggregate (never per-memory edges).
|
|
145
|
+
persistStat(result, opts?.pathOverride);
|
|
146
|
+
return result;
|
|
147
|
+
}
|
|
148
|
+
export class MemoryDedupStats extends Resource {
|
|
149
|
+
async allowCreate() {
|
|
150
|
+
return allowAdmin(this.getContext?.());
|
|
151
|
+
}
|
|
152
|
+
async post(_data) {
|
|
153
|
+
const ctx = this.getContext?.();
|
|
154
|
+
try {
|
|
155
|
+
return await computeAndPersist(ctx);
|
|
156
|
+
}
|
|
157
|
+
catch (err) {
|
|
158
|
+
return new Response(JSON.stringify({ error: err?.message ?? String(err) }), { status: 500, headers: { "content-type": "application/json" } });
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// ─── Instance-wide dedup-cluster counting primitives (Harper-free) ──────────
|
|
2
|
+
// flair-quality Slice 1c (ops/proposals/flair-quality-slice1c-dedup-spec.md,
|
|
3
|
+
// K&S-resolved to "Option C, server-side"). Same rationale as ./dedup.ts and
|
|
4
|
+
// ./bm25.ts: the graph algorithm here is pure math over an edge list — no
|
|
5
|
+
// Harper import, no embeddings, no DB — so it's unit-testable directly
|
|
6
|
+
// against the SHIPPED clustering logic without a live Harper.
|
|
7
|
+
//
|
|
8
|
+
// ── THE INVARIANT ────────────────────────────────────────────────────────────
|
|
9
|
+
// This module never sees embeddings and never sees memory content. It
|
|
10
|
+
// consumes a list of {a, b} edges (memory-id pairs already determined to be
|
|
11
|
+
// near-duplicates by the DB-touching sweep in resources/MemoryDedupStats.ts)
|
|
12
|
+
// and reduces them to a small aggregate: how many CONNECTED COMPONENTS
|
|
13
|
+
// (clusters) exist, how big the largest one is, and how many memories fall
|
|
14
|
+
// into any cluster. Per Kern's design note (K&S round), a cluster of 5
|
|
15
|
+
// memories is 1 cluster, not 10 pairwise edges — this module is what makes
|
|
16
|
+
// that true (union-find over the edge list, not a raw edge count).
|
|
17
|
+
//
|
|
18
|
+
// No per-memory membership is retained past the union-find pass — the
|
|
19
|
+
// public output is the aggregate ONLY (Sherlock's disclosure-surface
|
|
20
|
+
// constraint: no per-memory cluster-membership data leaves this module).
|
|
21
|
+
import { resolve } from "node:path";
|
|
22
|
+
import { homedir } from "node:os";
|
|
23
|
+
/** Re-exported home for the single-source-of-truth cosine threshold this
|
|
24
|
+
* module's caller (MemoryDedupStats.ts) gates edges on — see ./dedup.ts's
|
|
25
|
+
* DEDUP_COSINE_THRESHOLD_DEFAULT (0.95). Not redefined here; this module
|
|
26
|
+
* only counts edges, it never decides what counts as "near-duplicate". */
|
|
27
|
+
export { DEDUP_COSINE_THRESHOLD_DEFAULT } from "./dedup.js";
|
|
28
|
+
/** Bounded-k ANN fan-out per memory (Kern's efficiency framing: "bounded-k
|
|
29
|
+
* ANN query per memory... O(n·k·log n)"). 10 is generous headroom for a
|
|
30
|
+
* cosine>=0.95 near-duplicate gate — a true near-duplicate is overwhelmingly
|
|
31
|
+
* likely to be its counterpart's #1 or #2 nearest neighbor, so even a large
|
|
32
|
+
* (>10-member) cluster stays fully connected as long as each member finds
|
|
33
|
+
* AT LEAST ONE clustermate within its own top-k (transitivity does the
|
|
34
|
+
* rest via union-find). Exported so the sweep and its tests share one
|
|
35
|
+
* tunable instead of a duplicated literal. */
|
|
36
|
+
export const DEDUP_CLUSTER_ANN_K_DEFAULT = 10;
|
|
37
|
+
/** Defensive cap on how many (non-archived) memories the nightly sweep will
|
|
38
|
+
* walk in one cycle — protects wall-clock time on a very large instance.
|
|
39
|
+
* This is a SAFETY bound, not a routine sampling strategy: real-world
|
|
40
|
+
* self-hosted instances are expected to sit well under this for the
|
|
41
|
+
* foreseeable future, and the brief explicitly asked to prefer the full ANN
|
|
42
|
+
* sweep over sampling. When the instance exceeds this, the sweep covers the
|
|
43
|
+
* most-recently-created N memories (deterministic, documented) rather than
|
|
44
|
+
* a random sample. */
|
|
45
|
+
export const DEDUP_CLUSTER_MAX_MEMORIES_DEFAULT = 20_000;
|
|
46
|
+
/** Server-side stat artifact home (Slice 1c Part 2 — smallest-surface
|
|
47
|
+
* storage choice). A single small JSON file under the SERVER's own
|
|
48
|
+
* `~/.flair/`, written by MemoryDedupStats.ts's post() and read by
|
|
49
|
+
* health.ts's HealthDetail — both run INSIDE the same Harper process, so
|
|
50
|
+
* this is always the same file regardless of where the CLI/runner that
|
|
51
|
+
* TRIGGERED the sweep is running (see MemoryDedupStats.ts's module doc for
|
|
52
|
+
* why that distinction matters for remote/federated deployments). Mirrors
|
|
53
|
+
* the `REM_PAUSE_FLAG` "small well-known file directly under ~/.flair/"
|
|
54
|
+
* precedent in src/rem/runner.ts, rather than nesting under logs/ (this is
|
|
55
|
+
* a single overwritten snapshot, not an append-only log). */
|
|
56
|
+
export const REM_DEDUP_STATS_PATH = resolve(homedir(), ".flair", "rem-dedup-stats.json");
|
|
57
|
+
/** Minimal union-find (disjoint-set) with path compression + union-by-rank.
|
|
58
|
+
* Nodes are created lazily — a memory id only enters the structure when it
|
|
59
|
+
* appears in at least one edge, so a memory with zero near-duplicate edges
|
|
60
|
+
* (the common case) never shows up in the output at all. */
|
|
61
|
+
class UnionFind {
|
|
62
|
+
parent = new Map();
|
|
63
|
+
rank = new Map();
|
|
64
|
+
ensure(x) {
|
|
65
|
+
if (!this.parent.has(x)) {
|
|
66
|
+
this.parent.set(x, x);
|
|
67
|
+
this.rank.set(x, 0);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
find(x) {
|
|
71
|
+
this.ensure(x);
|
|
72
|
+
let root = x;
|
|
73
|
+
while (this.parent.get(root) !== root) {
|
|
74
|
+
root = this.parent.get(root);
|
|
75
|
+
}
|
|
76
|
+
// Path compression.
|
|
77
|
+
let cur = x;
|
|
78
|
+
while (this.parent.get(cur) !== root) {
|
|
79
|
+
const next = this.parent.get(cur);
|
|
80
|
+
this.parent.set(cur, root);
|
|
81
|
+
cur = next;
|
|
82
|
+
}
|
|
83
|
+
return root;
|
|
84
|
+
}
|
|
85
|
+
union(a, b) {
|
|
86
|
+
const ra = this.find(a);
|
|
87
|
+
const rb = this.find(b);
|
|
88
|
+
if (ra === rb)
|
|
89
|
+
return;
|
|
90
|
+
const rankA = this.rank.get(ra) ?? 0;
|
|
91
|
+
const rankB = this.rank.get(rb) ?? 0;
|
|
92
|
+
if (rankA < rankB) {
|
|
93
|
+
this.parent.set(ra, rb);
|
|
94
|
+
}
|
|
95
|
+
else if (rankA > rankB) {
|
|
96
|
+
this.parent.set(rb, ra);
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
this.parent.set(rb, ra);
|
|
100
|
+
this.rank.set(ra, rankA + 1);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
nodes() {
|
|
104
|
+
return [...this.parent.keys()];
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Reduces a near-duplicate edge list to the cluster aggregate `flair
|
|
109
|
+
* quality` renders. A "cluster" requires >=2 connected memories — a memory
|
|
110
|
+
* with no qualifying edge never enters the graph, so it's correctly excluded
|
|
111
|
+
* (never counted as a size-1 "cluster").
|
|
112
|
+
*
|
|
113
|
+
* Self-loop edges (a === b — a memory paired with itself, which the sweep
|
|
114
|
+
* should already exclude, but this function stays defensive) are ignored:
|
|
115
|
+
* they can never form or grow a cluster.
|
|
116
|
+
*
|
|
117
|
+
* Duplicate/symmetric edges (the sweep may discover the SAME pair from both
|
|
118
|
+
* directions — A's neighbor search finds B, and independently B's finds A)
|
|
119
|
+
* are harmless: union() on an already-joined pair is a no-op.
|
|
120
|
+
*/
|
|
121
|
+
export function countDedupClusters(edges) {
|
|
122
|
+
const uf = new UnionFind();
|
|
123
|
+
for (const { a, b } of edges) {
|
|
124
|
+
if (a === b)
|
|
125
|
+
continue;
|
|
126
|
+
uf.union(a, b);
|
|
127
|
+
}
|
|
128
|
+
const sizeByRoot = new Map();
|
|
129
|
+
for (const id of uf.nodes()) {
|
|
130
|
+
const root = uf.find(id);
|
|
131
|
+
sizeByRoot.set(root, (sizeByRoot.get(root) ?? 0) + 1);
|
|
132
|
+
}
|
|
133
|
+
let clusterCount = 0;
|
|
134
|
+
let largestClusterSize = 0;
|
|
135
|
+
let totalMemoriesInClusters = 0;
|
|
136
|
+
for (const size of sizeByRoot.values()) {
|
|
137
|
+
if (size < 2)
|
|
138
|
+
continue; // shouldn't happen (a root only exists via an edge), but stay defensive
|
|
139
|
+
clusterCount++;
|
|
140
|
+
totalMemoriesInClusters += size;
|
|
141
|
+
if (size > largestClusterSize)
|
|
142
|
+
largestClusterSize = size;
|
|
143
|
+
}
|
|
144
|
+
return { clusterCount, largestClusterSize, totalMemoriesInClusters };
|
|
145
|
+
}
|
package/dist/resources/health.js
CHANGED
|
@@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url";
|
|
|
6
6
|
import { getRerankStatus } from "./rerank-provider.js";
|
|
7
7
|
import { allowVerified, resolveAgentAuth } from "./agent-auth.js";
|
|
8
8
|
import { getMigrationStatusSnapshot } from "./migrations/status.js";
|
|
9
|
+
import { REM_DEDUP_STATS_PATH } from "./dedup-cluster.js";
|
|
9
10
|
const db = databases;
|
|
10
11
|
const redactHome = (p) => {
|
|
11
12
|
const home = homedir();
|
|
@@ -206,7 +207,7 @@ export class HealthDetail extends Resource {
|
|
|
206
207
|
for await (const a of db.flair.Agent.search({}))
|
|
207
208
|
agents.push(a);
|
|
208
209
|
const blank = (id) => ({
|
|
209
|
-
id, memoryCount: 0, hashFallback: 0, writes24h: 0, lastWriteAt: null,
|
|
210
|
+
id, memoryCount: 0, hashFallback: 0, writes24h: 0, lastWriteAt: null, usageCount: 0,
|
|
210
211
|
});
|
|
211
212
|
const perAgentMap = new Map();
|
|
212
213
|
for (const a of agents) {
|
|
@@ -219,6 +220,7 @@ export class HealthDetail extends Resource {
|
|
|
219
220
|
continue;
|
|
220
221
|
const row = perAgentMap.get(m.agentId) ?? blank(m.agentId);
|
|
221
222
|
row.memoryCount++;
|
|
223
|
+
row.usageCount += (m.usageCount ?? 0);
|
|
222
224
|
if (!m.embeddingModel || m.embeddingModel === "hash-512d")
|
|
223
225
|
row.hashFallback++;
|
|
224
226
|
if (m.createdAt) {
|
|
@@ -499,6 +501,36 @@ export class HealthDetail extends Resource {
|
|
|
499
501
|
catch {
|
|
500
502
|
stats.rem = null;
|
|
501
503
|
}
|
|
504
|
+
// ── Dedup clusters (flair-quality Slice 1c) ──
|
|
505
|
+
// Cheap read: a single small stat file, not a recomputation. The
|
|
506
|
+
// computation itself is a nightly server-side sweep (POST
|
|
507
|
+
// /MemoryDedupStats, triggered once per REM nightly cycle — see
|
|
508
|
+
// src/rem/runner.ts + resources/MemoryDedupStats.ts's module doc) that
|
|
509
|
+
// NEVER runs inline with a GET /HealthDetail request. `null` covers both
|
|
510
|
+
// "REM hasn't computed it yet" (fresh instance) and "file unreadable" —
|
|
511
|
+
// `flair quality` renders either as an honest gap, never a false zero.
|
|
512
|
+
try {
|
|
513
|
+
const raw = await fsp.readFile(REM_DEDUP_STATS_PATH, "utf-8");
|
|
514
|
+
const parsed = JSON.parse(raw);
|
|
515
|
+
if (parsed &&
|
|
516
|
+
typeof parsed.clusterCount === "number" &&
|
|
517
|
+
typeof parsed.largestClusterSize === "number" &&
|
|
518
|
+
typeof parsed.totalMemoriesInClusters === "number" &&
|
|
519
|
+
typeof parsed.computedAt === "string") {
|
|
520
|
+
stats.dedup = {
|
|
521
|
+
clusterCount: parsed.clusterCount,
|
|
522
|
+
largestClusterSize: parsed.largestClusterSize,
|
|
523
|
+
totalMemoriesInClusters: parsed.totalMemoriesInClusters,
|
|
524
|
+
computedAt: parsed.computedAt,
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
else {
|
|
528
|
+
stats.dedup = null;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
catch {
|
|
532
|
+
stats.dedup = null;
|
|
533
|
+
}
|
|
502
534
|
// ── Migrations (flair#695: zero-touch boot-keyed auto-migration) ──
|
|
503
535
|
// `{ id, rowsDone, rowsRemaining, state }` per registered migration, per
|
|
504
536
|
// flair#695 §A. `cyclePhase: "pre-hash"` is the
|
|
@@ -19,7 +19,6 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import * as harper from "@harperfast/harper";
|
|
21
21
|
import { mcpOAuthEnabled, mcpAuthConfig } from "./mcp-oauth-flag.js";
|
|
22
|
-
import { mcpHandler } from "./mcp-handler.js";
|
|
23
22
|
async function defaultLoadWithMCPAuth() {
|
|
24
23
|
// Dynamic import so the dep is only required when the surface is enabled.
|
|
25
24
|
const mod = (await import("@harperfast/oauth"));
|
|
@@ -49,6 +48,9 @@ export async function registerMcpOAuthRoute(deps = {}) {
|
|
|
49
48
|
console.error("[mcp-oauth] @harperfast/oauth has no withMCPAuth export — /mcp NOT mounted.");
|
|
50
49
|
return false;
|
|
51
50
|
}
|
|
51
|
+
// Resolve the handler lazily (injected in tests; real module otherwise) — see
|
|
52
|
+
// the top-of-file note on why it isn't a static import.
|
|
53
|
+
const handler = deps.mcpHandler ?? (await import("./mcp-handler.js")).mcpHandler;
|
|
52
54
|
// Read `server` lazily off the namespace (it's a runtime global on the Harper
|
|
53
55
|
// module, not a static named export) so this module links cleanly even where a
|
|
54
56
|
// stub build of @harperfast/harper lacks the export.
|
|
@@ -58,7 +60,7 @@ export async function registerMcpOAuthRoute(deps = {}) {
|
|
|
58
60
|
// wrapper's iss/aud checks match the minted tokens even if this component
|
|
59
61
|
// resolves a different node_modules copy of the plugin (docs/mcp-oauth.md
|
|
60
62
|
// §"Using withMCPAuth from a different component").
|
|
61
|
-
srv.http(withMCPAuth(
|
|
63
|
+
srv.http(withMCPAuth(handler, {
|
|
62
64
|
getConfig: () => mcpAuthConfig(),
|
|
63
65
|
}), { urlPath: "/mcp" });
|
|
64
66
|
console.error(`[mcp-oauth] /mcp mounted (OAuth-guarded); issuer=${config.issuer}`);
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{"version": 3, "term": {"cols": 96, "rows": 28}, "timestamp": 1778889600, "command": "/tmp/flair-cross-orchestrator-script.sh", "env": {"SHELL": "/bin/zsh"}}
|
|
2
|
+
[0.245, "o", "\u001b[H\u001b[J"]
|
|
3
|
+
[0.001, "o", " Flair \u00b7 same identity, every orchestrator\r\n \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\r\n\r\n"]
|
|
4
|
+
[1.3, "o", "$ # One agent identity. One memory store. Three orchestrators.\r\n"]
|
|
5
|
+
[1.2, "o", "$ flair init --client all --agent flint\r\n"]
|
|
6
|
+
[0.5, "o", " \u001b[32m\u2713\u001b[0m Claude Code wired (~/.claude/mcp.json)\r\n"]
|
|
7
|
+
[0.18, "o", " \u001b[32m\u2713\u001b[0m Codex CLI wired (~/.codex/config.toml)\r\n"]
|
|
8
|
+
[0.18, "o", " \u001b[32m\u2713\u001b[0m Gemini CLI wired (~/.gemini/settings.json)\r\n\r\n"]
|
|
9
|
+
[1.0, "o", " Each MCP client now connects to: http://127.0.0.1:9926\r\n"]
|
|
10
|
+
[0.0, "o", " Ed25519 keypair shared: ~/.flair/keys/flint.ed25519\r\n\r\n"]
|
|
11
|
+
[2.2, "o", "$ # \u2500\u2500 Claude Code: agent writes a memory \u2500\u2500\r\n"]
|
|
12
|
+
[0.7, "o", "$ claude\r\n"]
|
|
13
|
+
[0.5, "o", "\u001b[36mflint>\u001b[0m \u001b[2mremember: we picked Postgres for replication-lag reasons; ScyllaDB ruled out by ops complexity\u001b[0m\r\n"]
|
|
14
|
+
[1.8, "o", " \u001b[32m\u2713\u001b[0m wrote memory mem_2026-05-14_3f9a (durability=persistent)\r\n\r\n"]
|
|
15
|
+
[1.5, "o", "$ # \u2500\u2500 New session. Different orchestrator. Same identity. \u2500\u2500\r\n"]
|
|
16
|
+
[1.0, "o", "$ codex\r\n"]
|
|
17
|
+
[0.5, "o", "\u001b[36mflint>\u001b[0m \u001b[2mwhat database did we pick and why?\u001b[0m\r\n"]
|
|
18
|
+
[1.5, "o", " \u001b[32m\u2713\u001b[0m bootstrap: 287 memories loaded for agent flint\r\n"]
|
|
19
|
+
[0.4, "o", " Postgres \u2014 chosen for replication-lag reasons.\r\n"]
|
|
20
|
+
[0.0, "o", " ScyllaDB was ruled out by ops complexity.\r\n"]
|
|
21
|
+
[0.0, "o", " \u001b[2m(source: mem_2026-05-14_3f9a, written via Claude Code 4m ago)\u001b[0m\r\n\r\n"]
|
|
22
|
+
[2.0, "o", "$ # \u2500\u2500 New session. Another orchestrator. Same identity. \u2500\u2500\r\n"]
|
|
23
|
+
[1.0, "o", "$ gemini\r\n"]
|
|
24
|
+
[0.5, "o", "\u001b[36mflint>\u001b[0m \u001b[2msummarize our recent infra decisions\u001b[0m\r\n"]
|
|
25
|
+
[1.7, "o", " \u001b[32m\u2713\u001b[0m bootstrap: same 287 memories, same soul, same Ed25519 identity\r\n"]
|
|
26
|
+
[0.5, "o", " \u2022 Postgres for replication-lag (2026-05-14)\r\n"]
|
|
27
|
+
[0.0, "o", " \u2022 Federation pair: local\u2192Fabric via Ed25519 (2026-05-05)\r\n"]
|
|
28
|
+
[0.0, "o", " \u2022 REM nightly cycle ships in v0.9.0 (2026-05-14)\r\n\r\n"]
|
|
29
|
+
[2.5, "o", " \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\r\n"]
|
|
30
|
+
[0.0, "o", " No SaaS in the middle. No vendor lock.\r\n"]
|
|
31
|
+
[0.0, "o", " Memory follows the agent across orchestrators.\r\n"]
|
|
32
|
+
[0.0, "o", "\r\n github.com/tpsdev-ai/flair \u00b7 Apache 2.0\r\n\r\n"]
|
|
33
|
+
[5.0, "x", "0"]
|
|
Binary file
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{"version":3,"term":{"cols":80,"rows":24},"timestamp":1778359292,"command":"/tmp/flair-demo-script.sh","env":{"SHELL":"/bin/zsh"}}
|
|
2
|
+
[0.245, "o", "\u001b[H\u001b[J"]
|
|
3
|
+
[0.001, "o", " Flair · identity infrastructure for AI agents\r\n ─────────────────────────────────────────────\r\n\r\n"]
|
|
4
|
+
[1.520, "o", "$ # Agent writes a memory\r\n"]
|
|
5
|
+
[0.628, "o", "$ flair memory add --agent flint-demo --content \"Postgres replication lag spiked when we added the read-replica\"\r\n"]
|
|
6
|
+
[0.796, "o", "{\r\n \"ok\": true\r\n}\r\n"]
|
|
7
|
+
[1.411, "o", "\r\n"]
|
|
8
|
+
[0.000, "o", "$ # Six months later — find it by meaning, not keywords\r\n"]
|
|
9
|
+
[0.509, "o", "$ flair search \"why did our database fall behind during scaling\" --agent flint-demo\r\n"]
|
|
10
|
+
[0.824, "o", " Postgres replication lag spiked when we added the read-replica\r\n (2026-05-09)\r\n"]
|
|
11
|
+
[0.000, "o", "\r\n"]
|
|
12
|
+
[2.054, "o", "\r\n$ # Same memory, every harness.\r\n"]
|
|
13
|
+
[0.630, "o", "$ # Claude Code · Cursor · Codex CLI · Gemini CLI · Continue.dev · Goose\r\n$ # all via @tpsdev-ai/flair-mcp\r\n$ # LangGraph @tpsdev-ai/langgraph-flair\r\n$ # OpenClaw @tpsdev-ai/openclaw-flair\r\n$ # n8n @tpsdev-ai/n8n-nodes-flair\r\n$ # Hermes Agent packages/hermes-flair (Python)\r\n$ # Pi agent @tpsdev-ai/pi-flair\r\n"]
|
|
14
|
+
[2.125, "o", "\r\n"]
|
|
15
|
+
[0.000, "o", " ─────────────────────────────────────────────\r\n"]
|
|
16
|
+
[0.000, "o", " Apache 2.0 · self-hosted · no SaaS in the middle\r\n github.com/tpsdev-ai/flair\r\n"]
|
|
17
|
+
[0.000, "o", "\r\n"]
|
|
18
|
+
[5.147, "x", "0"]
|
|
Binary file
|