@tpsdev-ai/flair 0.49.0 → 0.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +15 -4
  2. package/dist/bridges/runtime/roundtrip.js +91 -2
  3. package/dist/build-info.json +3 -3
  4. package/dist/cli.js +565 -171
  5. package/dist/deploy.js +20 -3
  6. package/dist/doctor-client.js +62 -34
  7. package/dist/federation/scheduler.js +24 -3
  8. package/dist/hook-install.js +45 -13
  9. package/dist/install/clients.js +28 -17
  10. package/dist/lib/doctor-run.js +481 -0
  11. package/dist/lib/launchd-management.js +7 -26
  12. package/dist/lib/scheduler-platform.js +132 -10
  13. package/dist/lib/scratch-owner.js +49 -0
  14. package/dist/rem/scheduler.js +23 -5
  15. package/dist/resources/Federation.js +42 -20
  16. package/dist/resources/MemoryBootstrap.js +8 -4
  17. package/dist/resources/RecordUsage.js +13 -6
  18. package/dist/resources/SemanticSearch.js +8 -1
  19. package/dist/resources/federation-classify.js +90 -0
  20. package/dist/resources/health.js +51 -7
  21. package/dist/resources/mcp-tools.js +10 -6
  22. package/dist/resources/search-readiness.js +123 -0
  23. package/dist/resources/semantic-retrieval-core.js +48 -21
  24. package/dist/resources/sort-comparators.js +45 -0
  25. package/dist/resources/usage-ids.js +63 -0
  26. package/dist/src/lib/scheduler-platform.js +132 -10
  27. package/dist/src/rem/scheduler.js +23 -5
  28. package/docs/auth.md +5 -0
  29. package/docs/deepseek-harness.md +1 -1
  30. package/docs/federation.md +11 -0
  31. package/docs/hosted-on-fabric.md +2 -0
  32. package/docs/integrations.md +53 -1
  33. package/docs/mcp-clients.md +67 -15
  34. package/docs/quickstart-fabric.md +1 -1
  35. package/docs/supply-chain-policy.md +1 -1
  36. package/docs/troubleshooting.md +25 -0
  37. package/docs/upgrade.md +17 -1
  38. package/package.json +3 -3
  39. package/schemas/federation.graphql +1 -1
@@ -1,4 +1,4 @@
1
- import { Resource, databases } from "harper";
1
+ import { Resource, databases, server, logger } from "harper";
2
2
  import { promises as fsp, existsSync, readFileSync } from "node:fs";
3
3
  import { homedir, platform } from "node:os";
4
4
  import { join, dirname } from "node:path";
@@ -8,6 +8,9 @@ import { resolveBuildInfo } from "./build-info.js";
8
8
  import { getMigrationStatusSnapshot } from "./migrations/status.js";
9
9
  import { resolveMigrationDataDirForRead } from "./migrations/data-dir.js";
10
10
  import { REM_DEDUP_STATS_PATH } from "./dedup-cluster.js";
11
+ import { hybridEnabled } from "./bm25.js";
12
+ import { bm25IndexEnabled, bm25IndexStatus } from "./bm25-index-service.js";
13
+ import { buildPublicHealthBody, resolveSearchReadiness } from "./search-readiness.js";
11
14
  const db = databases;
12
15
  const redactHome = (p) => {
13
16
  const home = homedir();
@@ -52,7 +55,7 @@ function resolveVersion() {
52
55
  return process.env.npm_package_version ?? "dev";
53
56
  }
54
57
  /**
55
- * Health endpoint — truly public, returns only { ok: true }.
58
+ * Health endpoint — truly public. Identity-free; no auth, no role check.
56
59
  *
57
60
  * `allowRead() { return true }` opens Harper's role gate for anonymous GETs,
58
61
  * which is what makes /Health work for callers outside `authorizeLocal`'s
@@ -66,8 +69,15 @@ function resolveVersion() {
66
69
  *
67
70
  * Same pattern as `FederationPair.allowCreate(){ return true }` (PR #299):
68
71
  * declare the Resource anonymously-accessible at the Harper layer; let the
69
- * handler itself enforce whatever it needs (in this case, nothing — /Health
70
- * is intentionally and always public).
72
+ * handler itself enforce whatever it needs.
73
+ *
74
+ * flair#1326: `ok: true` used to mean only "this resource answered." That
75
+ * is a green light that lies when search routes are not mounted yet, or
76
+ * when the hybrid BM25 index is still cold (first search after restart
77
+ * scans the corpus; the lag grows with store size). `searchReady` is
78
+ * always present. When search cannot be served at all, this endpoint
79
+ * returns HTTP 503 and `ok: false`. When the process is live but recall
80
+ * is still cold, it stays 200 and names the lag on `searchReadyReason`.
71
81
  *
72
82
  * Rich stats (memory counts, agent names, etc.) are behind /HealthDetail
73
83
  * which requires authentication. This prevents information leakage on
@@ -96,13 +106,35 @@ export class Health extends Resource {
96
106
  // a 40-hex sha when the build ran in a git work tree, an honest null
97
107
  // otherwise (tarball builds) — never omitted, never fabricated (Sherlock).
98
108
  const build = resolveBuildInfo();
99
- return {
100
- ok: true,
109
+ const readiness = currentSearchReadiness();
110
+ const body = buildPublicHealthBody(readiness, {
101
111
  version: build?.version ?? resolveVersion(),
102
112
  buildCommit: build?.commit ?? null,
103
- };
113
+ });
114
+ if (readiness.status !== 200) {
115
+ return new Response(JSON.stringify(body), {
116
+ status: readiness.status,
117
+ headers: { "content-type": "application/json" },
118
+ });
119
+ }
120
+ return body;
104
121
  }
105
122
  }
123
+ /** Same sources /Health and /HealthDetail consult so they cannot disagree. */
124
+ export function currentSearchReadiness() {
125
+ // Fail-open when the registry is missing (Sherlock on #1406 / flair#1411):
126
+ // do not 503 forever. resolveSearchReadiness warns once and names the
127
+ // degradation; we do not treat "registry should always be here" as a given.
128
+ const resources = server.resources ?? null;
129
+ return resolveSearchReadiness({
130
+ resources,
131
+ memoryTable: db.flair?.Memory,
132
+ bm25: bm25IndexStatus(),
133
+ hybridEnabled: hybridEnabled(),
134
+ bm25IndexEnabled: bm25IndexEnabled(),
135
+ warn: (message) => { logger.warn?.(message); },
136
+ });
137
+ }
106
138
  /**
107
139
  * Authenticated health detail — returns memory/agent/soul stats + process info.
108
140
  * Requires Ed25519 agent auth or admin basic auth.
@@ -127,6 +159,18 @@ export class HealthDetail extends Resource {
127
159
  const stats = { ok: true };
128
160
  const nowMs = Date.now();
129
161
  const warnings = [];
162
+ // flair#1326: same search-ready signal as public /Health. HealthDetail
163
+ // stays HTTP 200 (it is a stats dump, not a traffic gate); the field
164
+ // and a warning name the lag so `flair status` / operators can see it.
165
+ const readiness = currentSearchReadiness();
166
+ stats.searchReady = readiness.searchReady;
167
+ // Public/detail shape is unchanged: searchReadyReason stays a lag signal
168
+ // (present iff !searchReady). Ready-path verification constants stay on
169
+ // the decision object (flair#1411).
170
+ if (!readiness.searchReady && readiness.searchReadyReason) {
171
+ stats.searchReadyReason = readiness.searchReadyReason;
172
+ warnings.push({ level: "warn", message: readiness.searchReadyReason });
173
+ }
130
174
  const ctx = this.getContext?.();
131
175
  // #614 fix: resolve identity via the shared three-way verdict
132
176
  // (internal/agent/anonymous — agent-auth.ts) instead of reading
@@ -37,6 +37,7 @@
37
37
  */
38
38
  import { resolveVersion } from "./version.js";
39
39
  import { agentContext, adminContext, collectionResource } from "./in-process.js";
40
+ import { RECORD_USAGE_ID_MERGE_CONTRACT, unionUsageMemoryIds } from "./usage-ids.js";
40
41
  const H = {};
41
42
  const LOADERS = {
42
43
  SemanticSearch: async () => (await import("./SemanticSearch.js")).SemanticSearch,
@@ -538,9 +539,10 @@ async function attention(agent, args) {
538
539
  async function recordUsage(agent, args) {
539
540
  const Cls = await handler("RecordUsage");
540
541
  const h = new Cls(undefined, delegationContext(agent));
541
- const memoryIds = Array.isArray(args?.memoryIds)
542
- ? args.memoryIds
543
- : typeof args?.memoryId === "string" ? [args.memoryId] : undefined;
542
+ // flair#1410: MERGE, do not prefer. The previous ternary dropped
543
+ // `memoryId` whenever `memoryIds` was an array (including `[]`).
544
+ const merged = unionUsageMemoryIds(args?.memoryId, args?.memoryIds);
545
+ const memoryIds = merged.length > 0 ? merged : undefined;
544
546
  return unwrap(await h.post({ memoryIds, attribution: args?.attribution }));
545
547
  }
546
548
  /**
@@ -811,6 +813,7 @@ export const TOOLS = {
811
813
  { count: "memoriesIncluded", containers: ["memories", "predicted"] }, // #1199
812
814
  { count: "teammateFindingsIncluded", containers: ["teammateFindings"] }, // #1199
813
815
  { count: "sections.events", containers: ["events"] }, // #1206
816
+ { count: "sections.soul", containers: ["soul"] }, // #1371
814
817
  ],
815
818
  // present + typed even when empty — never a bare {} / missing key (#1182).
816
819
  selfDescribingEmpty: [
@@ -1004,12 +1007,13 @@ export const TOOLS = {
1004
1007
  name: "record_usage",
1005
1008
  description: "Report that one or more memories were actually USED — cited or relied on to ground an answer or decision. " +
1006
1009
  "Distinct from search (surfacing a memory is not usage). Drives the recall-quality usage signal; dedup'd " +
1007
- "(you can only count once per memory) and rate-limited.",
1010
+ "(you can only count once per memory) and rate-limited. " +
1011
+ RECORD_USAGE_ID_MERGE_CONTRACT,
1008
1012
  inputSchema: {
1009
1013
  type: "object",
1010
1014
  properties: {
1011
- memoryIds: { type: "array", items: { type: "string" }, description: "IDs of the memories that were used (max 20 per call)" },
1012
- memoryId: { type: "string", description: "Convenience alias for a single memory id (use memoryIds for multiple)" },
1015
+ memoryIds: { type: "array", items: { type: "string" }, description: "IDs of the memories that were used (max 20 per call). Merged with memoryId when both are supplied." },
1016
+ memoryId: { type: "string", description: "Convenience alias for a single memory id. Merged with memoryIds when both are supplied — not dropped." },
1013
1017
  attribution: { type: "string", description: "Optional free-text note on what used it (opaque — stored for audit only, max 500 chars)" },
1014
1018
  },
1015
1019
  },
@@ -0,0 +1,123 @@
1
+ /**
2
+ * search-readiness.ts — the honest /Health search-ready signal (flair#1326).
3
+ *
4
+ * Harper's process can answer /health (and Flair's /Health resource can
5
+ * answer {ok:true}) while search is still unusable:
6
+ *
7
+ * 1. Boot window — jsResources register incrementally. /Health can be up
8
+ * while /Memory and /SemanticSearch still 404 from Harper's catch-all
9
+ * (documented in packages/adk-flair-js/test/helpers/boot-harper.mjs).
10
+ * 2. Cold BM25 index — hybrid retrieval's persistent index is lazy-built
11
+ * on the first search, not at component start (bm25-index-service.ts).
12
+ * That first-query corpus scan grows with store size. /Health answering
13
+ * is not "recall is warm."
14
+ *
15
+ * This module is Harper-free so the decision is unit-testable against the
16
+ * shipped function. Callers inject the registry / table / index status they
17
+ * already have.
18
+ *
19
+ * Two layers, on purpose:
20
+ * - Routes/table not mounted → not healthy (ok:false, HTTP 503). A
21
+ * traffic-gating probe that only looks at status must not get a green
22
+ * light for a node whose search routes are not serving.
23
+ * - Routes up but index still cold → process is live (ok:true, HTTP 200)
24
+ * and searchReady:false names the lag. We do NOT 503 on a cold index:
25
+ * the index builds on the first search, and a health check must not
26
+ * trigger that scan (bm25-index-service.ts: "Eager building would add a
27
+ * full corpus scan to every boot including … health checks"). 503-until-
28
+ * warm would deadlock — health waits for the index, the index waits for
29
+ * a search that never comes.
30
+ */
31
+ /**
32
+ * How a ready result was verified. Constant strings — no interpolation
33
+ * (flair#1411). Public /Health still omits this field when searchReady is
34
+ * true; the values live on the decision object so a caller of
35
+ * resolveSearchReadiness can tell registry-verified from table-only.
36
+ */
37
+ export const SEARCH_READY_REASON_VERIFIED_VIA_ROUTE_REGISTRY = "verified via route registry";
38
+ export const SEARCH_READY_REASON_REGISTRY_UNAVAILABLE_TABLE_ONLY = "registry unavailable, table check only";
39
+ /** Once-warn when the route-mount check is skipped (flair#1411). */
40
+ export const MISSING_REGISTRY_WARN = "search route-mount verification is degraded; searchReady now rests on the table check alone";
41
+ let warnedMissingRegistry = false;
42
+ /** Test-only: forget the one-shot missing-registry warning. */
43
+ export function _resetMissingRegistryWarnForTests() {
44
+ warnedMissingRegistry = false;
45
+ }
46
+ function routeMounted(resources, name) {
47
+ const entry = resources.get?.(name) ?? resources.getMatch?.(name);
48
+ return Boolean(entry?.Resource);
49
+ }
50
+ /**
51
+ * Decide whether search is actually usable, and whether /Health should claim
52
+ * the process is healthy.
53
+ *
54
+ * `resources` is optional. Stated fail-open (Sherlock on #1406 / flair#1411):
55
+ * when the registry is missing we skip the route-mount check rather than
56
+ * 503 forever. A table handle can exist while `/Memory` and `/SemanticSearch`
57
+ * still 404, so this is weaker than the primary defense. We do not fail
58
+ * closed. We warn once and name the degradation on `searchReadyReason`.
59
+ */
60
+ export function resolveSearchReadiness(opts) {
61
+ let readyReason = SEARCH_READY_REASON_REGISTRY_UNAVAILABLE_TABLE_ONLY;
62
+ if (opts.resources) {
63
+ const memoryMounted = routeMounted(opts.resources, "Memory");
64
+ const searchMounted = routeMounted(opts.resources, "SemanticSearch");
65
+ if (!memoryMounted || !searchMounted) {
66
+ const missing = [
67
+ !memoryMounted ? "Memory" : null,
68
+ !searchMounted ? "SemanticSearch" : null,
69
+ ].filter(Boolean).join(", ");
70
+ return notServing(`search routes not mounted (${missing})`);
71
+ }
72
+ readyReason = SEARCH_READY_REASON_VERIFIED_VIA_ROUTE_REGISTRY;
73
+ }
74
+ else {
75
+ if (!warnedMissingRegistry) {
76
+ warnedMissingRegistry = true;
77
+ opts.warn?.(MISSING_REGISTRY_WARN);
78
+ }
79
+ }
80
+ if (!opts.memoryTable || typeof opts.memoryTable.search !== "function") {
81
+ return notServing("memory table not queryable");
82
+ }
83
+ // Hybrid + the persistent index are default-on. A cold or in-flight BM25
84
+ // index means the first search will pay a full corpus scan — the #1326
85
+ // lag. Name it; do not fail liveness.
86
+ //
87
+ // `disabled` (feed/build failure) and `FLAIR_BM25_INDEX=false` both fall
88
+ // back to the per-query scan. The kill switch never calls ensureReady, so
89
+ // status stays `empty` for the life of the process — that is serving, not
90
+ // cold. Treating it as lag would make searchReady false forever and refuse
91
+ // a node that is already answering recall.
92
+ const indexInPath = opts.hybridEnabled !== false && opts.bm25IndexEnabled !== false;
93
+ if (indexInPath && opts.bm25) {
94
+ if (opts.bm25.state === "building") {
95
+ return namesLag("bm25 index building — first search is still scanning the corpus");
96
+ }
97
+ if (opts.bm25.state === "empty") {
98
+ return namesLag("bm25 index not built (cold boot; first search scans the corpus)");
99
+ }
100
+ }
101
+ return { searchReady: true, ok: true, status: 200, searchReadyReason: readyReason };
102
+ }
103
+ function notServing(searchReadyReason) {
104
+ return { searchReady: false, ok: false, status: 503, searchReadyReason };
105
+ }
106
+ function namesLag(searchReadyReason) {
107
+ return { searchReady: false, ok: true, status: 200, searchReadyReason };
108
+ }
109
+ /** Public /Health JSON body. `searchReady` is always present (never omitted). */
110
+ export function buildPublicHealthBody(readiness, identity) {
111
+ const body = {
112
+ ok: readiness.ok,
113
+ version: identity.version,
114
+ buildCommit: identity.buildCommit,
115
+ searchReady: readiness.searchReady,
116
+ };
117
+ // Public shape is unchanged: searchReadyReason is still present iff !searchReady.
118
+ // Ready-path verification constants stay on the decision object (flair#1411).
119
+ if (!readiness.searchReady && readiness.searchReadyReason) {
120
+ body.searchReadyReason = readiness.searchReadyReason;
121
+ }
122
+ return body;
123
+ }
@@ -34,9 +34,11 @@
34
34
  // multiplies `limit` internally; any overfetch policy — SemanticSearch's
35
35
  // CANDIDATE_MULTIPLIER, MemoryBootstrap's K formula — is the CALLER's
36
36
  // decision, made
37
- // before calling in). Never exposes which internal leg (BM25+RRF hybrid vs.
38
- // legacy HNSW-only vs. keyword-only fallback) produced a given result — the
39
- // output shape is identical regardless of `hybrid`.
37
+ // before calling in). The result array never annotates which internal leg
38
+ // (BM25+RRF hybrid vs. legacy HNSW-only vs. keyword-only fallback) produced
39
+ // a given row — that output shape is identical regardless of `hybrid`.
40
+ // Per-leg membership is available only through the opt-in `onLegs`
41
+ // callback (flair#1358); unset ⇒ the return is byte-identical to before.
40
42
  //
41
43
  // ── SCORE CONTRACT (flair#985) ───────────────────────────────────────────────
42
44
  // `_score` under `scoring:"raw"` is ALWAYS an ABSOLUTE similarity (cosine of
@@ -60,6 +62,15 @@ import { compositeScore } from "./scoring.js";
60
62
  import { buildBM25, fuseRrfNormalized, SEM_LIMIT } from "./bm25.js";
61
63
  import { isAllowedBm25Candidate } from "./bm25-filter.js";
62
64
  import { indexedBm25Ids } from "./bm25-index-service.js";
65
+ import { byRecencyThenId } from "./sort-comparators.js";
66
+ /**
67
+ * The only way a row enters the ranked pool. `_rank` is a required
68
+ * argument — not an object property that `...any` can swallow — so a
69
+ * push site that omits it is a compile error (flair#1415).
70
+ */
71
+ function pushRanked(rows, row, _rank) {
72
+ rows.push({ ...row, _rank });
73
+ }
63
74
  // Convert HNSW cosine distance (1 - similarity) to similarity score.
64
75
  function distanceToSimilarity(distance) {
65
76
  return 1 - distance;
@@ -86,10 +97,12 @@ export const DEFAULT_SELECT = ["id", "agentId", "content", "contentHash", "visib
86
97
  "parentId", "derivedFrom", "sessionId", "lastReflected", "supersedes", "subject", "summary",
87
98
  "validFrom", "validTo", "_safetyFlags"];
88
99
  export async function retrieveCandidates(params) {
89
- const { queryEmbedding: qEmb, q, conditions, limit, select = DEFAULT_SELECT, includeSuperseded = false, scoring = "raw", temporalBoost = 1.0, sinceDate = null, asOf, minScore = 0, agentId, isAllowed, hybrid, ctx, withSemSimilarity = false, } = params;
100
+ const { queryEmbedding: qEmb, q, conditions, limit, select = DEFAULT_SELECT, includeSuperseded = false, scoring = "raw", temporalBoost = 1.0, sinceDate = null, asOf, minScore = 0, agentId, isAllowed, hybrid, ctx, withSemSimilarity = false, onLegs, } = params;
90
101
  const passesAllowed = (record) => !isAllowed || isAllowed(record);
91
102
  const hnswSelect = [...select, "$distance"];
92
103
  const results = [];
104
+ const hnswLegIds = [];
105
+ const bm25LegIds = [];
93
106
  if (hybrid) {
94
107
  // ─── BM25 + union-RRF hybrid path ────────────────────────────────────
95
108
  // 1. Semantic candidates via HNSW (unchanged fetch). 2. BM25 lexical pass
@@ -153,6 +166,7 @@ export async function retrieveCandidates(params) {
153
166
  }
154
167
  semRecords.push(record);
155
168
  semIds.push(record.id);
169
+ hnswLegIds.push(record.id);
156
170
  }
157
171
  }
158
172
  // ── (b) The BM25 lexical leg ─────────────────────────────────────────
@@ -259,6 +273,8 @@ export async function retrieveCandidates(params) {
259
273
  }
260
274
  bm25Ids = resolved;
261
275
  }
276
+ if (q)
277
+ bm25LegIds.push(...bm25Ids);
262
278
  // ── (d) No retrieval signal at all → full scoped listing ────────────
263
279
  if (!q && !qEmb) {
264
280
  for (const record of allowedById.values()) {
@@ -268,14 +284,13 @@ export async function retrieveCandidates(params) {
268
284
  finalScore *= temporalBoost;
269
285
  const isFlagged = record._safetyFlags && Array.isArray(record._safetyFlags) && record._safetyFlags.length > 0;
270
286
  const source = record.agentId !== agentId ? record.agentId : undefined;
271
- results.push({
287
+ pushRanked(results, {
272
288
  ...record,
273
289
  content: isFlagged ? wrapUntrusted(record.content, source) : record.content,
274
290
  _score: Math.round(finalScore * 1000) / 1000,
275
291
  _rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
276
292
  _source: source,
277
- _rank: finalScore,
278
- });
293
+ }, finalScore);
279
294
  }
280
295
  }
281
296
  else {
@@ -323,22 +338,21 @@ export async function retrieveCandidates(params) {
323
338
  finalScore *= temporalBoost;
324
339
  const isFlagged = record._safetyFlags && Array.isArray(record._safetyFlags) && record._safetyFlags.length > 0;
325
340
  const source = record.agentId !== agentId ? record.agentId : undefined;
326
- results.push({
341
+ // Ordering key: fused rank for raw mode; composite value for
342
+ // composite mode (composite ordering is unchanged by #985 — its
343
+ // rrfRaw input and result order are exactly the pre-#985 behavior).
344
+ pushRanked(results, {
327
345
  ...record,
328
346
  content: isFlagged ? wrapUntrusted(record.content, source) : record.content,
329
347
  _score: Math.round(finalScore * 1000) / 1000,
330
348
  _rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
331
349
  _source: source,
332
- // Ordering key: fused rank for raw mode; composite value for
333
- // composite mode (composite ordering is unchanged by #985 — its
334
- // rrfRaw input and result order are exactly the pre-#985 behavior).
335
- _rank: scoring === "raw" ? rrfRaw : finalScore,
336
350
  // flair#744 slice 2: the opt-in absolute-confidence field for the
337
351
  // abstention decision. Attach remains OPT-IN so non-abstain
338
352
  // responses stay byte-identical (the capture above is now
339
353
  // unconditional, but the response field is not).
340
354
  ...(withSemSimilarity && semSim !== undefined ? { _semSimilarity: semSim } : {}),
341
- });
355
+ }, scoring === "raw" ? rrfRaw : finalScore);
342
356
  }
343
357
  }
344
358
  }
@@ -400,15 +414,15 @@ export async function retrieveCandidates(params) {
400
414
  // flair#744 slice 2: the absolute cosine (`semanticScore`, pre keyword
401
415
  // bump) is the abstention confidence signal on this legacy/bootstrap
402
416
  // (HNSW-leg-only) path.
403
- results.push({
417
+ pushRanked(results, {
404
418
  ...rest,
405
419
  content: isFlagged ? wrapUntrusted(rest.content, source) : rest.content,
406
420
  _score: Math.round(finalScore * 1000) / 1000,
407
421
  _rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
408
422
  _source: source,
409
- _rank: finalScore,
410
423
  ...(withSemSimilarity ? { _semSimilarity: semanticScore } : {}),
411
- });
424
+ }, finalScore);
425
+ hnswLegIds.push(record.id);
412
426
  }
413
427
  }
414
428
  else {
@@ -446,14 +460,13 @@ export async function retrieveCandidates(params) {
446
460
  finalScore *= temporalBoost;
447
461
  const isFlagged = rest._safetyFlags && Array.isArray(rest._safetyFlags) && rest._safetyFlags.length > 0;
448
462
  const source = record.agentId !== agentId ? record.agentId : undefined;
449
- results.push({
463
+ pushRanked(results, {
450
464
  ...rest,
451
465
  content: isFlagged ? wrapUntrusted(rest.content, source) : rest.content,
452
466
  _score: Math.round(finalScore * 1000) / 1000,
453
467
  _rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
454
468
  _source: source,
455
- _rank: finalScore,
456
- });
469
+ }, finalScore);
457
470
  }
458
471
  }
459
472
  // Build superseded set and filter (unless caller opts in to see full
@@ -479,13 +492,27 @@ export async function retrieveCandidates(params) {
479
492
  }
480
493
  // Order by the internal ranking key (fused RRF rank on the hybrid raw path;
481
494
  // identical to `_score` everywhere else), then strip it — `_rank` is an
482
- // ordering key, never part of the response shape. Note the hybrid raw
495
+ // ordering key, never part of the response shape. Required on
496
+ // RetrievalRankedRow / pushRanked (flair#1415) so a missing value cannot
497
+ // silently NaN this comparator into the recency tail. Note the hybrid raw
483
498
  // ordering is deliberately NOT by `_score`: the recall win of hybrid
484
499
  // retrieval lives in the fused ORDER (a BM25 rank-1 rescue outranks weak
485
500
  // semantic hits), while `_score` carries the honest absolute evidence for
486
501
  // each result — the two can disagree, and that is correct.
487
- filteredResults.sort((a, b) => b._rank - a._rank);
502
+ //
503
+ // Ties at this sort are cross-leg RRF identities (flair#1412): no
504
+ // within-leg tie-break can prevent `1/(k+3)+1/(k+5) === 1/(k+5)+1/(k+3)`.
505
+ // Break them by createdAt DESC (null → oldest, never NaN) then id ASC.
506
+ // Determinism is the id ASC total order. createdAt is best-effort recency
507
+ // within an exact `_rank` tie — federated writer clocks can skew, and
508
+ // that cannot reintroduce nondeterminism (see byRecencyThenId).
509
+ filteredResults.sort((a, b) => (b._rank - a._rank) || byRecencyThenId(a, b));
488
510
  for (const r of filteredResults)
489
511
  delete r._rank;
512
+ onLegs?.({
513
+ hnsw: hnswLegIds,
514
+ bm25: bm25LegIds,
515
+ fused: filteredResults.map((r) => r.id),
516
+ });
490
517
  return filteredResults;
491
518
  }
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Deterministic sort comparators (flair#1412).
3
+ *
4
+ * Retrieval and selection sorts used to key on one number (score, rank,
5
+ * priority) and then inherit `Array.prototype.sort`'s input order on ties.
6
+ * That input order is Harper scan order, which is free to change across
7
+ * restarts. The family below makes the tie-break the default thing to
8
+ * reach for: each helper has its own primary key, and they share the
9
+ * `compareKey` tail — not one comparator forced onto every site.
10
+ *
11
+ * Do not collapse this into a single `byScoreThenId`. cosine.ts keys on
12
+ * corpus `.index`; MemoryBootstrap (flair#1409, not this PR) keys on the
13
+ * soul `key`. The shared part is the tail.
14
+ *
15
+ * Locale-independent: `compareKey` is code-unit / numeric `<`/`>`, never
16
+ * `localeCompare`. ISO-8601 UTC strings compare chronologically that way.
17
+ */
18
+ /** Total-order tail. Works for ids, corpus indexes, soul keys. */
19
+ export function compareKey(a, b) {
20
+ return a < b ? -1 : a > b ? 1 : 0;
21
+ }
22
+ /** Score / rank descending, then `id` ascending. */
23
+ export function byNumberDescThenId(getNumber) {
24
+ return (a, b) => (getNumber(b) - getNumber(a)) || compareKey(a.id, b.id);
25
+ }
26
+ /**
27
+ * `createdAt` DESC (null → oldest, never NaN) then `id` ASC.
28
+ *
29
+ * Determinism comes from `id` ASC — ids are unique, so this is a total
30
+ * order no matter what `createdAt` does. Do not date-arithmetic a
31
+ * possibly-null field: `new Date(null).getTime()` is 0, but subtracting
32
+ * NaN (unparseable / missing-after-coercion) is undefined behaviour for
33
+ * `Array.prototype.sort`. Empty-string fallback keeps the comparison in
34
+ * string space; an empty value is less than any ISO-8601 UTC timestamp,
35
+ * so it sorts last under DESC.
36
+ *
37
+ * Clock-skew caveat: `createdAt` is writer-stamped. Across federated
38
+ * writers this is best-effort recency within an exact `_rank` tie, not a
39
+ * correctness claim. Skew can misorder rows the ranker already called
40
+ * equivalent; it cannot reintroduce nondeterminism, because `id` ASC
41
+ * still resolves every remaining tie.
42
+ */
43
+ export function byRecencyThenId(a, b) {
44
+ return compareKey(b.createdAt ?? "", a.createdAt ?? "") || compareKey(a.id, b.id);
45
+ }
@@ -0,0 +1,63 @@
1
+ /**
2
+ * usage-ids.ts — which memory ids a usage-feedback call credits (flair#1410).
3
+ *
4
+ * `record_usage` / `POST /RecordUsage` accept both singular `memoryId` and
5
+ * plural `memoryIds`. MERGE, not prefer: a caller who supplies both means
6
+ * both. Preferring `memoryIds` (`data?.memoryIds ?? [data?.memoryId]`)
7
+ * silently dropped the singular id — quiet data loss, same class as
8
+ * #1206 / #1371.
9
+ *
10
+ * Pure: no Harper. Native `/mcp` (`resources/mcp-tools.ts`) and the HTTP
11
+ * endpoint (`resources/RecordUsage.ts`) both call this so the credited set
12
+ * does not depend on every client flattening first. Stdio `flair-mcp`
13
+ * merges independently in `buildRecordUsageBody`; the conformance test
14
+ * pins the two surfaces to the same set.
15
+ */
16
+ /** Stated on both MCP tool schemas so a caller can predict the merge without reading source. */
17
+ export const RECORD_USAGE_ID_MERGE_CONTRACT = "When both memoryId and memoryIds are supplied they are merged (union, then deduped) — a caller who passes both means both.";
18
+ /**
19
+ * Union `memoryId` + `memoryIds`, then dedupe. Empty / non-string entries
20
+ * in the plural list are skipped (same filter as stdio `buildRecordUsageBody`).
21
+ * Used by native `/mcp` to flatten the tool args before `RecordUsage.post`.
22
+ */
23
+ export function unionUsageMemoryIds(memoryId, memoryIds) {
24
+ const ids = [];
25
+ if (Array.isArray(memoryIds)) {
26
+ for (const id of memoryIds) {
27
+ if (typeof id === "string" && id.length > 0)
28
+ ids.push(id);
29
+ }
30
+ }
31
+ if (typeof memoryId === "string" && memoryId.length > 0) {
32
+ ids.push(memoryId);
33
+ }
34
+ return [...new Set(ids)];
35
+ }
36
+ /**
37
+ * HTTP-endpoint resolver: union both fields, then apply RecordUsage.post()'s
38
+ * existing validation (non-empty strings, per-call cap on the unique credited
39
+ * set). Cap after dedupe so an overlapping `memoryId` does not 400 a legal
40
+ * unique set that native `/mcp` and stdio already accept. A present-but-invalid
41
+ * `memoryIds` still 400s. The max-20 anti-gaming bound is unchanged — it
42
+ * limits unique ids credited, not raw concatenation length.
43
+ */
44
+ export function resolveRecordUsageIds(data, maxIds) {
45
+ const memoryIds = data?.memoryIds;
46
+ const memoryId = data?.memoryId;
47
+ if (memoryIds != null) {
48
+ if (!Array.isArray(memoryIds) || !memoryIds.every((id) => typeof id === "string" && id.length > 0)) {
49
+ return { ok: false, error: "invalid" };
50
+ }
51
+ }
52
+ const raw = [];
53
+ if (Array.isArray(memoryIds))
54
+ raw.push(...memoryIds);
55
+ if (typeof memoryId === "string" && memoryId.length > 0)
56
+ raw.push(memoryId);
57
+ if (raw.length === 0)
58
+ return { ok: false, error: "empty" };
59
+ const ids = [...new Set(raw)];
60
+ if (ids.length > maxIds)
61
+ return { ok: false, error: "cap" };
62
+ return { ok: true, ids };
63
+ }