@tpsdev-ai/flair 0.49.0 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -4
- package/dist/bridges/runtime/roundtrip.js +91 -2
- package/dist/build-info.json +3 -3
- package/dist/cli.js +565 -171
- package/dist/deploy.js +20 -3
- package/dist/doctor-client.js +62 -34
- package/dist/federation/scheduler.js +24 -3
- package/dist/hook-install.js +45 -13
- package/dist/install/clients.js +28 -17
- package/dist/lib/doctor-run.js +481 -0
- package/dist/lib/launchd-management.js +7 -26
- package/dist/lib/scheduler-platform.js +132 -10
- package/dist/lib/scratch-owner.js +49 -0
- package/dist/rem/scheduler.js +23 -5
- package/dist/resources/Federation.js +42 -20
- package/dist/resources/MemoryBootstrap.js +8 -4
- package/dist/resources/RecordUsage.js +13 -6
- package/dist/resources/SemanticSearch.js +8 -1
- package/dist/resources/federation-classify.js +90 -0
- package/dist/resources/health.js +51 -7
- package/dist/resources/mcp-tools.js +10 -6
- package/dist/resources/search-readiness.js +123 -0
- package/dist/resources/semantic-retrieval-core.js +48 -21
- package/dist/resources/sort-comparators.js +45 -0
- package/dist/resources/usage-ids.js +63 -0
- package/dist/src/lib/scheduler-platform.js +132 -10
- package/dist/src/rem/scheduler.js +23 -5
- package/docs/auth.md +5 -0
- package/docs/deepseek-harness.md +1 -1
- package/docs/federation.md +11 -0
- package/docs/hosted-on-fabric.md +2 -0
- package/docs/integrations.md +53 -1
- package/docs/mcp-clients.md +67 -15
- package/docs/quickstart-fabric.md +1 -1
- package/docs/supply-chain-policy.md +1 -1
- package/docs/troubleshooting.md +25 -0
- package/docs/upgrade.md +17 -1
- package/package.json +3 -3
- package/schemas/federation.graphql +1 -1
package/dist/resources/health.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Resource, databases } from "harper";
|
|
1
|
+
import { Resource, databases, server, logger } from "harper";
|
|
2
2
|
import { promises as fsp, existsSync, readFileSync } from "node:fs";
|
|
3
3
|
import { homedir, platform } from "node:os";
|
|
4
4
|
import { join, dirname } from "node:path";
|
|
@@ -8,6 +8,9 @@ import { resolveBuildInfo } from "./build-info.js";
|
|
|
8
8
|
import { getMigrationStatusSnapshot } from "./migrations/status.js";
|
|
9
9
|
import { resolveMigrationDataDirForRead } from "./migrations/data-dir.js";
|
|
10
10
|
import { REM_DEDUP_STATS_PATH } from "./dedup-cluster.js";
|
|
11
|
+
import { hybridEnabled } from "./bm25.js";
|
|
12
|
+
import { bm25IndexEnabled, bm25IndexStatus } from "./bm25-index-service.js";
|
|
13
|
+
import { buildPublicHealthBody, resolveSearchReadiness } from "./search-readiness.js";
|
|
11
14
|
const db = databases;
|
|
12
15
|
const redactHome = (p) => {
|
|
13
16
|
const home = homedir();
|
|
@@ -52,7 +55,7 @@ function resolveVersion() {
|
|
|
52
55
|
return process.env.npm_package_version ?? "dev";
|
|
53
56
|
}
|
|
54
57
|
/**
|
|
55
|
-
* Health endpoint — truly public
|
|
58
|
+
* Health endpoint — truly public. Identity-free; no auth, no role check.
|
|
56
59
|
*
|
|
57
60
|
* `allowRead() { return true }` opens Harper's role gate for anonymous GETs,
|
|
58
61
|
* which is what makes /Health work for callers outside `authorizeLocal`'s
|
|
@@ -66,8 +69,15 @@ function resolveVersion() {
|
|
|
66
69
|
*
|
|
67
70
|
* Same pattern as `FederationPair.allowCreate(){ return true }` (PR #299):
|
|
68
71
|
* declare the Resource anonymously-accessible at the Harper layer; let the
|
|
69
|
-
* handler itself enforce whatever it needs
|
|
70
|
-
*
|
|
72
|
+
* handler itself enforce whatever it needs.
|
|
73
|
+
*
|
|
74
|
+
* flair#1326: `ok: true` used to mean only "this resource answered." That
|
|
75
|
+
* is a green light that lies when search routes are not mounted yet, or
|
|
76
|
+
* when the hybrid BM25 index is still cold (first search after restart
|
|
77
|
+
* scans the corpus; the lag grows with store size). `searchReady` is
|
|
78
|
+
* always present. When search cannot be served at all, this endpoint
|
|
79
|
+
* returns HTTP 503 and `ok: false`. When the process is live but recall
|
|
80
|
+
* is still cold, it stays 200 and names the lag on `searchReadyReason`.
|
|
71
81
|
*
|
|
72
82
|
* Rich stats (memory counts, agent names, etc.) are behind /HealthDetail
|
|
73
83
|
* which requires authentication. This prevents information leakage on
|
|
@@ -96,13 +106,35 @@ export class Health extends Resource {
|
|
|
96
106
|
// a 40-hex sha when the build ran in a git work tree, an honest null
|
|
97
107
|
// otherwise (tarball builds) — never omitted, never fabricated (Sherlock).
|
|
98
108
|
const build = resolveBuildInfo();
|
|
99
|
-
|
|
100
|
-
|
|
109
|
+
const readiness = currentSearchReadiness();
|
|
110
|
+
const body = buildPublicHealthBody(readiness, {
|
|
101
111
|
version: build?.version ?? resolveVersion(),
|
|
102
112
|
buildCommit: build?.commit ?? null,
|
|
103
|
-
};
|
|
113
|
+
});
|
|
114
|
+
if (readiness.status !== 200) {
|
|
115
|
+
return new Response(JSON.stringify(body), {
|
|
116
|
+
status: readiness.status,
|
|
117
|
+
headers: { "content-type": "application/json" },
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
return body;
|
|
104
121
|
}
|
|
105
122
|
}
|
|
123
|
+
/** Same sources /Health and /HealthDetail consult so they cannot disagree. */
|
|
124
|
+
export function currentSearchReadiness() {
|
|
125
|
+
// Fail-open when the registry is missing (Sherlock on #1406 / flair#1411):
|
|
126
|
+
// do not 503 forever. resolveSearchReadiness warns once and names the
|
|
127
|
+
// degradation; we do not treat "registry should always be here" as a given.
|
|
128
|
+
const resources = server.resources ?? null;
|
|
129
|
+
return resolveSearchReadiness({
|
|
130
|
+
resources,
|
|
131
|
+
memoryTable: db.flair?.Memory,
|
|
132
|
+
bm25: bm25IndexStatus(),
|
|
133
|
+
hybridEnabled: hybridEnabled(),
|
|
134
|
+
bm25IndexEnabled: bm25IndexEnabled(),
|
|
135
|
+
warn: (message) => { logger.warn?.(message); },
|
|
136
|
+
});
|
|
137
|
+
}
|
|
106
138
|
/**
|
|
107
139
|
* Authenticated health detail — returns memory/agent/soul stats + process info.
|
|
108
140
|
* Requires Ed25519 agent auth or admin basic auth.
|
|
@@ -127,6 +159,18 @@ export class HealthDetail extends Resource {
|
|
|
127
159
|
const stats = { ok: true };
|
|
128
160
|
const nowMs = Date.now();
|
|
129
161
|
const warnings = [];
|
|
162
|
+
// flair#1326: same search-ready signal as public /Health. HealthDetail
|
|
163
|
+
// stays HTTP 200 (it is a stats dump, not a traffic gate); the field
|
|
164
|
+
// and a warning name the lag so `flair status` / operators can see it.
|
|
165
|
+
const readiness = currentSearchReadiness();
|
|
166
|
+
stats.searchReady = readiness.searchReady;
|
|
167
|
+
// Public/detail shape is unchanged: searchReadyReason stays a lag signal
|
|
168
|
+
// (present iff !searchReady). Ready-path verification constants stay on
|
|
169
|
+
// the decision object (flair#1411).
|
|
170
|
+
if (!readiness.searchReady && readiness.searchReadyReason) {
|
|
171
|
+
stats.searchReadyReason = readiness.searchReadyReason;
|
|
172
|
+
warnings.push({ level: "warn", message: readiness.searchReadyReason });
|
|
173
|
+
}
|
|
130
174
|
const ctx = this.getContext?.();
|
|
131
175
|
// #614 fix: resolve identity via the shared three-way verdict
|
|
132
176
|
// (internal/agent/anonymous — agent-auth.ts) instead of reading
|
|
@@ -37,6 +37,7 @@
|
|
|
37
37
|
*/
|
|
38
38
|
import { resolveVersion } from "./version.js";
|
|
39
39
|
import { agentContext, adminContext, collectionResource } from "./in-process.js";
|
|
40
|
+
import { RECORD_USAGE_ID_MERGE_CONTRACT, unionUsageMemoryIds } from "./usage-ids.js";
|
|
40
41
|
const H = {};
|
|
41
42
|
const LOADERS = {
|
|
42
43
|
SemanticSearch: async () => (await import("./SemanticSearch.js")).SemanticSearch,
|
|
@@ -538,9 +539,10 @@ async function attention(agent, args) {
|
|
|
538
539
|
async function recordUsage(agent, args) {
|
|
539
540
|
const Cls = await handler("RecordUsage");
|
|
540
541
|
const h = new Cls(undefined, delegationContext(agent));
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
542
|
+
// flair#1410: MERGE, do not prefer. The previous ternary dropped
|
|
543
|
+
// `memoryId` whenever `memoryIds` was an array (including `[]`).
|
|
544
|
+
const merged = unionUsageMemoryIds(args?.memoryId, args?.memoryIds);
|
|
545
|
+
const memoryIds = merged.length > 0 ? merged : undefined;
|
|
544
546
|
return unwrap(await h.post({ memoryIds, attribution: args?.attribution }));
|
|
545
547
|
}
|
|
546
548
|
/**
|
|
@@ -811,6 +813,7 @@ export const TOOLS = {
|
|
|
811
813
|
{ count: "memoriesIncluded", containers: ["memories", "predicted"] }, // #1199
|
|
812
814
|
{ count: "teammateFindingsIncluded", containers: ["teammateFindings"] }, // #1199
|
|
813
815
|
{ count: "sections.events", containers: ["events"] }, // #1206
|
|
816
|
+
{ count: "sections.soul", containers: ["soul"] }, // #1371
|
|
814
817
|
],
|
|
815
818
|
// present + typed even when empty — never a bare {} / missing key (#1182).
|
|
816
819
|
selfDescribingEmpty: [
|
|
@@ -1004,12 +1007,13 @@ export const TOOLS = {
|
|
|
1004
1007
|
name: "record_usage",
|
|
1005
1008
|
description: "Report that one or more memories were actually USED — cited or relied on to ground an answer or decision. " +
|
|
1006
1009
|
"Distinct from search (surfacing a memory is not usage). Drives the recall-quality usage signal; dedup'd " +
|
|
1007
|
-
"(you can only count once per memory) and rate-limited."
|
|
1010
|
+
"(you can only count once per memory) and rate-limited. " +
|
|
1011
|
+
RECORD_USAGE_ID_MERGE_CONTRACT,
|
|
1008
1012
|
inputSchema: {
|
|
1009
1013
|
type: "object",
|
|
1010
1014
|
properties: {
|
|
1011
|
-
memoryIds: { type: "array", items: { type: "string" }, description: "IDs of the memories that were used (max 20 per call)" },
|
|
1012
|
-
memoryId: { type: "string", description: "Convenience alias for a single memory id
|
|
1015
|
+
memoryIds: { type: "array", items: { type: "string" }, description: "IDs of the memories that were used (max 20 per call). Merged with memoryId when both are supplied." },
|
|
1016
|
+
memoryId: { type: "string", description: "Convenience alias for a single memory id. Merged with memoryIds when both are supplied — not dropped." },
|
|
1013
1017
|
attribution: { type: "string", description: "Optional free-text note on what used it (opaque — stored for audit only, max 500 chars)" },
|
|
1014
1018
|
},
|
|
1015
1019
|
},
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* search-readiness.ts — the honest /Health search-ready signal (flair#1326).
|
|
3
|
+
*
|
|
4
|
+
* Harper's process can answer /health (and Flair's /Health resource can
|
|
5
|
+
* answer {ok:true}) while search is still unusable:
|
|
6
|
+
*
|
|
7
|
+
* 1. Boot window — jsResources register incrementally. /Health can be up
|
|
8
|
+
* while /Memory and /SemanticSearch still 404 from Harper's catch-all
|
|
9
|
+
* (documented in packages/adk-flair-js/test/helpers/boot-harper.mjs).
|
|
10
|
+
* 2. Cold BM25 index — hybrid retrieval's persistent index is lazy-built
|
|
11
|
+
* on the first search, not at component start (bm25-index-service.ts).
|
|
12
|
+
* That first-query corpus scan grows with store size. /Health answering
|
|
13
|
+
* is not "recall is warm."
|
|
14
|
+
*
|
|
15
|
+
* This module is Harper-free so the decision is unit-testable against the
|
|
16
|
+
* shipped function. Callers inject the registry / table / index status they
|
|
17
|
+
* already have.
|
|
18
|
+
*
|
|
19
|
+
* Two layers, on purpose:
|
|
20
|
+
* - Routes/table not mounted → not healthy (ok:false, HTTP 503). A
|
|
21
|
+
* traffic-gating probe that only looks at status must not get a green
|
|
22
|
+
* light for a node whose search routes are not serving.
|
|
23
|
+
* - Routes up but index still cold → process is live (ok:true, HTTP 200)
|
|
24
|
+
* and searchReady:false names the lag. We do NOT 503 on a cold index:
|
|
25
|
+
* the index builds on the first search, and a health check must not
|
|
26
|
+
* trigger that scan (bm25-index-service.ts: "Eager building would add a
|
|
27
|
+
* full corpus scan to every boot including … health checks"). 503-until-
|
|
28
|
+
* warm would deadlock — health waits for the index, the index waits for
|
|
29
|
+
* a search that never comes.
|
|
30
|
+
*/
|
|
31
|
+
/**
|
|
32
|
+
* How a ready result was verified. Constant strings — no interpolation
|
|
33
|
+
* (flair#1411). Public /Health still omits this field when searchReady is
|
|
34
|
+
* true; the values live on the decision object so a caller of
|
|
35
|
+
* resolveSearchReadiness can tell registry-verified from table-only.
|
|
36
|
+
*/
|
|
37
|
+
export const SEARCH_READY_REASON_VERIFIED_VIA_ROUTE_REGISTRY = "verified via route registry";
|
|
38
|
+
export const SEARCH_READY_REASON_REGISTRY_UNAVAILABLE_TABLE_ONLY = "registry unavailable, table check only";
|
|
39
|
+
/** Once-warn when the route-mount check is skipped (flair#1411). */
|
|
40
|
+
export const MISSING_REGISTRY_WARN = "search route-mount verification is degraded; searchReady now rests on the table check alone";
|
|
41
|
+
let warnedMissingRegistry = false;
|
|
42
|
+
/** Test-only: forget the one-shot missing-registry warning. */
|
|
43
|
+
export function _resetMissingRegistryWarnForTests() {
|
|
44
|
+
warnedMissingRegistry = false;
|
|
45
|
+
}
|
|
46
|
+
function routeMounted(resources, name) {
|
|
47
|
+
const entry = resources.get?.(name) ?? resources.getMatch?.(name);
|
|
48
|
+
return Boolean(entry?.Resource);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Decide whether search is actually usable, and whether /Health should claim
|
|
52
|
+
* the process is healthy.
|
|
53
|
+
*
|
|
54
|
+
* `resources` is optional. Stated fail-open (Sherlock on #1406 / flair#1411):
|
|
55
|
+
* when the registry is missing we skip the route-mount check rather than
|
|
56
|
+
* 503 forever. A table handle can exist while `/Memory` and `/SemanticSearch`
|
|
57
|
+
* still 404, so this is weaker than the primary defense. We do not fail
|
|
58
|
+
* closed. We warn once and name the degradation on `searchReadyReason`.
|
|
59
|
+
*/
|
|
60
|
+
export function resolveSearchReadiness(opts) {
|
|
61
|
+
let readyReason = SEARCH_READY_REASON_REGISTRY_UNAVAILABLE_TABLE_ONLY;
|
|
62
|
+
if (opts.resources) {
|
|
63
|
+
const memoryMounted = routeMounted(opts.resources, "Memory");
|
|
64
|
+
const searchMounted = routeMounted(opts.resources, "SemanticSearch");
|
|
65
|
+
if (!memoryMounted || !searchMounted) {
|
|
66
|
+
const missing = [
|
|
67
|
+
!memoryMounted ? "Memory" : null,
|
|
68
|
+
!searchMounted ? "SemanticSearch" : null,
|
|
69
|
+
].filter(Boolean).join(", ");
|
|
70
|
+
return notServing(`search routes not mounted (${missing})`);
|
|
71
|
+
}
|
|
72
|
+
readyReason = SEARCH_READY_REASON_VERIFIED_VIA_ROUTE_REGISTRY;
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
if (!warnedMissingRegistry) {
|
|
76
|
+
warnedMissingRegistry = true;
|
|
77
|
+
opts.warn?.(MISSING_REGISTRY_WARN);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (!opts.memoryTable || typeof opts.memoryTable.search !== "function") {
|
|
81
|
+
return notServing("memory table not queryable");
|
|
82
|
+
}
|
|
83
|
+
// Hybrid + the persistent index are default-on. A cold or in-flight BM25
|
|
84
|
+
// index means the first search will pay a full corpus scan — the #1326
|
|
85
|
+
// lag. Name it; do not fail liveness.
|
|
86
|
+
//
|
|
87
|
+
// `disabled` (feed/build failure) and `FLAIR_BM25_INDEX=false` both fall
|
|
88
|
+
// back to the per-query scan. The kill switch never calls ensureReady, so
|
|
89
|
+
// status stays `empty` for the life of the process — that is serving, not
|
|
90
|
+
// cold. Treating it as lag would make searchReady false forever and refuse
|
|
91
|
+
// a node that is already answering recall.
|
|
92
|
+
const indexInPath = opts.hybridEnabled !== false && opts.bm25IndexEnabled !== false;
|
|
93
|
+
if (indexInPath && opts.bm25) {
|
|
94
|
+
if (opts.bm25.state === "building") {
|
|
95
|
+
return namesLag("bm25 index building — first search is still scanning the corpus");
|
|
96
|
+
}
|
|
97
|
+
if (opts.bm25.state === "empty") {
|
|
98
|
+
return namesLag("bm25 index not built (cold boot; first search scans the corpus)");
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return { searchReady: true, ok: true, status: 200, searchReadyReason: readyReason };
|
|
102
|
+
}
|
|
103
|
+
function notServing(searchReadyReason) {
|
|
104
|
+
return { searchReady: false, ok: false, status: 503, searchReadyReason };
|
|
105
|
+
}
|
|
106
|
+
function namesLag(searchReadyReason) {
|
|
107
|
+
return { searchReady: false, ok: true, status: 200, searchReadyReason };
|
|
108
|
+
}
|
|
109
|
+
/** Public /Health JSON body. `searchReady` is always present (never omitted). */
|
|
110
|
+
export function buildPublicHealthBody(readiness, identity) {
|
|
111
|
+
const body = {
|
|
112
|
+
ok: readiness.ok,
|
|
113
|
+
version: identity.version,
|
|
114
|
+
buildCommit: identity.buildCommit,
|
|
115
|
+
searchReady: readiness.searchReady,
|
|
116
|
+
};
|
|
117
|
+
// Public shape is unchanged: searchReadyReason is still present iff !searchReady.
|
|
118
|
+
// Ready-path verification constants stay on the decision object (flair#1411).
|
|
119
|
+
if (!readiness.searchReady && readiness.searchReadyReason) {
|
|
120
|
+
body.searchReadyReason = readiness.searchReadyReason;
|
|
121
|
+
}
|
|
122
|
+
return body;
|
|
123
|
+
}
|
|
@@ -34,9 +34,11 @@
|
|
|
34
34
|
// multiplies `limit` internally; any overfetch policy — SemanticSearch's
|
|
35
35
|
// CANDIDATE_MULTIPLIER, MemoryBootstrap's K formula — is the CALLER's
|
|
36
36
|
// decision, made
|
|
37
|
-
// before calling in).
|
|
38
|
-
// legacy HNSW-only vs. keyword-only fallback) produced
|
|
39
|
-
// output shape is identical regardless of `hybrid`.
|
|
37
|
+
// before calling in). The result array never annotates which internal leg
|
|
38
|
+
// (BM25+RRF hybrid vs. legacy HNSW-only vs. keyword-only fallback) produced
|
|
39
|
+
// a given row — that output shape is identical regardless of `hybrid`.
|
|
40
|
+
// Per-leg membership is available only through the opt-in `onLegs`
|
|
41
|
+
// callback (flair#1358); unset ⇒ the return is byte-identical to before.
|
|
40
42
|
//
|
|
41
43
|
// ── SCORE CONTRACT (flair#985) ───────────────────────────────────────────────
|
|
42
44
|
// `_score` under `scoring:"raw"` is ALWAYS an ABSOLUTE similarity (cosine of
|
|
@@ -60,6 +62,15 @@ import { compositeScore } from "./scoring.js";
|
|
|
60
62
|
import { buildBM25, fuseRrfNormalized, SEM_LIMIT } from "./bm25.js";
|
|
61
63
|
import { isAllowedBm25Candidate } from "./bm25-filter.js";
|
|
62
64
|
import { indexedBm25Ids } from "./bm25-index-service.js";
|
|
65
|
+
import { byRecencyThenId } from "./sort-comparators.js";
|
|
66
|
+
/**
|
|
67
|
+
* The only way a row enters the ranked pool. `_rank` is a required
|
|
68
|
+
* argument — not an object property that `...any` can swallow — so a
|
|
69
|
+
* push site that omits it is a compile error (flair#1415).
|
|
70
|
+
*/
|
|
71
|
+
function pushRanked(rows, row, _rank) {
|
|
72
|
+
rows.push({ ...row, _rank });
|
|
73
|
+
}
|
|
63
74
|
// Convert HNSW cosine distance (1 - similarity) to similarity score.
|
|
64
75
|
function distanceToSimilarity(distance) {
|
|
65
76
|
return 1 - distance;
|
|
@@ -86,10 +97,12 @@ export const DEFAULT_SELECT = ["id", "agentId", "content", "contentHash", "visib
|
|
|
86
97
|
"parentId", "derivedFrom", "sessionId", "lastReflected", "supersedes", "subject", "summary",
|
|
87
98
|
"validFrom", "validTo", "_safetyFlags"];
|
|
88
99
|
export async function retrieveCandidates(params) {
|
|
89
|
-
const { queryEmbedding: qEmb, q, conditions, limit, select = DEFAULT_SELECT, includeSuperseded = false, scoring = "raw", temporalBoost = 1.0, sinceDate = null, asOf, minScore = 0, agentId, isAllowed, hybrid, ctx, withSemSimilarity = false, } = params;
|
|
100
|
+
const { queryEmbedding: qEmb, q, conditions, limit, select = DEFAULT_SELECT, includeSuperseded = false, scoring = "raw", temporalBoost = 1.0, sinceDate = null, asOf, minScore = 0, agentId, isAllowed, hybrid, ctx, withSemSimilarity = false, onLegs, } = params;
|
|
90
101
|
const passesAllowed = (record) => !isAllowed || isAllowed(record);
|
|
91
102
|
const hnswSelect = [...select, "$distance"];
|
|
92
103
|
const results = [];
|
|
104
|
+
const hnswLegIds = [];
|
|
105
|
+
const bm25LegIds = [];
|
|
93
106
|
if (hybrid) {
|
|
94
107
|
// ─── BM25 + union-RRF hybrid path ────────────────────────────────────
|
|
95
108
|
// 1. Semantic candidates via HNSW (unchanged fetch). 2. BM25 lexical pass
|
|
@@ -153,6 +166,7 @@ export async function retrieveCandidates(params) {
|
|
|
153
166
|
}
|
|
154
167
|
semRecords.push(record);
|
|
155
168
|
semIds.push(record.id);
|
|
169
|
+
hnswLegIds.push(record.id);
|
|
156
170
|
}
|
|
157
171
|
}
|
|
158
172
|
// ── (b) The BM25 lexical leg ─────────────────────────────────────────
|
|
@@ -259,6 +273,8 @@ export async function retrieveCandidates(params) {
|
|
|
259
273
|
}
|
|
260
274
|
bm25Ids = resolved;
|
|
261
275
|
}
|
|
276
|
+
if (q)
|
|
277
|
+
bm25LegIds.push(...bm25Ids);
|
|
262
278
|
// ── (d) No retrieval signal at all → full scoped listing ────────────
|
|
263
279
|
if (!q && !qEmb) {
|
|
264
280
|
for (const record of allowedById.values()) {
|
|
@@ -268,14 +284,13 @@ export async function retrieveCandidates(params) {
|
|
|
268
284
|
finalScore *= temporalBoost;
|
|
269
285
|
const isFlagged = record._safetyFlags && Array.isArray(record._safetyFlags) && record._safetyFlags.length > 0;
|
|
270
286
|
const source = record.agentId !== agentId ? record.agentId : undefined;
|
|
271
|
-
results
|
|
287
|
+
pushRanked(results, {
|
|
272
288
|
...record,
|
|
273
289
|
content: isFlagged ? wrapUntrusted(record.content, source) : record.content,
|
|
274
290
|
_score: Math.round(finalScore * 1000) / 1000,
|
|
275
291
|
_rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
|
|
276
292
|
_source: source,
|
|
277
|
-
|
|
278
|
-
});
|
|
293
|
+
}, finalScore);
|
|
279
294
|
}
|
|
280
295
|
}
|
|
281
296
|
else {
|
|
@@ -323,22 +338,21 @@ export async function retrieveCandidates(params) {
|
|
|
323
338
|
finalScore *= temporalBoost;
|
|
324
339
|
const isFlagged = record._safetyFlags && Array.isArray(record._safetyFlags) && record._safetyFlags.length > 0;
|
|
325
340
|
const source = record.agentId !== agentId ? record.agentId : undefined;
|
|
326
|
-
|
|
341
|
+
// Ordering key: fused rank for raw mode; composite value for
|
|
342
|
+
// composite mode (composite ordering is unchanged by #985 — its
|
|
343
|
+
// rrfRaw input and result order are exactly the pre-#985 behavior).
|
|
344
|
+
pushRanked(results, {
|
|
327
345
|
...record,
|
|
328
346
|
content: isFlagged ? wrapUntrusted(record.content, source) : record.content,
|
|
329
347
|
_score: Math.round(finalScore * 1000) / 1000,
|
|
330
348
|
_rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
|
|
331
349
|
_source: source,
|
|
332
|
-
// Ordering key: fused rank for raw mode; composite value for
|
|
333
|
-
// composite mode (composite ordering is unchanged by #985 — its
|
|
334
|
-
// rrfRaw input and result order are exactly the pre-#985 behavior).
|
|
335
|
-
_rank: scoring === "raw" ? rrfRaw : finalScore,
|
|
336
350
|
// flair#744 slice 2: the opt-in absolute-confidence field for the
|
|
337
351
|
// abstention decision. Attach remains OPT-IN so non-abstain
|
|
338
352
|
// responses stay byte-identical (the capture above is now
|
|
339
353
|
// unconditional, but the response field is not).
|
|
340
354
|
...(withSemSimilarity && semSim !== undefined ? { _semSimilarity: semSim } : {}),
|
|
341
|
-
});
|
|
355
|
+
}, scoring === "raw" ? rrfRaw : finalScore);
|
|
342
356
|
}
|
|
343
357
|
}
|
|
344
358
|
}
|
|
@@ -400,15 +414,15 @@ export async function retrieveCandidates(params) {
|
|
|
400
414
|
// flair#744 slice 2: the absolute cosine (`semanticScore`, pre keyword
|
|
401
415
|
// bump) is the abstention confidence signal on this legacy/bootstrap
|
|
402
416
|
// (HNSW-leg-only) path.
|
|
403
|
-
results
|
|
417
|
+
pushRanked(results, {
|
|
404
418
|
...rest,
|
|
405
419
|
content: isFlagged ? wrapUntrusted(rest.content, source) : rest.content,
|
|
406
420
|
_score: Math.round(finalScore * 1000) / 1000,
|
|
407
421
|
_rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
|
|
408
422
|
_source: source,
|
|
409
|
-
_rank: finalScore,
|
|
410
423
|
...(withSemSimilarity ? { _semSimilarity: semanticScore } : {}),
|
|
411
|
-
});
|
|
424
|
+
}, finalScore);
|
|
425
|
+
hnswLegIds.push(record.id);
|
|
412
426
|
}
|
|
413
427
|
}
|
|
414
428
|
else {
|
|
@@ -446,14 +460,13 @@ export async function retrieveCandidates(params) {
|
|
|
446
460
|
finalScore *= temporalBoost;
|
|
447
461
|
const isFlagged = rest._safetyFlags && Array.isArray(rest._safetyFlags) && rest._safetyFlags.length > 0;
|
|
448
462
|
const source = record.agentId !== agentId ? record.agentId : undefined;
|
|
449
|
-
results
|
|
463
|
+
pushRanked(results, {
|
|
450
464
|
...rest,
|
|
451
465
|
content: isFlagged ? wrapUntrusted(rest.content, source) : rest.content,
|
|
452
466
|
_score: Math.round(finalScore * 1000) / 1000,
|
|
453
467
|
_rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
|
|
454
468
|
_source: source,
|
|
455
|
-
|
|
456
|
-
});
|
|
469
|
+
}, finalScore);
|
|
457
470
|
}
|
|
458
471
|
}
|
|
459
472
|
// Build superseded set and filter (unless caller opts in to see full
|
|
@@ -479,13 +492,27 @@ export async function retrieveCandidates(params) {
|
|
|
479
492
|
}
|
|
480
493
|
// Order by the internal ranking key (fused RRF rank on the hybrid raw path;
|
|
481
494
|
// identical to `_score` everywhere else), then strip it — `_rank` is an
|
|
482
|
-
// ordering key, never part of the response shape.
|
|
495
|
+
// ordering key, never part of the response shape. Required on
|
|
496
|
+
// RetrievalRankedRow / pushRanked (flair#1415) so a missing value cannot
|
|
497
|
+
// silently NaN this comparator into the recency tail. Note the hybrid raw
|
|
483
498
|
// ordering is deliberately NOT by `_score`: the recall win of hybrid
|
|
484
499
|
// retrieval lives in the fused ORDER (a BM25 rank-1 rescue outranks weak
|
|
485
500
|
// semantic hits), while `_score` carries the honest absolute evidence for
|
|
486
501
|
// each result — the two can disagree, and that is correct.
|
|
487
|
-
|
|
502
|
+
//
|
|
503
|
+
// Ties at this sort are cross-leg RRF identities (flair#1412): no
|
|
504
|
+
// within-leg tie-break can prevent `1/(k+3)+1/(k+5) === 1/(k+5)+1/(k+3)`.
|
|
505
|
+
// Break them by createdAt DESC (null → oldest, never NaN) then id ASC.
|
|
506
|
+
// Determinism is the id ASC total order. createdAt is best-effort recency
|
|
507
|
+
// within an exact `_rank` tie — federated writer clocks can skew, and
|
|
508
|
+
// that cannot reintroduce nondeterminism (see byRecencyThenId).
|
|
509
|
+
filteredResults.sort((a, b) => (b._rank - a._rank) || byRecencyThenId(a, b));
|
|
488
510
|
for (const r of filteredResults)
|
|
489
511
|
delete r._rank;
|
|
512
|
+
onLegs?.({
|
|
513
|
+
hnsw: hnswLegIds,
|
|
514
|
+
bm25: bm25LegIds,
|
|
515
|
+
fused: filteredResults.map((r) => r.id),
|
|
516
|
+
});
|
|
490
517
|
return filteredResults;
|
|
491
518
|
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic sort comparators (flair#1412).
|
|
3
|
+
*
|
|
4
|
+
* Retrieval and selection sorts used to key on one number (score, rank,
|
|
5
|
+
* priority) and then inherit `Array.prototype.sort`'s input order on ties.
|
|
6
|
+
* That input order is Harper scan order, which is free to change across
|
|
7
|
+
* restarts. The family below makes the tie-break the default thing to
|
|
8
|
+
* reach for: each helper has its own primary key, and they share the
|
|
9
|
+
* `compareKey` tail — not one comparator forced onto every site.
|
|
10
|
+
*
|
|
11
|
+
* Do not collapse this into a single `byScoreThenId`. cosine.ts keys on
|
|
12
|
+
* corpus `.index`; MemoryBootstrap (flair#1409, not this PR) keys on the
|
|
13
|
+
* soul `key`. The shared part is the tail.
|
|
14
|
+
*
|
|
15
|
+
* Locale-independent: `compareKey` is code-unit / numeric `<`/`>`, never
|
|
16
|
+
* `localeCompare`. ISO-8601 UTC strings compare chronologically that way.
|
|
17
|
+
*/
|
|
18
|
+
/** Total-order tail. Works for ids, corpus indexes, soul keys. */
|
|
19
|
+
export function compareKey(a, b) {
|
|
20
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
21
|
+
}
|
|
22
|
+
/** Score / rank descending, then `id` ascending. */
|
|
23
|
+
export function byNumberDescThenId(getNumber) {
|
|
24
|
+
return (a, b) => (getNumber(b) - getNumber(a)) || compareKey(a.id, b.id);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* `createdAt` DESC (null → oldest, never NaN) then `id` ASC.
|
|
28
|
+
*
|
|
29
|
+
* Determinism comes from `id` ASC — ids are unique, so this is a total
|
|
30
|
+
* order no matter what `createdAt` does. Do not date-arithmetic a
|
|
31
|
+
* possibly-null field: `new Date(null).getTime()` is 0, but subtracting
|
|
32
|
+
* NaN (unparseable / missing-after-coercion) is undefined behaviour for
|
|
33
|
+
* `Array.prototype.sort`. Empty-string fallback keeps the comparison in
|
|
34
|
+
* string space; an empty value is less than any ISO-8601 UTC timestamp,
|
|
35
|
+
* so it sorts last under DESC.
|
|
36
|
+
*
|
|
37
|
+
* Clock-skew caveat: `createdAt` is writer-stamped. Across federated
|
|
38
|
+
* writers this is best-effort recency within an exact `_rank` tie, not a
|
|
39
|
+
* correctness claim. Skew can misorder rows the ranker already called
|
|
40
|
+
* equivalent; it cannot reintroduce nondeterminism, because `id` ASC
|
|
41
|
+
* still resolves every remaining tie.
|
|
42
|
+
*/
|
|
43
|
+
export function byRecencyThenId(a, b) {
|
|
44
|
+
return compareKey(b.createdAt ?? "", a.createdAt ?? "") || compareKey(a.id, b.id);
|
|
45
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* usage-ids.ts — which memory ids a usage-feedback call credits (flair#1410).
|
|
3
|
+
*
|
|
4
|
+
* `record_usage` / `POST /RecordUsage` accept both singular `memoryId` and
|
|
5
|
+
* plural `memoryIds`. MERGE, not prefer: a caller who supplies both means
|
|
6
|
+
* both. Preferring `memoryIds` (`data?.memoryIds ?? [data?.memoryId]`)
|
|
7
|
+
* silently dropped the singular id — quiet data loss, same class as
|
|
8
|
+
* #1206 / #1371.
|
|
9
|
+
*
|
|
10
|
+
* Pure: no Harper. Native `/mcp` (`resources/mcp-tools.ts`) and the HTTP
|
|
11
|
+
* endpoint (`resources/RecordUsage.ts`) both call this so the credited set
|
|
12
|
+
* does not depend on every client flattening first. Stdio `flair-mcp`
|
|
13
|
+
* merges independently in `buildRecordUsageBody`; the conformance test
|
|
14
|
+
* pins the two surfaces to the same set.
|
|
15
|
+
*/
|
|
16
|
+
/** Stated on both MCP tool schemas so a caller can predict the merge without reading source. */
|
|
17
|
+
export const RECORD_USAGE_ID_MERGE_CONTRACT = "When both memoryId and memoryIds are supplied they are merged (union, then deduped) — a caller who passes both means both.";
|
|
18
|
+
/**
|
|
19
|
+
* Union `memoryId` + `memoryIds`, then dedupe. Empty / non-string entries
|
|
20
|
+
* in the plural list are skipped (same filter as stdio `buildRecordUsageBody`).
|
|
21
|
+
* Used by native `/mcp` to flatten the tool args before `RecordUsage.post`.
|
|
22
|
+
*/
|
|
23
|
+
export function unionUsageMemoryIds(memoryId, memoryIds) {
|
|
24
|
+
const ids = [];
|
|
25
|
+
if (Array.isArray(memoryIds)) {
|
|
26
|
+
for (const id of memoryIds) {
|
|
27
|
+
if (typeof id === "string" && id.length > 0)
|
|
28
|
+
ids.push(id);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (typeof memoryId === "string" && memoryId.length > 0) {
|
|
32
|
+
ids.push(memoryId);
|
|
33
|
+
}
|
|
34
|
+
return [...new Set(ids)];
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* HTTP-endpoint resolver: union both fields, then apply RecordUsage.post()'s
|
|
38
|
+
* existing validation (non-empty strings, per-call cap on the unique credited
|
|
39
|
+
* set). Cap after dedupe so an overlapping `memoryId` does not 400 a legal
|
|
40
|
+
* unique set that native `/mcp` and stdio already accept. A present-but-invalid
|
|
41
|
+
* `memoryIds` still 400s. The max-20 anti-gaming bound is unchanged — it
|
|
42
|
+
* limits unique ids credited, not raw concatenation length.
|
|
43
|
+
*/
|
|
44
|
+
export function resolveRecordUsageIds(data, maxIds) {
|
|
45
|
+
const memoryIds = data?.memoryIds;
|
|
46
|
+
const memoryId = data?.memoryId;
|
|
47
|
+
if (memoryIds != null) {
|
|
48
|
+
if (!Array.isArray(memoryIds) || !memoryIds.every((id) => typeof id === "string" && id.length > 0)) {
|
|
49
|
+
return { ok: false, error: "invalid" };
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const raw = [];
|
|
53
|
+
if (Array.isArray(memoryIds))
|
|
54
|
+
raw.push(...memoryIds);
|
|
55
|
+
if (typeof memoryId === "string" && memoryId.length > 0)
|
|
56
|
+
raw.push(memoryId);
|
|
57
|
+
if (raw.length === 0)
|
|
58
|
+
return { ok: false, error: "empty" };
|
|
59
|
+
const ids = [...new Set(raw)];
|
|
60
|
+
if (ids.length > maxIds)
|
|
61
|
+
return { ok: false, error: "cap" };
|
|
62
|
+
return { ok: true, ids };
|
|
63
|
+
}
|