@tpsdev-ai/flair 0.50.0 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -4
- package/dist/build-info.json +3 -3
- package/dist/cli.js +188 -67
- package/dist/doctor-client.js +3 -3
- package/dist/install/clients.js +28 -17
- package/dist/lib/doctor-run.js +481 -0
- package/dist/lib/launchd-management.js +7 -26
- package/dist/resources/Federation.js +42 -20
- package/dist/resources/RecordUsage.js +13 -6
- package/dist/resources/SemanticSearch.js +8 -1
- package/dist/resources/federation-classify.js +90 -0
- package/dist/resources/health.js +9 -10
- package/dist/resources/mcp-tools.js +9 -6
- package/dist/resources/search-readiness.js +33 -10
- package/dist/resources/semantic-retrieval-core.js +39 -20
- package/dist/resources/usage-ids.js +63 -0
- package/docs/federation.md +11 -0
- package/docs/supply-chain-policy.md +1 -1
- package/docs/upgrade.md +17 -1
- package/package.json +2 -2
- package/schemas/federation.graphql +1 -1
|
@@ -61,7 +61,7 @@ export class SemanticSearch extends Resource {
|
|
|
61
61
|
// recall-harness (test/bench/recall-harness/run.ts) and `recall-eval.mjs`
|
|
62
62
|
// before reconsidering this default if the compositeScore formula or
|
|
63
63
|
// corpus changes.
|
|
64
|
-
const { agentId: bodyAgentId, q, queryEmbedding, tag, subject, subjects, limit = 10, includeSuperseded = false, scoring = "raw", minScore = 0, since, asOf, includeTrust = false, includeMetadata = false, abstain = false, explain = false } = data || {};
|
|
64
|
+
const { agentId: bodyAgentId, q, queryEmbedding, tag, subject, subjects, limit = 10, includeSuperseded = false, scoring = "raw", minScore = 0, since, asOf, includeTrust = false, includeMetadata = false, abstain = false, explain = false, includeLegs = false } = data || {};
|
|
65
65
|
// Authenticated identity lives on the Harper Resource context (getContext().request).
|
|
66
66
|
// `this.request` is NOT populated on Harper v5 Resources — prior reads here
|
|
67
67
|
// silently returned undefined and the defense-in-depth scope check below
|
|
@@ -215,6 +215,7 @@ export class SemanticSearch extends Resource {
|
|
|
215
215
|
// composite re-scoring headroom to reorder before the final slice.
|
|
216
216
|
const candidateLimit = limit * CANDIDATE_MULTIPLIER;
|
|
217
217
|
const ctx = this.getContext?.();
|
|
218
|
+
let legs;
|
|
218
219
|
const filteredResults = await retrieveCandidates({
|
|
219
220
|
queryEmbedding: qEmb,
|
|
220
221
|
q,
|
|
@@ -230,6 +231,7 @@ export class SemanticSearch extends Resource {
|
|
|
230
231
|
isAllowed: scope?.isAllowed,
|
|
231
232
|
hybrid,
|
|
232
233
|
ctx,
|
|
234
|
+
onLegs: includeLegs ? (l) => { legs = l; } : undefined,
|
|
233
235
|
// flair#744 slice 1: the trust block needs `provenance`, which the
|
|
234
236
|
// default projection omits. Widen the select ONLY when the caller opts
|
|
235
237
|
// in — passing undefined otherwise keeps the default (no `provenance`)
|
|
@@ -330,6 +332,11 @@ export class SemanticSearch extends Resource {
|
|
|
330
332
|
if (!qEmb && q && getMode() === "none") {
|
|
331
333
|
response._warning = "semantic search unavailable — results are keyword-only";
|
|
332
334
|
}
|
|
335
|
+
// flair#1358: opt-in per-leg candidate ids for the bench instrument.
|
|
336
|
+
// Default OFF ⇒ response is byte-identical (no `legs` key). The ranked
|
|
337
|
+
// `results` slice is unchanged either way — this is observation only.
|
|
338
|
+
if (includeLegs && legs)
|
|
339
|
+
response.legs = legs;
|
|
333
340
|
return response;
|
|
334
341
|
}
|
|
335
342
|
}
|
|
@@ -5,6 +5,96 @@
|
|
|
5
5
|
* spinning up Harper's database module. The same SkipReason names are used
|
|
6
6
|
* in SyncLog.skippedReasons so operators can grep for them.
|
|
7
7
|
*/
|
|
8
|
+
/**
|
|
9
|
+
* Static policy for every table FederationSync will merge.
|
|
10
|
+
*
|
|
11
|
+
* Lives next to SkipReason / SyncRecord so the principal-owning decision is
|
|
12
|
+
* one visible list, not a condition scattered through the apply path.
|
|
13
|
+
* `Federation.ts` types its `tableMap` as `Record<FederationSyncTable, …>`,
|
|
14
|
+
* so adding a federated table without deciding `principalOwning` here is a
|
|
15
|
+
* type error rather than a silent default.
|
|
16
|
+
*
|
|
17
|
+
* Scope the principalId requirement by TABLE, never by field presence.
|
|
18
|
+
* Memory carries agentId / a provenance stamp; Soul, Agent, and
|
|
19
|
+
* Relationship do not, and will legitimately have no principalId.
|
|
20
|
+
*/
|
|
21
|
+
export const FEDERATION_TABLE_POLICY = {
|
|
22
|
+
Memory: { principalOwning: true },
|
|
23
|
+
Soul: { principalOwning: false },
|
|
24
|
+
Agent: { principalOwning: false },
|
|
25
|
+
Relationship: { principalOwning: false },
|
|
26
|
+
};
|
|
27
|
+
export const FEDERATION_SYNC_TABLES = Object.keys(FEDERATION_TABLE_POLICY);
|
|
28
|
+
export const PRINCIPAL_OWNING_TABLES = new Set(FEDERATION_SYNC_TABLES.filter((t) => FEDERATION_TABLE_POLICY[t].principalOwning));
|
|
29
|
+
/** Wire `v` when present; assume 1 when absent (today's records omit it). */
|
|
30
|
+
export function recordSignatureVersion(record) {
|
|
31
|
+
return record.v ?? 1;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Rebuild the object FederationSync verifies against the originator's key.
|
|
35
|
+
*
|
|
36
|
+
* Load-bearing details, both of which fail every existing record if missed:
|
|
37
|
+
*
|
|
38
|
+
* 1. `v` is not on the wire today. The push side signs a body containing
|
|
39
|
+
* `v: 1` but sends a SyncRecord without it. Verification works today
|
|
40
|
+
* only because the receiver pinned the same literal. Default it.
|
|
41
|
+
* Apply `v` AFTER the spread so an absent/undefined `record.v` cannot
|
|
42
|
+
* overwrite the default.
|
|
43
|
+
*
|
|
44
|
+
* 2. `principalId` IS on the wire today for some records, but it is
|
|
45
|
+
* attached AFTER signing (informational). Spreading it into a v:1
|
|
46
|
+
* verify body changes the field set and fails those records. v:1
|
|
47
|
+
* therefore strips it; v:2 signs it, so it stays.
|
|
48
|
+
*
|
|
49
|
+
* `originatorInstanceId` is the classifyRecord originator (same override
|
|
50
|
+
* the pre-3a hardcoded reconstruction used), not a blind spread of the
|
|
51
|
+
* wire field.
|
|
52
|
+
*/
|
|
53
|
+
export function reconstructRecordVerifyBody(record, originator) {
|
|
54
|
+
const v = recordSignatureVersion(record);
|
|
55
|
+
const { signature, v: _wireV, principalId, ...payload } = record;
|
|
56
|
+
const verifyPayload = v >= 2 && principalId !== undefined ? { ...payload, principalId } : payload;
|
|
57
|
+
return {
|
|
58
|
+
...verifyPayload,
|
|
59
|
+
v,
|
|
60
|
+
originatorInstanceId: originator,
|
|
61
|
+
signature,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Per-record principal entitlement — apply-site check, DB-free.
|
|
66
|
+
*
|
|
67
|
+
* Table in PRINCIPAL_OWNING_TABLES → principalId is mandatory on v:2
|
|
68
|
+
* (absent is a skip, mismatched is a skip). Table not in the set →
|
|
69
|
+
* principalId is not consulted at all.
|
|
70
|
+
*
|
|
71
|
+
* Does not load Agent, does not read originatorInstanceId off an Agent
|
|
72
|
+
* row. The record's own stamp is the binding.
|
|
73
|
+
*
|
|
74
|
+
* `enforceV1Principal` is Phase 3 (FLAIR_FEDERATION_REQUIRE_RECORD_PRINCIPAL):
|
|
75
|
+
* skip leftover v:1 records on principal-owning tables that lack
|
|
76
|
+
* principalId. Off by default — v:1 Memory keeps merging until an
|
|
77
|
+
* operator flips the flag after the fleet is on v:2.
|
|
78
|
+
*/
|
|
79
|
+
export function checkPrincipalEntitlement(record, opts = {}) {
|
|
80
|
+
if (!PRINCIPAL_OWNING_TABLES.has(record.table)) {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
const v = recordSignatureVersion(record);
|
|
84
|
+
if (v >= 2) {
|
|
85
|
+
if (typeof record.principalId !== "string" ||
|
|
86
|
+
record.principalId.length === 0 ||
|
|
87
|
+
record.principalId !== record.data?.agentId) {
|
|
88
|
+
return "principal_mismatch";
|
|
89
|
+
}
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
if (opts.enforceV1Principal &&
|
|
93
|
+
(typeof record.principalId !== "string" || record.principalId.length === 0)) {
|
|
94
|
+
return "principal_mismatch";
|
|
95
|
+
}
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
8
98
|
export function classifyRecord(record, peerRole, receiverInstanceId, local, knownTables, now = new Date()) {
|
|
9
99
|
if (!knownTables.has(record.table)) {
|
|
10
100
|
return { action: "skip", reason: "unknown_table" };
|
package/dist/resources/health.js
CHANGED
|
@@ -121,22 +121,18 @@ export class Health extends Resource {
|
|
|
121
121
|
}
|
|
122
122
|
}
|
|
123
123
|
/** Same sources /Health and /HealthDetail consult so they cannot disagree. */
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
// (Sherlock on #1406) for the injectable/test path — not an accident.
|
|
124
|
+
export function currentSearchReadiness() {
|
|
125
|
+
// Fail-open when the registry is missing (Sherlock on #1406 / flair#1411):
|
|
126
|
+
// do not 503 forever. resolveSearchReadiness warns once and names the
|
|
127
|
+
// degradation; we do not treat "registry should always be here" as a given.
|
|
129
128
|
const resources = server.resources ?? null;
|
|
130
|
-
if (!resources && !_warnedMissingRegistry) {
|
|
131
|
-
_warnedMissingRegistry = true;
|
|
132
|
-
logger.warn?.("Health: server.resources is absent — skipping the search-route mount check (table-only fail-open). Shipped Harper launch always exposes the registry.");
|
|
133
|
-
}
|
|
134
129
|
return resolveSearchReadiness({
|
|
135
130
|
resources,
|
|
136
131
|
memoryTable: db.flair?.Memory,
|
|
137
132
|
bm25: bm25IndexStatus(),
|
|
138
133
|
hybridEnabled: hybridEnabled(),
|
|
139
134
|
bm25IndexEnabled: bm25IndexEnabled(),
|
|
135
|
+
warn: (message) => { logger.warn?.(message); },
|
|
140
136
|
});
|
|
141
137
|
}
|
|
142
138
|
/**
|
|
@@ -168,7 +164,10 @@ export class HealthDetail extends Resource {
|
|
|
168
164
|
// and a warning name the lag so `flair status` / operators can see it.
|
|
169
165
|
const readiness = currentSearchReadiness();
|
|
170
166
|
stats.searchReady = readiness.searchReady;
|
|
171
|
-
|
|
167
|
+
// Public/detail shape is unchanged: searchReadyReason stays a lag signal
|
|
168
|
+
// (present iff !searchReady). Ready-path verification constants stay on
|
|
169
|
+
// the decision object (flair#1411).
|
|
170
|
+
if (!readiness.searchReady && readiness.searchReadyReason) {
|
|
172
171
|
stats.searchReadyReason = readiness.searchReadyReason;
|
|
173
172
|
warnings.push({ level: "warn", message: readiness.searchReadyReason });
|
|
174
173
|
}
|
|
@@ -37,6 +37,7 @@
|
|
|
37
37
|
*/
|
|
38
38
|
import { resolveVersion } from "./version.js";
|
|
39
39
|
import { agentContext, adminContext, collectionResource } from "./in-process.js";
|
|
40
|
+
import { RECORD_USAGE_ID_MERGE_CONTRACT, unionUsageMemoryIds } from "./usage-ids.js";
|
|
40
41
|
const H = {};
|
|
41
42
|
const LOADERS = {
|
|
42
43
|
SemanticSearch: async () => (await import("./SemanticSearch.js")).SemanticSearch,
|
|
@@ -538,9 +539,10 @@ async function attention(agent, args) {
|
|
|
538
539
|
async function recordUsage(agent, args) {
|
|
539
540
|
const Cls = await handler("RecordUsage");
|
|
540
541
|
const h = new Cls(undefined, delegationContext(agent));
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
542
|
+
// flair#1410: MERGE, do not prefer. The previous ternary dropped
|
|
543
|
+
// `memoryId` whenever `memoryIds` was an array (including `[]`).
|
|
544
|
+
const merged = unionUsageMemoryIds(args?.memoryId, args?.memoryIds);
|
|
545
|
+
const memoryIds = merged.length > 0 ? merged : undefined;
|
|
544
546
|
return unwrap(await h.post({ memoryIds, attribution: args?.attribution }));
|
|
545
547
|
}
|
|
546
548
|
/**
|
|
@@ -1005,12 +1007,13 @@ export const TOOLS = {
|
|
|
1005
1007
|
name: "record_usage",
|
|
1006
1008
|
description: "Report that one or more memories were actually USED — cited or relied on to ground an answer or decision. " +
|
|
1007
1009
|
"Distinct from search (surfacing a memory is not usage). Drives the recall-quality usage signal; dedup'd " +
|
|
1008
|
-
"(you can only count once per memory) and rate-limited."
|
|
1010
|
+
"(you can only count once per memory) and rate-limited. " +
|
|
1011
|
+
RECORD_USAGE_ID_MERGE_CONTRACT,
|
|
1009
1012
|
inputSchema: {
|
|
1010
1013
|
type: "object",
|
|
1011
1014
|
properties: {
|
|
1012
|
-
memoryIds: { type: "array", items: { type: "string" }, description: "IDs of the memories that were used (max 20 per call)" },
|
|
1013
|
-
memoryId: { type: "string", description: "Convenience alias for a single memory id
|
|
1015
|
+
memoryIds: { type: "array", items: { type: "string" }, description: "IDs of the memories that were used (max 20 per call). Merged with memoryId when both are supplied." },
|
|
1016
|
+
memoryId: { type: "string", description: "Convenience alias for a single memory id. Merged with memoryIds when both are supplied — not dropped." },
|
|
1014
1017
|
attribution: { type: "string", description: "Optional free-text note on what used it (opaque — stored for audit only, max 500 chars)" },
|
|
1015
1018
|
},
|
|
1016
1019
|
},
|
|
@@ -28,6 +28,21 @@
|
|
|
28
28
|
* warm would deadlock — health waits for the index, the index waits for
|
|
29
29
|
* a search that never comes.
|
|
30
30
|
*/
|
|
31
|
+
/**
|
|
32
|
+
* How a ready result was verified. Constant strings — no interpolation
|
|
33
|
+
* (flair#1411). Public /Health still omits this field when searchReady is
|
|
34
|
+
* true; the values live on the decision object so a caller of
|
|
35
|
+
* resolveSearchReadiness can tell registry-verified from table-only.
|
|
36
|
+
*/
|
|
37
|
+
export const SEARCH_READY_REASON_VERIFIED_VIA_ROUTE_REGISTRY = "verified via route registry";
|
|
38
|
+
export const SEARCH_READY_REASON_REGISTRY_UNAVAILABLE_TABLE_ONLY = "registry unavailable, table check only";
|
|
39
|
+
/** Once-warn when the route-mount check is skipped (flair#1411). */
|
|
40
|
+
export const MISSING_REGISTRY_WARN = "search route-mount verification is degraded; searchReady now rests on the table check alone";
|
|
41
|
+
let warnedMissingRegistry = false;
|
|
42
|
+
/** Test-only: forget the one-shot missing-registry warning. */
|
|
43
|
+
export function _resetMissingRegistryWarnForTests() {
|
|
44
|
+
warnedMissingRegistry = false;
|
|
45
|
+
}
|
|
31
46
|
function routeMounted(resources, name) {
|
|
32
47
|
const entry = resources.get?.(name) ?? resources.getMatch?.(name);
|
|
33
48
|
return Boolean(entry?.Resource);
|
|
@@ -36,16 +51,14 @@ function routeMounted(resources, name) {
|
|
|
36
51
|
* Decide whether search is actually usable, and whether /Health should claim
|
|
37
52
|
* the process is healthy.
|
|
38
53
|
*
|
|
39
|
-
* `resources` is optional. Stated fail-open (Sherlock on #1406):
|
|
40
|
-
* registry is missing we skip the route-mount check rather than
|
|
41
|
-
* A table handle can exist while `/Memory` and `/SemanticSearch`
|
|
42
|
-
* so this is weaker than the primary defense.
|
|
43
|
-
*
|
|
44
|
-
* `server.resources` is populated; the skip is the injectable/test path
|
|
45
|
-
* (and a theoretical export without a registry). Health.ts logs once if
|
|
46
|
-
* the live call site actually takes it.
|
|
54
|
+
* `resources` is optional. Stated fail-open (Sherlock on #1406 / flair#1411):
|
|
55
|
+
* when the registry is missing we skip the route-mount check rather than
|
|
56
|
+
* 503 forever. A table handle can exist while `/Memory` and `/SemanticSearch`
|
|
57
|
+
* still 404, so this is weaker than the primary defense. We do not fail
|
|
58
|
+
* closed. We warn once and name the degradation on `searchReadyReason`.
|
|
47
59
|
*/
|
|
48
60
|
export function resolveSearchReadiness(opts) {
|
|
61
|
+
let readyReason = SEARCH_READY_REASON_REGISTRY_UNAVAILABLE_TABLE_ONLY;
|
|
49
62
|
if (opts.resources) {
|
|
50
63
|
const memoryMounted = routeMounted(opts.resources, "Memory");
|
|
51
64
|
const searchMounted = routeMounted(opts.resources, "SemanticSearch");
|
|
@@ -56,6 +69,13 @@ export function resolveSearchReadiness(opts) {
|
|
|
56
69
|
].filter(Boolean).join(", ");
|
|
57
70
|
return notServing(`search routes not mounted (${missing})`);
|
|
58
71
|
}
|
|
72
|
+
readyReason = SEARCH_READY_REASON_VERIFIED_VIA_ROUTE_REGISTRY;
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
if (!warnedMissingRegistry) {
|
|
76
|
+
warnedMissingRegistry = true;
|
|
77
|
+
opts.warn?.(MISSING_REGISTRY_WARN);
|
|
78
|
+
}
|
|
59
79
|
}
|
|
60
80
|
if (!opts.memoryTable || typeof opts.memoryTable.search !== "function") {
|
|
61
81
|
return notServing("memory table not queryable");
|
|
@@ -78,7 +98,7 @@ export function resolveSearchReadiness(opts) {
|
|
|
78
98
|
return namesLag("bm25 index not built (cold boot; first search scans the corpus)");
|
|
79
99
|
}
|
|
80
100
|
}
|
|
81
|
-
return { searchReady: true, ok: true, status: 200 };
|
|
101
|
+
return { searchReady: true, ok: true, status: 200, searchReadyReason: readyReason };
|
|
82
102
|
}
|
|
83
103
|
function notServing(searchReadyReason) {
|
|
84
104
|
return { searchReady: false, ok: false, status: 503, searchReadyReason };
|
|
@@ -94,7 +114,10 @@ export function buildPublicHealthBody(readiness, identity) {
|
|
|
94
114
|
buildCommit: identity.buildCommit,
|
|
95
115
|
searchReady: readiness.searchReady,
|
|
96
116
|
};
|
|
97
|
-
|
|
117
|
+
// Public shape is unchanged: searchReadyReason is still present iff !searchReady.
|
|
118
|
+
// Ready-path verification constants stay on the decision object (flair#1411).
|
|
119
|
+
if (!readiness.searchReady && readiness.searchReadyReason) {
|
|
98
120
|
body.searchReadyReason = readiness.searchReadyReason;
|
|
121
|
+
}
|
|
99
122
|
return body;
|
|
100
123
|
}
|
|
@@ -34,9 +34,11 @@
|
|
|
34
34
|
// multiplies `limit` internally; any overfetch policy — SemanticSearch's
|
|
35
35
|
// CANDIDATE_MULTIPLIER, MemoryBootstrap's K formula — is the CALLER's
|
|
36
36
|
// decision, made
|
|
37
|
-
// before calling in).
|
|
38
|
-
// legacy HNSW-only vs. keyword-only fallback) produced
|
|
39
|
-
// output shape is identical regardless of `hybrid`.
|
|
37
|
+
// before calling in). The result array never annotates which internal leg
|
|
38
|
+
// (BM25+RRF hybrid vs. legacy HNSW-only vs. keyword-only fallback) produced
|
|
39
|
+
// a given row — that output shape is identical regardless of `hybrid`.
|
|
40
|
+
// Per-leg membership is available only through the opt-in `onLegs`
|
|
41
|
+
// callback (flair#1358); unset ⇒ the return is byte-identical to before.
|
|
40
42
|
//
|
|
41
43
|
// ── SCORE CONTRACT (flair#985) ───────────────────────────────────────────────
|
|
42
44
|
// `_score` under `scoring:"raw"` is ALWAYS an ABSOLUTE similarity (cosine of
|
|
@@ -61,6 +63,14 @@ import { buildBM25, fuseRrfNormalized, SEM_LIMIT } from "./bm25.js";
|
|
|
61
63
|
import { isAllowedBm25Candidate } from "./bm25-filter.js";
|
|
62
64
|
import { indexedBm25Ids } from "./bm25-index-service.js";
|
|
63
65
|
import { byRecencyThenId } from "./sort-comparators.js";
|
|
66
|
+
/**
|
|
67
|
+
* The only way a row enters the ranked pool. `_rank` is a required
|
|
68
|
+
* argument — not an object property that `...any` can swallow — so a
|
|
69
|
+
* push site that omits it is a compile error (flair#1415).
|
|
70
|
+
*/
|
|
71
|
+
function pushRanked(rows, row, _rank) {
|
|
72
|
+
rows.push({ ...row, _rank });
|
|
73
|
+
}
|
|
64
74
|
// Convert HNSW cosine distance (1 - similarity) to similarity score.
|
|
65
75
|
function distanceToSimilarity(distance) {
|
|
66
76
|
return 1 - distance;
|
|
@@ -87,10 +97,12 @@ export const DEFAULT_SELECT = ["id", "agentId", "content", "contentHash", "visib
|
|
|
87
97
|
"parentId", "derivedFrom", "sessionId", "lastReflected", "supersedes", "subject", "summary",
|
|
88
98
|
"validFrom", "validTo", "_safetyFlags"];
|
|
89
99
|
export async function retrieveCandidates(params) {
|
|
90
|
-
const { queryEmbedding: qEmb, q, conditions, limit, select = DEFAULT_SELECT, includeSuperseded = false, scoring = "raw", temporalBoost = 1.0, sinceDate = null, asOf, minScore = 0, agentId, isAllowed, hybrid, ctx, withSemSimilarity = false, } = params;
|
|
100
|
+
const { queryEmbedding: qEmb, q, conditions, limit, select = DEFAULT_SELECT, includeSuperseded = false, scoring = "raw", temporalBoost = 1.0, sinceDate = null, asOf, minScore = 0, agentId, isAllowed, hybrid, ctx, withSemSimilarity = false, onLegs, } = params;
|
|
91
101
|
const passesAllowed = (record) => !isAllowed || isAllowed(record);
|
|
92
102
|
const hnswSelect = [...select, "$distance"];
|
|
93
103
|
const results = [];
|
|
104
|
+
const hnswLegIds = [];
|
|
105
|
+
const bm25LegIds = [];
|
|
94
106
|
if (hybrid) {
|
|
95
107
|
// ─── BM25 + union-RRF hybrid path ────────────────────────────────────
|
|
96
108
|
// 1. Semantic candidates via HNSW (unchanged fetch). 2. BM25 lexical pass
|
|
@@ -154,6 +166,7 @@ export async function retrieveCandidates(params) {
|
|
|
154
166
|
}
|
|
155
167
|
semRecords.push(record);
|
|
156
168
|
semIds.push(record.id);
|
|
169
|
+
hnswLegIds.push(record.id);
|
|
157
170
|
}
|
|
158
171
|
}
|
|
159
172
|
// ── (b) The BM25 lexical leg ─────────────────────────────────────────
|
|
@@ -260,6 +273,8 @@ export async function retrieveCandidates(params) {
|
|
|
260
273
|
}
|
|
261
274
|
bm25Ids = resolved;
|
|
262
275
|
}
|
|
276
|
+
if (q)
|
|
277
|
+
bm25LegIds.push(...bm25Ids);
|
|
263
278
|
// ── (d) No retrieval signal at all → full scoped listing ────────────
|
|
264
279
|
if (!q && !qEmb) {
|
|
265
280
|
for (const record of allowedById.values()) {
|
|
@@ -269,14 +284,13 @@ export async function retrieveCandidates(params) {
|
|
|
269
284
|
finalScore *= temporalBoost;
|
|
270
285
|
const isFlagged = record._safetyFlags && Array.isArray(record._safetyFlags) && record._safetyFlags.length > 0;
|
|
271
286
|
const source = record.agentId !== agentId ? record.agentId : undefined;
|
|
272
|
-
results
|
|
287
|
+
pushRanked(results, {
|
|
273
288
|
...record,
|
|
274
289
|
content: isFlagged ? wrapUntrusted(record.content, source) : record.content,
|
|
275
290
|
_score: Math.round(finalScore * 1000) / 1000,
|
|
276
291
|
_rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
|
|
277
292
|
_source: source,
|
|
278
|
-
|
|
279
|
-
});
|
|
293
|
+
}, finalScore);
|
|
280
294
|
}
|
|
281
295
|
}
|
|
282
296
|
else {
|
|
@@ -324,22 +338,21 @@ export async function retrieveCandidates(params) {
|
|
|
324
338
|
finalScore *= temporalBoost;
|
|
325
339
|
const isFlagged = record._safetyFlags && Array.isArray(record._safetyFlags) && record._safetyFlags.length > 0;
|
|
326
340
|
const source = record.agentId !== agentId ? record.agentId : undefined;
|
|
327
|
-
|
|
341
|
+
// Ordering key: fused rank for raw mode; composite value for
|
|
342
|
+
// composite mode (composite ordering is unchanged by #985 — its
|
|
343
|
+
// rrfRaw input and result order are exactly the pre-#985 behavior).
|
|
344
|
+
pushRanked(results, {
|
|
328
345
|
...record,
|
|
329
346
|
content: isFlagged ? wrapUntrusted(record.content, source) : record.content,
|
|
330
347
|
_score: Math.round(finalScore * 1000) / 1000,
|
|
331
348
|
_rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
|
|
332
349
|
_source: source,
|
|
333
|
-
// Ordering key: fused rank for raw mode; composite value for
|
|
334
|
-
// composite mode (composite ordering is unchanged by #985 — its
|
|
335
|
-
// rrfRaw input and result order are exactly the pre-#985 behavior).
|
|
336
|
-
_rank: scoring === "raw" ? rrfRaw : finalScore,
|
|
337
350
|
// flair#744 slice 2: the opt-in absolute-confidence field for the
|
|
338
351
|
// abstention decision. Attach remains OPT-IN so non-abstain
|
|
339
352
|
// responses stay byte-identical (the capture above is now
|
|
340
353
|
// unconditional, but the response field is not).
|
|
341
354
|
...(withSemSimilarity && semSim !== undefined ? { _semSimilarity: semSim } : {}),
|
|
342
|
-
});
|
|
355
|
+
}, scoring === "raw" ? rrfRaw : finalScore);
|
|
343
356
|
}
|
|
344
357
|
}
|
|
345
358
|
}
|
|
@@ -401,15 +414,15 @@ export async function retrieveCandidates(params) {
|
|
|
401
414
|
// flair#744 slice 2: the absolute cosine (`semanticScore`, pre keyword
|
|
402
415
|
// bump) is the abstention confidence signal on this legacy/bootstrap
|
|
403
416
|
// (HNSW-leg-only) path.
|
|
404
|
-
results
|
|
417
|
+
pushRanked(results, {
|
|
405
418
|
...rest,
|
|
406
419
|
content: isFlagged ? wrapUntrusted(rest.content, source) : rest.content,
|
|
407
420
|
_score: Math.round(finalScore * 1000) / 1000,
|
|
408
421
|
_rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
|
|
409
422
|
_source: source,
|
|
410
|
-
_rank: finalScore,
|
|
411
423
|
...(withSemSimilarity ? { _semSimilarity: semanticScore } : {}),
|
|
412
|
-
});
|
|
424
|
+
}, finalScore);
|
|
425
|
+
hnswLegIds.push(record.id);
|
|
413
426
|
}
|
|
414
427
|
}
|
|
415
428
|
else {
|
|
@@ -447,14 +460,13 @@ export async function retrieveCandidates(params) {
|
|
|
447
460
|
finalScore *= temporalBoost;
|
|
448
461
|
const isFlagged = rest._safetyFlags && Array.isArray(rest._safetyFlags) && rest._safetyFlags.length > 0;
|
|
449
462
|
const source = record.agentId !== agentId ? record.agentId : undefined;
|
|
450
|
-
results
|
|
463
|
+
pushRanked(results, {
|
|
451
464
|
...rest,
|
|
452
465
|
content: isFlagged ? wrapUntrusted(rest.content, source) : rest.content,
|
|
453
466
|
_score: Math.round(finalScore * 1000) / 1000,
|
|
454
467
|
_rawScore: scoring !== "raw" ? Math.round(rawScore * 1000) / 1000 : undefined,
|
|
455
468
|
_source: source,
|
|
456
|
-
|
|
457
|
-
});
|
|
469
|
+
}, finalScore);
|
|
458
470
|
}
|
|
459
471
|
}
|
|
460
472
|
// Build superseded set and filter (unless caller opts in to see full
|
|
@@ -480,7 +492,9 @@ export async function retrieveCandidates(params) {
|
|
|
480
492
|
}
|
|
481
493
|
// Order by the internal ranking key (fused RRF rank on the hybrid raw path;
|
|
482
494
|
// identical to `_score` everywhere else), then strip it — `_rank` is an
|
|
483
|
-
// ordering key, never part of the response shape.
|
|
495
|
+
// ordering key, never part of the response shape. Required on
|
|
496
|
+
// RetrievalRankedRow / pushRanked (flair#1415) so a missing value cannot
|
|
497
|
+
// silently NaN this comparator into the recency tail. Note the hybrid raw
|
|
484
498
|
// ordering is deliberately NOT by `_score`: the recall win of hybrid
|
|
485
499
|
// retrieval lives in the fused ORDER (a BM25 rank-1 rescue outranks weak
|
|
486
500
|
// semantic hits), while `_score` carries the honest absolute evidence for
|
|
@@ -495,5 +509,10 @@ export async function retrieveCandidates(params) {
|
|
|
495
509
|
filteredResults.sort((a, b) => (b._rank - a._rank) || byRecencyThenId(a, b));
|
|
496
510
|
for (const r of filteredResults)
|
|
497
511
|
delete r._rank;
|
|
512
|
+
onLegs?.({
|
|
513
|
+
hnsw: hnswLegIds,
|
|
514
|
+
bm25: bm25LegIds,
|
|
515
|
+
fused: filteredResults.map((r) => r.id),
|
|
516
|
+
});
|
|
498
517
|
return filteredResults;
|
|
499
518
|
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* usage-ids.ts — which memory ids a usage-feedback call credits (flair#1410).
|
|
3
|
+
*
|
|
4
|
+
* `record_usage` / `POST /RecordUsage` accept both singular `memoryId` and
|
|
5
|
+
* plural `memoryIds`. MERGE, not prefer: a caller who supplies both means
|
|
6
|
+
* both. Preferring `memoryIds` (`data?.memoryIds ?? [data?.memoryId]`)
|
|
7
|
+
* silently dropped the singular id — quiet data loss, same class as
|
|
8
|
+
* #1206 / #1371.
|
|
9
|
+
*
|
|
10
|
+
* Pure: no Harper. Native `/mcp` (`resources/mcp-tools.ts`) and the HTTP
|
|
11
|
+
* endpoint (`resources/RecordUsage.ts`) both call this so the credited set
|
|
12
|
+
* does not depend on every client flattening first. Stdio `flair-mcp`
|
|
13
|
+
* merges independently in `buildRecordUsageBody`; the conformance test
|
|
14
|
+
* pins the two surfaces to the same set.
|
|
15
|
+
*/
|
|
16
|
+
/** Stated on both MCP tool schemas so a caller can predict the merge without reading source. */
|
|
17
|
+
export const RECORD_USAGE_ID_MERGE_CONTRACT = "When both memoryId and memoryIds are supplied they are merged (union, then deduped) — a caller who passes both means both.";
|
|
18
|
+
/**
|
|
19
|
+
* Union `memoryId` + `memoryIds`, then dedupe. Empty / non-string entries
|
|
20
|
+
* in the plural list are skipped (same filter as stdio `buildRecordUsageBody`).
|
|
21
|
+
* Used by native `/mcp` to flatten the tool args before `RecordUsage.post`.
|
|
22
|
+
*/
|
|
23
|
+
export function unionUsageMemoryIds(memoryId, memoryIds) {
|
|
24
|
+
const ids = [];
|
|
25
|
+
if (Array.isArray(memoryIds)) {
|
|
26
|
+
for (const id of memoryIds) {
|
|
27
|
+
if (typeof id === "string" && id.length > 0)
|
|
28
|
+
ids.push(id);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (typeof memoryId === "string" && memoryId.length > 0) {
|
|
32
|
+
ids.push(memoryId);
|
|
33
|
+
}
|
|
34
|
+
return [...new Set(ids)];
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* HTTP-endpoint resolver: union both fields, then apply RecordUsage.post()'s
|
|
38
|
+
* existing validation (non-empty strings, per-call cap on the unique credited
|
|
39
|
+
* set). Cap after dedupe so an overlapping `memoryId` does not 400 a legal
|
|
40
|
+
* unique set that native `/mcp` and stdio already accept. A present-but-invalid
|
|
41
|
+
* `memoryIds` still 400s. The max-20 anti-gaming bound is unchanged — it
|
|
42
|
+
* limits unique ids credited, not raw concatenation length.
|
|
43
|
+
*/
|
|
44
|
+
export function resolveRecordUsageIds(data, maxIds) {
|
|
45
|
+
const memoryIds = data?.memoryIds;
|
|
46
|
+
const memoryId = data?.memoryId;
|
|
47
|
+
if (memoryIds != null) {
|
|
48
|
+
if (!Array.isArray(memoryIds) || !memoryIds.every((id) => typeof id === "string" && id.length > 0)) {
|
|
49
|
+
return { ok: false, error: "invalid" };
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const raw = [];
|
|
53
|
+
if (Array.isArray(memoryIds))
|
|
54
|
+
raw.push(...memoryIds);
|
|
55
|
+
if (typeof memoryId === "string" && memoryId.length > 0)
|
|
56
|
+
raw.push(memoryId);
|
|
57
|
+
if (raw.length === 0)
|
|
58
|
+
return { ok: false, error: "empty" };
|
|
59
|
+
const ids = [...new Set(raw)];
|
|
60
|
+
if (ids.length > maxIds)
|
|
61
|
+
return { ok: false, error: "cap" };
|
|
62
|
+
return { ok: true, ids };
|
|
63
|
+
}
|
package/docs/federation.md
CHANGED
|
@@ -171,6 +171,17 @@ Re-pairing an existing peer (same instance ID, same public key) does not require
|
|
|
171
171
|
|
|
172
172
|
Spoke instances can only push records they originated. A spoke cannot overwrite records from another spoke or from the hub. The hub can relay records from any origin.
|
|
173
173
|
|
|
174
|
+
### Per-record signatures and principalId
|
|
175
|
+
|
|
176
|
+
Each pushed record carries an Ed25519 signature over a versioned canonical body. `v` lives inside that body so versions are distinguishable: a `v: 1` signature cannot verify as `v: 2`.
|
|
177
|
+
|
|
178
|
+
- **`v: 1` (today's wire):** signed fields are `{ v, table, id, data, updatedAt, originatorInstanceId }`. Senders did not put `v` on the wire; receivers default absent `v` to `1`. `principalId` may appear on the record (from a Memory provenance stamp) but was not in the signed field set.
|
|
179
|
+
- **`v: 2`:** `principalId` is included in the signed payload when the row has a write-time provenance stamp (`provenance.verified.agentId`). `v` is on the wire.
|
|
180
|
+
|
|
181
|
+
On apply, after the signature checks, **Memory** (the only principal-owning federated table) requires `principalId` to be present and equal to `data.agentId`. Absent is a skip (`principal_mismatch`), not an accept. Soul, Agent, and Relationship are not in that set and still sync without a principal.
|
|
182
|
+
|
|
183
|
+
Receivers must be upgraded before senders. A Phase 1 receiver verifies both shapes in one batch. Old receivers cannot reconstruct a `v: 2` body and will skip those records (per-record, not a batch outage) until they upgrade. Optional `FLAIR_FEDERATION_REQUIRE_RECORD_PRINCIPAL=true` skips leftover `v: 1` Memory records that lack `principalId` once the fleet is on `v: 2`.
|
|
184
|
+
|
|
174
185
|
### Timestamp ceiling
|
|
175
186
|
|
|
176
187
|
Records with `updatedAt` more than 5 minutes in the future are rejected. This prevents an attacker from using far-future timestamps to permanently win last-write-wins (LWW) merge conflicts.
|
|
@@ -151,7 +151,7 @@ Mirrors the CI test-unit gates locally so issues are caught at `git commit` time
|
|
|
151
151
|
Runs three checks before each commit:
|
|
152
152
|
- `check-workspace-deps.mjs` — workspace internal-dep version lockstep
|
|
153
153
|
- `check-dep-ages.mjs` — supply-chain bake-time (≥7 days for external pinned deps)
|
|
154
|
-
- `check-impl-term-leaks.sh` — no Bead refs / impl labels in user-facing docs
|
|
154
|
+
- `check-impl-term-leaks.sh` — no Bead refs / impl labels in user-facing docs, `CHANGELOG.md`, or `.changelog/`
|
|
155
155
|
|
|
156
156
|
Each check matches a CI gate exactly so the local and remote outcomes can't drift. Bypass with `git commit --no-verify` when warranted (rare; CI will still catch you). Skip just the dep-ages check (the slowest one, ~2-5s of registry fetches) with `FLAIR_PRECOMMIT_SKIP_DEP_AGES=1 git commit`.
|
|
157
157
|
|
package/docs/upgrade.md
CHANGED
|
@@ -65,7 +65,15 @@ actually running:
|
|
|
65
65
|
- **Post-restart verification** (skip with `--no-verify`) confirms the
|
|
66
66
|
restarted instance answers `/Health`, that an authenticated request
|
|
67
67
|
round-trips, and that the reported running version matches what was just
|
|
68
|
-
installed.
|
|
68
|
+
installed. It then runs the same enumerable doctor install-health checks
|
|
69
|
+
(`flair doctor`'s client-integration catalog, plus launchd management)
|
|
70
|
+
and prints `✅ verified: healthy` only when every check ran and none
|
|
71
|
+
failed. A missing Codex SessionStart hook — which `flair init` before
|
|
72
|
+
0.50.0 never wrote — is one of those checks. Installing the hook is
|
|
73
|
+
consent-bearing (it executes at every session start): an interactive
|
|
74
|
+
upgrade prompts; a non-interactive upgrade states the gap and withholds
|
|
75
|
+
✅. Pass `--install-hooks` to consent without a prompt, then `flair
|
|
76
|
+
doctor` exits 0.
|
|
69
77
|
- **On a failed restart OR a failed verification**, `flair upgrade`
|
|
70
78
|
automatically reinstalls the previously-running `@tpsdev-ai/flair` version,
|
|
71
79
|
restarts again, and re-verifies — then exits nonzero with a clear report of
|
|
@@ -517,6 +525,14 @@ The backwards-boot refusal (flair#1049) catches this: the old binary refuses to
|
|
|
517
525
|
naming both versions and the data directory, with recovery instructions. A pre-upgrade
|
|
518
526
|
snapshot exists at the named path. Restoring it returns the store to a working state.
|
|
519
527
|
|
|
528
|
+
**Patch-level break inside 5.2:** Harper 5.2.7 writes LZ4-compressed RocksDB that
|
|
529
|
+
Harper 5.2.0 cannot open (`LZ4 not supported in this build`). Downgrade from a
|
|
530
|
+
5.2.7-written store to the npm-published 5.2.0 pin is forward-only — same
|
|
531
|
+
recovery as the 5.1 → 5.2 break: `flair snapshot restore <path>`. The
|
|
532
|
+
`downgrade-boot` suite treats that Harper crash as the loud-refusal branch of
|
|
533
|
+
the flair#1050 invariant (it boots Harper via `startHarper`, so the CLI stamp
|
|
534
|
+
phrasing is not on that path).
|
|
535
|
+
|
|
520
536
|
**As observed when this suite was added (2026-07-08):** the npm-published baseline
|
|
521
537
|
(0.21.0) boots cleanly against data written by a HEAD build roughly 14 commits ahead of
|
|
522
538
|
it (several security-hardening and CLI-behavior changes, no Flair schema migration, and
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tpsdev-ai/flair",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.51.0",
|
|
4
4
|
"packageManager": "bun@1.3.10",
|
|
5
5
|
"description": "Identity, memory, and soul for AI agents. Cryptographic identity (Ed25519), semantic memory with local embeddings, and persistent personality — all in a single process.",
|
|
6
6
|
"type": "module",
|
|
@@ -65,7 +65,7 @@
|
|
|
65
65
|
"@harperfast/oauth": "2.5.0",
|
|
66
66
|
"@types/js-yaml": "4.0.9",
|
|
67
67
|
"commander": "14.0.3",
|
|
68
|
-
"harper": "5.2.
|
|
68
|
+
"harper": "5.2.7",
|
|
69
69
|
"harper-fabric-embeddings": "^0.5.0",
|
|
70
70
|
"jose": "6.2.2",
|
|
71
71
|
"js-yaml": "^4.3.1",
|
|
@@ -58,7 +58,7 @@ type SyncLog @table(database: "flair") {
|
|
|
58
58
|
direction: String! @indexed # "push" | "pull"
|
|
59
59
|
recordCount: Int # how many records merged in this batch
|
|
60
60
|
skippedCount: Int # how many records skipped (sum of skippedReasons)
|
|
61
|
-
skippedReasons: String # JSON: { unknown_table: N, non_originator: N, future_timestamp: N, no_op_same_hash: N, merge_error: N }
|
|
61
|
+
skippedReasons: String # JSON: { unknown_table: N, non_originator: N, future_timestamp: N, no_op_same_hash: N, merge_error: N, unknown_originator_key: N, invalid_signature: N, missing_signature: N, principal_mismatch: N }
|
|
62
62
|
status: String @indexed # "success" | "partial" | "failed"
|
|
63
63
|
error: String # human-readable summary if partial/failed (includes per-record error messages, capped)
|
|
64
64
|
durationMs: Int # how long the sync took
|