@tpsdev-ai/flair 0.20.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -11
- package/SECURITY.md +52 -20
- package/config.yaml +32 -5
- package/dist/cli.js +2235 -260
- package/dist/deploy.js +96 -15
- package/dist/doctor-client.js +312 -0
- package/dist/fleet-presence.js +98 -0
- package/dist/fleet-verify.js +291 -0
- package/dist/install/clients.js +18 -0
- package/dist/mcp-client-assertion.js +364 -0
- package/dist/probe.js +97 -0
- package/dist/rem/restore.js +1 -1
- package/dist/rem/runner.js +82 -12
- package/dist/resources/Admin.js +1 -1
- package/dist/resources/AdminConnectors.js +1 -1
- package/dist/resources/AdminDashboard.js +1 -1
- package/dist/resources/AdminIdp.js +1 -1
- package/dist/resources/AdminInstance.js +1 -1
- package/dist/resources/AdminMemory.js +2 -2
- package/dist/resources/AdminPrincipals.js +1 -1
- package/dist/resources/Agent.js +14 -0
- package/dist/resources/AgentCard.js +1 -1
- package/dist/resources/AgentSeed.js +1 -1
- package/dist/resources/AttentionQuery.js +356 -0
- package/dist/resources/Credential.js +11 -1
- package/dist/resources/Federation.js +121 -2
- package/dist/resources/Integration.js +1 -1
- package/dist/resources/MCPClientMetadata.js +88 -0
- package/dist/resources/Memory.js +127 -22
- package/dist/resources/MemoryBootstrap.js +441 -85
- package/dist/resources/MemoryGrant.js +1 -1
- package/dist/resources/MemoryReflect.js +105 -49
- package/dist/resources/MemoryUsage.js +104 -0
- package/dist/resources/OAuth.js +69 -5
- package/dist/resources/OrgEvent.js +11 -0
- package/dist/resources/OrgEventCatchup.js +1 -1
- package/dist/resources/Presence.js +268 -17
- package/dist/resources/RecordUsage.js +230 -0
- package/dist/resources/Relationship.js +108 -26
- package/dist/resources/SemanticSearch.js +88 -320
- package/dist/resources/SkillScan.js +15 -0
- package/dist/resources/Soul.js +14 -0
- package/dist/resources/WorkspaceLatest.js +1 -1
- package/dist/resources/WorkspaceState.js +12 -1
- package/dist/resources/agent-auth.js +61 -3
- package/dist/resources/agentcard-fields.js +2 -2
- package/dist/resources/auth-middleware.js +42 -14
- package/dist/resources/bm25-filter.js +1 -1
- package/dist/resources/bm25.js +13 -7
- package/dist/resources/collision-lib.js +157 -0
- package/dist/resources/dedup.js +2 -2
- package/dist/resources/ed25519-auth.js +2 -2
- package/dist/resources/embeddings-boot.js +149 -0
- package/dist/resources/embeddings-provider.js +364 -106
- package/dist/resources/entity-vocab.js +139 -0
- package/dist/resources/federation-nonce-store.js +195 -0
- package/dist/resources/health.js +97 -6
- package/dist/resources/instance-identity.js +53 -0
- package/dist/resources/mcp-client-metadata-fields.js +112 -0
- package/dist/resources/mcp-handler.js +1 -1
- package/dist/resources/mcp-tools.js +88 -10
- package/dist/resources/memory-bootstrap-lib.js +1 -1
- package/dist/resources/memory-read-scope.js +58 -71
- package/dist/resources/memory-reflect-lib.js +289 -0
- package/dist/resources/memory-visibility.js +37 -0
- package/dist/resources/migration-boot.js +143 -0
- package/dist/resources/migrations/dir-safety.js +71 -0
- package/dist/resources/migrations/embedding-stamp.js +178 -0
- package/dist/resources/migrations/envelope.js +43 -0
- package/dist/resources/migrations/export.js +38 -0
- package/dist/resources/migrations/ledger.js +55 -0
- package/dist/resources/migrations/lock.js +154 -0
- package/dist/resources/migrations/progress.js +31 -0
- package/dist/resources/migrations/registry.js +36 -0
- package/dist/resources/migrations/risk-policy.js +23 -0
- package/dist/resources/migrations/runner.js +456 -0
- package/dist/resources/migrations/snapshot.js +127 -0
- package/dist/resources/migrations/source-fields.js +93 -0
- package/dist/resources/migrations/space.js +73 -0
- package/dist/resources/migrations/state.js +64 -0
- package/dist/resources/migrations/status.js +39 -0
- package/dist/resources/migrations/synthetic-test-migration.js +93 -0
- package/dist/resources/migrations/types.js +9 -0
- package/dist/resources/presence-internal.js +29 -0
- package/dist/resources/provenance.js +50 -0
- package/dist/resources/rate-limiter.js +8 -1
- package/dist/resources/scoring.js +124 -5
- package/dist/resources/semantic-retrieval-core.js +317 -0
- package/dist/version-check.js +167 -0
- package/dist/version-handshake.js +122 -0
- package/package.json +4 -5
- package/schemas/agent.graphql +7 -0
- package/schemas/event.graphql +5 -0
- package/schemas/federation.graphql +12 -0
- package/schemas/memory.graphql +82 -0
- package/schemas/schema.graphql +5 -43
- package/schemas/workspace.graphql +3 -0
- package/dist/resources/IngestEvents.js +0 -162
- package/dist/resources/IssueTokens.js +0 -19
- package/dist/resources/ObsAgentSnapshot.js +0 -13
- package/dist/resources/ObsEventFeed.js +0 -13
- package/dist/resources/ObsOffice.js +0 -19
- package/dist/resources/ObservationCenter.js +0 -23
- package/ui/observation-center.html +0 -385
|
@@ -26,7 +26,7 @@ import { WINDOW_MS, isNonceReplay, recordNonce, importEd25519Key, b64ToArrayBuff
|
|
|
26
26
|
export const FLAIR_AGENT_USERNAME = "flair-agent";
|
|
27
27
|
// ─── Crypto + replay-guard helpers ────────────────────────────────────────────
|
|
28
28
|
// WINDOW_MS, isNonceReplay/recordNonce (the ONE shared nonce store), and
|
|
29
|
-
// importEd25519Key all live in ./ed25519-auth.ts
|
|
29
|
+
// importEd25519Key all live in ./ed25519-auth.ts — the single
|
|
30
30
|
// shared implementation imported by auth-middleware.ts, agent-auth.ts, and
|
|
31
31
|
// Presence.ts so a nonce recorded via any one of the three call sites is
|
|
32
32
|
// visible to the other two, and the crypto/decoder logic can't drift.
|
|
@@ -132,6 +132,52 @@ export async function verifyAgentRequest(request) {
|
|
|
132
132
|
* — and if a request object IS present but yields no agent → anonymous
|
|
133
133
|
* 5. nothing at all → trusted internal call
|
|
134
134
|
*/
|
|
135
|
+
/**
|
|
136
|
+
* Header names that can carry a CLIENT-PRESENTED credential. Today only the
|
|
137
|
+
* standard `authorization` header — it carries Basic, Bearer, AND our custom
|
|
138
|
+
* `TPS-Ed25519 …` scheme (all three ride the Authorization header; see
|
|
139
|
+
* doVerify() above and auth-middleware.ts, both of which read exactly this
|
|
140
|
+
* header to authenticate a caller). Kept as a LIST, read via every header-object
|
|
141
|
+
* shape below, so a future signed-header auth scheme is a one-line addition here
|
|
142
|
+
* — not a scatter of ad-hoc `req.headers.authorization` truthiness checks that
|
|
143
|
+
* would each have to be rediscovered and updated (Kern's review point).
|
|
144
|
+
*
|
|
145
|
+
* DELIBERATELY EXCLUDES the server-set trust markers `x-tps-agent` /
|
|
146
|
+
* `x-tps-anonymous`: auth-middleware.ts STAMPS those AFTER it has verified a
|
|
147
|
+
* caller — they are never presented by the client. Counting them as "credential
|
|
148
|
+
* evidence" would let an unauthenticated caller forge one and defeat this gate.
|
|
149
|
+
*/
|
|
150
|
+
const CREDENTIAL_HEADERS = ["authorization"];
|
|
151
|
+
/**
|
|
152
|
+
* Read a header off either header-object shape Harper hands us: the Web
|
|
153
|
+
* Headers-like `.get(name)` (populated for GET/search requests) or the plain
|
|
154
|
+
* `.asObject` bag (PUT/POST). Mirrors doVerify()'s own read exactly, so the
|
|
155
|
+
* evidence check can't miss a shape the verifier would have seen.
|
|
156
|
+
*/
|
|
157
|
+
function readRequestHeader(reqLike, name) {
|
|
158
|
+
return (reqLike?.headers?.get?.(name) ??
|
|
159
|
+
reqLike?.headers?.asObject?.[name] ??
|
|
160
|
+
"");
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* True iff the request carries ANY client-presented credential (CREDENTIAL_HEADERS).
|
|
164
|
+
*
|
|
165
|
+
* WHY THIS EXISTS (flair#610): Harper's `authorizeLocal: true` forges
|
|
166
|
+
* `request.user = super_user` (and can populate `.username`) for a credential-
|
|
167
|
+
* LESS loopback request — one with NO Authorization header at all. A genuine
|
|
168
|
+
* Basic/Bearer/TPS-Ed25519 header SUPPRESSES that forgery. So "is a credential
|
|
169
|
+
* present?" is precisely what separates a genuinely-authenticated caller from
|
|
170
|
+
* Harper's ambient forgery — the gate resolveAgentAuth applies before trusting a
|
|
171
|
+
* `context.user` identity. Handles BOTH context shapes (`getContext().request`
|
|
172
|
+
* for GET/search, context-only for PUT/POST) by resolving `request ?? self`
|
|
173
|
+
* first, then reads both header-object shapes.
|
|
174
|
+
*/
|
|
175
|
+
export function hasCredentialEvidence(context) {
|
|
176
|
+
const reqLike = context?.request ?? context;
|
|
177
|
+
if (!reqLike)
|
|
178
|
+
return false;
|
|
179
|
+
return CREDENTIAL_HEADERS.some((h) => readRequestHeader(reqLike, h) !== "");
|
|
180
|
+
}
|
|
135
181
|
/**
|
|
136
182
|
* allow* for AGENT-FACING resources: permit verified agents, admins/super_user,
|
|
137
183
|
* and trusted internal calls; deny anonymous HTTP. Pass getContext(). Per-record
|
|
@@ -159,11 +205,23 @@ export async function resolveAgentAuth(context) {
|
|
|
159
205
|
if (c.tpsAgent) {
|
|
160
206
|
return { kind: "agent", agentId: String(c.tpsAgent), isAdmin: c.tpsAgentIsAdmin === true };
|
|
161
207
|
}
|
|
208
|
+
// flair#610 — CREDENTIAL-EVIDENCE GATE. Harper's `authorizeLocal: true` forges
|
|
209
|
+
// `context.user = super_user` (and can populate `.username`) for a credential-
|
|
210
|
+
// LESS loopback request. Trusting that ambient identity was the forgery
|
|
211
|
+
// vector: a bare local caller resolved to admin with no signature and no
|
|
212
|
+
// password. Only trust a `context.user` identity when the request actually
|
|
213
|
+
// carries a credential (real Basic/Bearer/TPS-Ed25519 header) — the one thing
|
|
214
|
+
// authorizeLocal's forgery cannot manufacture. Without evidence, FALL THROUGH
|
|
215
|
+
// to the verifyAgentRequest fallback below: a genuine signed request still
|
|
216
|
+
// authenticates; a credential-less HTTP request lands on `anonymous` (it has a
|
|
217
|
+
// headers object but no valid agent), and a true in-process call with no
|
|
218
|
+
// request object at all lands on `internal`.
|
|
219
|
+
const credentialed = hasCredentialEvidence(c);
|
|
162
220
|
const user = context?.user ?? c.user;
|
|
163
|
-
if (user?.role?.permission?.super_user === true) {
|
|
221
|
+
if (credentialed && user?.role?.permission?.super_user === true) {
|
|
164
222
|
return { kind: "agent", agentId: String(user.username ?? "admin"), isAdmin: true };
|
|
165
223
|
}
|
|
166
|
-
if (user?.username && user.username !== FLAIR_AGENT_USERNAME) {
|
|
224
|
+
if (credentialed && user?.username && user.username !== FLAIR_AGENT_USERNAME) {
|
|
167
225
|
return { kind: "agent", agentId: String(user.username), isAdmin: false };
|
|
168
226
|
}
|
|
169
227
|
// A raw request with headers is present → verify it; an HTTP request that
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
//
|
|
10
10
|
// Kept free of any @harperfast/harper import so the real logic is unit-testable
|
|
11
11
|
// without spinning up Harper (avoids the simulator-pattern that let the
|
|
12
|
-
// description-fallback leak ship untested
|
|
12
|
+
// description-fallback leak ship untested).
|
|
13
13
|
export function readSoulKind(entry) {
|
|
14
14
|
return String(entry.kind ?? entry.key ?? "").trim().toLowerCase();
|
|
15
15
|
}
|
|
@@ -20,7 +20,7 @@ export function readSoulContent(entry) {
|
|
|
20
20
|
* The public description is ONLY an explicit `kind="description"` soul.
|
|
21
21
|
*
|
|
22
22
|
* There is deliberately no fallback to "the first soul with any content": that
|
|
23
|
-
* was the
|
|
23
|
+
* was the description-fallback leak — an agent with no description soul would publish an
|
|
24
24
|
* arbitrary private soul on the unauthenticated card. Absent an explicit
|
|
25
25
|
* description, publish nothing.
|
|
26
26
|
*/
|
|
@@ -38,7 +38,7 @@ function getAdminPass() {
|
|
|
38
38
|
// an admin — one implementation guarantees they can't diverge.
|
|
39
39
|
// ─── Crypto + replay-guard helpers ────────────────────────────────────────────
|
|
40
40
|
// WINDOW_MS, isNonceReplay/recordNonce (the ONE shared nonce store), and
|
|
41
|
-
// importEd25519Key all live in ./ed25519-auth.ts
|
|
41
|
+
// importEd25519Key all live in ./ed25519-auth.ts — the single
|
|
42
42
|
// shared implementation imported by auth-middleware.ts, agent-auth.ts, and
|
|
43
43
|
// Presence.ts so a nonce recorded via any one of the three call sites is
|
|
44
44
|
// visible to the other two, and the crypto/decoder logic can't drift.
|
|
@@ -49,7 +49,9 @@ async function backfillEmbedding(memoryId) {
|
|
|
49
49
|
return;
|
|
50
50
|
if (record.embedding?.length > 100)
|
|
51
51
|
return;
|
|
52
|
-
|
|
52
|
+
// flair#504 Phase 2: 'document' — a backfilled embedding IS a stored
|
|
53
|
+
// document vector, same as the three Memory.ts sites; must match.
|
|
54
|
+
const embedding = await getEmbedding(record.content, "document");
|
|
53
55
|
if (!embedding)
|
|
54
56
|
return;
|
|
55
57
|
await patchRecord(databases.flair.Memory, memoryId, { embedding });
|
|
@@ -90,23 +92,50 @@ server.http(async (request, nextLayer) => {
|
|
|
90
92
|
url.pathname === "/OAuthRevoke" ||
|
|
91
93
|
url.pathname === "/.well-known/oauth-authorization-server" ||
|
|
92
94
|
url.pathname === "/OAuthMetadata" ||
|
|
93
|
-
// ObservationCenter HTML shell is public — the page itself is just markup
|
|
94
|
-
// and inline JS, with no embedded data. The JS prompts for admin-pass and
|
|
95
|
-
// auths every API call (/Agent, /SemanticSearch, /FederationPeers, etc).
|
|
96
|
-
// Without this allow-list entry, the HTML is 401-blocked on hosted Flair
|
|
97
|
-
// instances (rockit-local works only because authorizeLocal=true).
|
|
98
|
-
url.pathname === "/ObservationCenter" ||
|
|
99
95
|
// Presence roster is public-safe (field-allowlisted); GET serves the
|
|
100
|
-
// Office Space renderer without auth.
|
|
101
|
-
//
|
|
102
|
-
|
|
96
|
+
// Office Space renderer without auth. Scoped to GET only (#604): the
|
|
97
|
+
// exact-path match used to match ANY method, so a bare `PUT /Presence`
|
|
98
|
+
// (collection-level, no id — Harper routes it to the same .put() as
|
|
99
|
+
// by-id PUT) early-returned here too, skipping this middleware entirely.
|
|
100
|
+
// A credential-less loopback PUT then reached Presence.put()'s
|
|
101
|
+
// resolveAgentAuth() call with NO tpsAnonymous/tpsAgent annotation, which
|
|
102
|
+
// fell through to raw `context.user` — populated by Harper's
|
|
103
|
+
// `authorizeLocal` (config true) ambient super_user injection for ANY
|
|
104
|
+
// credential-less loopback request — so the ownership check saw an
|
|
105
|
+
// "admin" caller (isAdmin=true) and let the write through unauthenticated
|
|
106
|
+
// (`super.put()`, no signature, no password). Mirrors the A2A GET-only
|
|
107
|
+
// pattern above: POST/PUT/DELETE now always transit the general
|
|
108
|
+
// middleware path below, which marks a genuinely headerless request
|
|
109
|
+
// tpsAnonymous BEFORE Harper's ambient elevation lands (resolveAgentAuth
|
|
110
|
+
// checks tpsAnonymous first — see agent-auth.ts's resolution order), so
|
|
111
|
+
// the ownership check in Presence.put()/delete() correctly denies it.
|
|
112
|
+
// POST (the heartbeat) is unaffected in practice: it already prefers
|
|
113
|
+
// request.tpsAgent when the middleware set it, and falls back to its own
|
|
114
|
+
// Ed25519 header parse otherwise — transiting the general path now just
|
|
115
|
+
// means a genuinely headerless POST gets marked anonymous (still 401)
|
|
116
|
+
// instead of skipping straight to that fallback parse.
|
|
117
|
+
(request.method === "GET" && url.pathname === "/Presence"))
|
|
103
118
|
return nextLayer(request);
|
|
119
|
+
// Read the Authorization header ONCE, up front — the super_user branch below
|
|
120
|
+
// needs it too (hoisted from its former position just after the branch as part
|
|
121
|
+
// of the flair#610 belt-and-suspenders check).
|
|
122
|
+
const header = request.headers.get("authorization") || request.headers?.asObject?.authorization || "";
|
|
104
123
|
// If Harper has already authorized this request (e.g. Basic admin, or
|
|
105
124
|
// authorizeLocal=true on localhost), trust Harper's auth decision and pass
|
|
106
125
|
// through. Annotate the admin identity so resources' resolveAgentAuth recognizes
|
|
107
126
|
// this as an ADMIN caller (not anonymous) — otherwise a Basic-admin request,
|
|
108
127
|
// which carries no TPS-Ed25519 header, gets classified anonymous and denied.
|
|
109
|
-
|
|
128
|
+
//
|
|
129
|
+
// flair#610 BELT-AND-SUSPENDERS: require an Authorization header to be present
|
|
130
|
+
// before trusting a super_user `request.user`. Harper's `authorizeLocal: true`
|
|
131
|
+
// forges request.user=super_user for a credential-LESS loopback request; a
|
|
132
|
+
// genuine Basic/super_user caller always carries a header. This is defense-in-
|
|
133
|
+
// depth — the general middleware path below already marks a headerless request
|
|
134
|
+
// tpsAnonymous BEFORE Harper's ambient elevation lands, so this branch isn't a
|
|
135
|
+
// live vector today — but it keeps the trust decision from ever hinging on
|
|
136
|
+
// ambient elevation alone. (The root-cause gate lives in resolveAgentAuth; see
|
|
137
|
+
// agent-auth.ts hasCredentialEvidence.)
|
|
138
|
+
if (header && request.user?.role?.permission?.super_user === true) {
|
|
110
139
|
request.tpsAgent = request.user.username ?? "admin";
|
|
111
140
|
request.tpsAgentIsAdmin = true;
|
|
112
141
|
try {
|
|
@@ -120,7 +149,6 @@ server.http(async (request, nextLayer) => {
|
|
|
120
149
|
// Skip re-entry: if we already swapped auth to Basic, pass through
|
|
121
150
|
if (request._tpsAuthVerified)
|
|
122
151
|
return nextLayer(request);
|
|
123
|
-
const header = request.headers.get("authorization") || request.headers?.asObject?.authorization || "";
|
|
124
152
|
// ── Basic admin / super_user auth ──────────────────────────────────────────
|
|
125
153
|
// Allow Basic auth for CLI operations (backup, etc.). Two paths:
|
|
126
154
|
// 1. HDB_ADMIN_PASSWORD env-var fast-path (user must be "admin" with exact pass)
|
|
@@ -477,7 +505,7 @@ server.http(async (request, nextLayer) => {
|
|
|
477
505
|
if (memId) {
|
|
478
506
|
const record = await databases.flair.Memory.get(memId);
|
|
479
507
|
if (record && record.agentId && record.agentId !== agentId) {
|
|
480
|
-
// Centralized read-scope (
|
|
508
|
+
// Centralized read-scope (Layer 1): a grant only covers
|
|
481
509
|
// the owner's SHARED memories, never their private ones. This
|
|
482
510
|
// used to be a `visibility === "office"` bypass (any authenticated
|
|
483
511
|
// agent, no grant needed) — that's gone; the private-exclusion is
|
|
@@ -72,7 +72,7 @@ export function passesRecordFilters(record, f = {}) {
|
|
|
72
72
|
return false;
|
|
73
73
|
if (f.asOf && record?.validTo && record.validTo <= f.asOf)
|
|
74
74
|
return false;
|
|
75
|
-
//
|
|
75
|
+
// A past validTo ALWAYS means the record has been closed out —
|
|
76
76
|
// either by the server supersede path (Memory.ts closeSupersededRecord sets
|
|
77
77
|
// validTo without necessarily setting `archived`) or any other writer. This
|
|
78
78
|
// is unconditional (not gated on `f.asOf`) so a server-superseded record
|
package/dist/resources/bm25.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// ─── BM25 + union-RRF hybrid retrieval (Harper-free, unit-testable) ──────────
|
|
2
|
-
// Per spec FLAIR-BM25-HYBRID-RETRIEVAL (Kern-approved
|
|
2
|
+
// Per spec FLAIR-BM25-HYBRID-RETRIEVAL (Kern-approved). This module is
|
|
3
3
|
// deliberately Harper-free — same rationale as ./scoring.ts — so the BM25
|
|
4
4
|
// scoring, the candidate-union RRF fusion, and the security conditions-filter
|
|
5
5
|
// can be unit-tested directly against the SHIPPED code without a live Harper.
|
|
@@ -9,13 +9,19 @@
|
|
|
9
9
|
// candidate-UNION RRF (NOT naive whole-corpus RRF) recovers 4/6 into top-10 with
|
|
10
10
|
// no regression on the within-cluster gate (p@3 holds 0.88).
|
|
11
11
|
// ─── Feature flag: BM25 + union-RRF hybrid retrieval ────────────────────────
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
12
|
+
// ACTIVATED 2026-07-08 (ops-i39b activation follow-up to #519): default is now
|
|
13
|
+
// ON. Recall-eval validated at build time (CHANGELOG 0.20.x): NEW-8
|
|
14
|
+
// within-cluster gate p@3 holds 0.88 (no regression), OLD-6 severe
|
|
15
|
+
// near-verbatim misses recover 0/6 → 4/6 into top-10. A fresh isolated-Harper
|
|
16
|
+
// measurement at activation time (no prod contact) confirmed zero regression
|
|
17
|
+
// on both severe- and within-cluster-style synthetic queries and a small
|
|
18
|
+
// (~+4ms/query) latency delta. Set FLAIR_HYBRID_RETRIEVAL=false (also "0" /
|
|
19
|
+
// "off") to REVERT to the pre-hybrid legacy path — byte-identical to the
|
|
20
|
+
// original default-OFF behavior, no code rollback needed. Read per-call so it
|
|
21
|
+
// can be flipped without a rebuild and set per-case in tests. Lives here
|
|
22
|
+
// (Harper-free) so it's unit-testable.
|
|
17
23
|
export function hybridEnabled() {
|
|
18
|
-
const v = (process.env.FLAIR_HYBRID_RETRIEVAL ?? "").toLowerCase();
|
|
24
|
+
const v = (process.env.FLAIR_HYBRID_RETRIEVAL ?? "true").toLowerCase();
|
|
19
25
|
return v === "true" || v === "1" || v === "on";
|
|
20
26
|
}
|
|
21
27
|
// BM25 parameters (Kern-approved): k1≈1.2, b≈0.75; standard IDF + BM25.
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* collision-lib.ts — pure join/rank/format logic for MemoryBootstrap's
|
|
3
|
+
* "Others in the room" collision-surfacing block (flair#681, the attention-
|
|
4
|
+
* plane flagship — spec: flair#681 "Phase 2 — collision
|
|
5
|
+
* surfacing in bootstrap").
|
|
6
|
+
*
|
|
7
|
+
* This module does NO Harper reads of its own. resources/MemoryBootstrap.ts's
|
|
8
|
+
* post() does the (already-scoped, already-gated) reads —
|
|
9
|
+
* - entity-overlap candidates from WorkspaceState (internal server-side
|
|
10
|
+
* path, Sherlock Option 1 — the #678 AttentionQuery pattern) and OrgEvent
|
|
11
|
+
* (org-open read model, no per-agent scoping to respect),
|
|
12
|
+
* - semantic-match candidates from the SAME scored candidate pool #550's
|
|
13
|
+
* existing code already computes during bootstrap (HNSW cosine
|
|
14
|
+
* similarity against the caller's embedded currentTask, via the bounded
|
|
15
|
+
* retrieveCandidates() core — flair-bootstrap-scale-fix replaced the
|
|
16
|
+
* original full-corpus JS dot-product scan with this bounded pushdown;
|
|
17
|
+
* same `score > 0.3` relevance floor, same "no new embedding code"
|
|
18
|
+
* property — see resources/MemoryBootstrap.ts and resources/
|
|
19
|
+
* semantic-retrieval-core.ts),
|
|
20
|
+
* - the freshness gate from the Presence roster (resources/
|
|
21
|
+
* presence-internal.ts's getPresenceRoster(), the SAME internal path
|
|
22
|
+
* #678 established — never the raw table),
|
|
23
|
+
* and hands the resulting plain-object candidates to this module to join,
|
|
24
|
+
* freshness-gate, rank, and format. Kept Harper-free so it can be unit-tested
|
|
25
|
+
* directly (test/unit/collision-lib.test.ts) — the same reason
|
|
26
|
+
* resources/memory-bootstrap-lib.ts exists for the Team-roster helpers.
|
|
27
|
+
*
|
|
28
|
+
* Per the K&S verdict's correction: Memory is the SEMANTIC surface (reused,
|
|
29
|
+
* via #550); WorkspaceState/OrgEvent are the ENTITY surface (exact index
|
|
30
|
+
* overlap). This module joins the two, never conflating them — an
|
|
31
|
+
* entity-overlap match is always high-precision (exact vocabulary-string
|
|
32
|
+
* equality) and never needs its own relevance score; a semantic-only match
|
|
33
|
+
* always carries the score #550 already floor-gated.
|
|
34
|
+
*/
|
|
35
|
+
/**
|
|
36
|
+
* Build a Harper query condition matching ANY of `entities` against an
|
|
37
|
+
* indexed `entities` array attribute. Harper's real query engine THROWS
|
|
38
|
+
* ("An 'or' operator requires at least two conditions") for an `operator:
|
|
39
|
+
* "or"` condition with fewer than two sub-conditions — a single-entity
|
|
40
|
+
* caller (the common case) would otherwise silently break the whole
|
|
41
|
+
* collision block (caught only by a real-Harper e2e test, never the mocked
|
|
42
|
+
* unit suite — see test/integration/bootstrap-collision-e2e.test.ts).
|
|
43
|
+
* Callers must always call this instead of hand-rolling the OR wrapper.
|
|
44
|
+
*/
|
|
45
|
+
export function buildEntityMatchCondition(entities, attribute = "entities") {
|
|
46
|
+
if (entities.length === 1) {
|
|
47
|
+
return { attribute, comparator: "equals", value: entities[0] };
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
operator: "or",
|
|
51
|
+
conditions: entities.map((e) => ({ attribute, comparator: "equals", value: e })),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
const MS_MIN = 60_000;
|
|
55
|
+
const MS_HOUR = 3600_000;
|
|
56
|
+
const MS_DAY = 24 * MS_HOUR;
|
|
57
|
+
/** "4m ago" / "2h ago" / "3d ago" — mirrors the issue's own example phrasing. */
|
|
58
|
+
export function formatRelativeTime(fromMs, nowMs = Date.now()) {
|
|
59
|
+
const elapsed = Math.max(0, nowMs - fromMs);
|
|
60
|
+
if (elapsed < MS_MIN)
|
|
61
|
+
return "just now";
|
|
62
|
+
if (elapsed < MS_HOUR)
|
|
63
|
+
return `${Math.floor(elapsed / MS_MIN)}m ago`;
|
|
64
|
+
if (elapsed < MS_DAY)
|
|
65
|
+
return `${Math.floor(elapsed / MS_HOUR)}h ago`;
|
|
66
|
+
return `${Math.floor(elapsed / MS_DAY)}d ago`;
|
|
67
|
+
}
|
|
68
|
+
function clip(s, max = 80) {
|
|
69
|
+
const t = s.trim().replace(/\s+/g, " ");
|
|
70
|
+
if (t.length === 0)
|
|
71
|
+
return t;
|
|
72
|
+
return t.length > max ? `${t.slice(0, max - 1)}…` : t;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The freshness gate: only agents with a Presence roster row whose
|
|
76
|
+
* `presenceStatus` is NOT "offline" are candidates at all — reuses
|
|
77
|
+
* Presence.ts's OWN derivePresenceStatus()-driven verdict (a heartbeat
|
|
78
|
+
* within the offline threshold), never a re-implemented threshold. An agent
|
|
79
|
+
* absent from the roster entirely (never heartbeated) is excluded outright —
|
|
80
|
+
* no heartbeat is the strictest possible "not fresh."
|
|
81
|
+
*/
|
|
82
|
+
export function freshPresenceByAgent(roster) {
|
|
83
|
+
const out = new Map();
|
|
84
|
+
for (const row of roster) {
|
|
85
|
+
const id = row?.id;
|
|
86
|
+
if (typeof id !== "string" || id.length === 0)
|
|
87
|
+
continue;
|
|
88
|
+
if (row.presenceStatus === "offline")
|
|
89
|
+
continue;
|
|
90
|
+
const hb = row.lastHeartbeatAt;
|
|
91
|
+
if (hb === null || hb === undefined)
|
|
92
|
+
continue; // no heartbeat at all → not fresh
|
|
93
|
+
const hbNum = typeof hb === "number" ? hb : Number(hb);
|
|
94
|
+
if (!Number.isFinite(hbNum))
|
|
95
|
+
continue;
|
|
96
|
+
out.set(id, row);
|
|
97
|
+
}
|
|
98
|
+
return out;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Join entity-overlap + semantic-match candidates against the freshness-
|
|
102
|
+
* gated presence map, rank (entity overlap first — high precision, exact
|
|
103
|
+
* match — then semantic by score desc), and format each into a single
|
|
104
|
+
* "others in the room" line. Never returns more than one entry per agent
|
|
105
|
+
* (entity detail wins over semantic when a teammate has both — the
|
|
106
|
+
* higher-precision signal leads). Never surfaces an agent absent from
|
|
107
|
+
* `freshByAgent` (the freshness gate) or the caller itself.
|
|
108
|
+
*/
|
|
109
|
+
export function buildCollisionEntries(entityMatches, semanticMatches, freshByAgent, callerAgentId, nowMs = Date.now()) {
|
|
110
|
+
const byAgent = new Map();
|
|
111
|
+
const sortedEntity = entityMatches
|
|
112
|
+
.filter((m) => m.agentId !== callerAgentId && m.entities.length > 0)
|
|
113
|
+
.slice()
|
|
114
|
+
.sort((a, b) => (b.timestamp || "").localeCompare(a.timestamp || ""));
|
|
115
|
+
for (const m of sortedEntity) {
|
|
116
|
+
if (byAgent.has(m.agentId))
|
|
117
|
+
continue; // best (most recent) row per agent wins
|
|
118
|
+
const presence = freshByAgent.get(m.agentId);
|
|
119
|
+
if (!presence)
|
|
120
|
+
continue; // freshness gate
|
|
121
|
+
const displayName = typeof presence.displayName === "string" && presence.displayName
|
|
122
|
+
? presence.displayName
|
|
123
|
+
: m.agentId;
|
|
124
|
+
const hbRaw = presence.lastHeartbeatAt;
|
|
125
|
+
const hb = typeof hbRaw === "number" ? hbRaw : Number(hbRaw);
|
|
126
|
+
const entityList = m.entities.join(", ");
|
|
127
|
+
const detail = m.summary ? ` (${clip(m.summary)})` : "";
|
|
128
|
+
const line = `${displayName} is touching ${entityList}${detail} — last active ${formatRelativeTime(hb, nowMs)}`;
|
|
129
|
+
byAgent.set(m.agentId, { agentId: m.agentId, displayName, kind: "entity", line, lastHeartbeatAt: hb });
|
|
130
|
+
}
|
|
131
|
+
const sortedSemantic = semanticMatches
|
|
132
|
+
.filter((m) => m.agentId !== callerAgentId)
|
|
133
|
+
.slice()
|
|
134
|
+
.sort((a, b) => b.score - a.score);
|
|
135
|
+
for (const m of sortedSemantic) {
|
|
136
|
+
if (byAgent.has(m.agentId))
|
|
137
|
+
continue; // entity-overlap already covers this agent — higher precision wins
|
|
138
|
+
const presence = freshByAgent.get(m.agentId);
|
|
139
|
+
if (!presence)
|
|
140
|
+
continue; // freshness gate
|
|
141
|
+
const displayName = typeof presence.displayName === "string" && presence.displayName
|
|
142
|
+
? presence.displayName
|
|
143
|
+
: m.agentId;
|
|
144
|
+
const hbRaw = presence.lastHeartbeatAt;
|
|
145
|
+
const hb = typeof hbRaw === "number" ? hbRaw : Number(hbRaw);
|
|
146
|
+
const line = `${displayName} has related work in progress (${clip(m.content)}) — last active ${formatRelativeTime(hb, nowMs)}`;
|
|
147
|
+
byAgent.set(m.agentId, { agentId: m.agentId, displayName, kind: "semantic", line, lastHeartbeatAt: hb });
|
|
148
|
+
}
|
|
149
|
+
// Entity-kind entries first (rank order preserved by insertion above is not
|
|
150
|
+
// guaranteed across the two passes' Map writes), then semantic — within
|
|
151
|
+
// each kind, most-recently-active first.
|
|
152
|
+
return [...byAgent.values()].sort((a, b) => {
|
|
153
|
+
if (a.kind !== b.kind)
|
|
154
|
+
return a.kind === "entity" ? -1 : 1;
|
|
155
|
+
return b.lastHeartbeatAt - a.lastHeartbeatAt;
|
|
156
|
+
});
|
|
157
|
+
}
|
package/dist/resources/dedup.js
CHANGED
|
@@ -30,10 +30,10 @@ export const DEDUP_MIN_CONTENT_LENGTH = 20;
|
|
|
30
30
|
* Cosine similarity of two equal-length embedding vectors, computed directly
|
|
31
31
|
* in JS. Used as a fallback for Harper's HNSW cosine-sort query omitting a
|
|
32
32
|
* computed `$distance` on its top candidate when the query's post-filter
|
|
33
|
-
* result set contains exactly ONE matching record (
|
|
33
|
+
* result set contains exactly ONE matching record (found in
|
|
34
34
|
* resources/Memory.ts's findConservativeDedupMatch; the identical quirk is
|
|
35
35
|
* also fixed in resources/SemanticSearch.ts's scoring loop for the same
|
|
36
|
-
* reason
|
|
36
|
+
* reason). A mismatched length, empty vector, or
|
|
37
37
|
* zero-magnitude side yields 0 (no signal, never treated as "identical" by a
|
|
38
38
|
* degenerate computation).
|
|
39
39
|
*/
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
* `importEd25519Key`. Three independent replay windows meant a nonce
|
|
12
12
|
* recorded as "seen" via one path was invisible to the other two — a
|
|
13
13
|
* defense-in-depth gap, and a drift hazard (any future fix to one copy
|
|
14
|
-
* silently didn't apply to the other two).
|
|
15
|
-
*
|
|
14
|
+
* silently didn't apply to the other two). Consolidating all three into
|
|
15
|
+
* this one module means there is exactly ONE replay guard and ONE
|
|
16
16
|
* key-import implementation, imported by all 3 sites.
|
|
17
17
|
*
|
|
18
18
|
* `b64ToArrayBuffer` was already unified into resources/b64.ts in a prior
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* embeddings-boot.ts — registers harper-fabric-embeddings as Harper's
|
|
3
|
+
* `embedding`/`default` model backend DIRECTLY, in-process, on every boot
|
|
4
|
+
* (flair#694 fix; invariants at flair#695).
|
|
5
|
+
*
|
|
6
|
+
* ─── Why this file exists (flair#694) ──────────────────────────────────────
|
|
7
|
+
* The previous mechanism (removed by this change) delivered the registration
|
|
8
|
+
* as a `models.embedding.default` block via the `HARPER_CONFIG` env var
|
|
9
|
+
* (src/cli.ts's old `buildEmbeddingsHarperConfigEnv`) — a "merge layer" that
|
|
10
|
+
* Harper's environment-manager (`@harperfast/harper`'s
|
|
11
|
+
* `config/harperConfigEnvVars.js`) reasserts on every boot AND PERSISTS into
|
|
12
|
+
* the instance-root `harper-config.yaml`. That file proved to be config-as-
|
|
13
|
+
* STATE, not config-as-intent: `flair#694`'s downgrade-and-revert CI lane
|
|
14
|
+
* (flair#692) caught a real bricking sequence —
|
|
15
|
+
*
|
|
16
|
+
* 1. A build with this feature boots and HARPER_CONFIG reasserts
|
|
17
|
+
* `models.embedding.default.{backend,modelName,modelsDir}`, which
|
|
18
|
+
* Harper's env-var layer persists to `harper-config.yaml` AND records
|
|
19
|
+
* each of those THREE flattened leaf paths as `sources[path] =
|
|
20
|
+
* 'HARPER_CONFIG'` in `.harper-config-state.json` (no "original value"
|
|
21
|
+
* stored, because the key didn't exist before this boot introduced it).
|
|
22
|
+
* 2. On ANY later boot that does not reassert `HARPER_CONFIG` for those
|
|
23
|
+
* paths (a downgrade to a build that predates this feature; equally, a
|
|
24
|
+
* transient resolution failure on the current build) — Harper's own
|
|
25
|
+
* `applyRuntimeEnvConfig` treats the env var's absence as "the operator
|
|
26
|
+
* removed this config" and calls `cleanupRemovedEnvVar`, which deletes
|
|
27
|
+
* each of the three leaves INDIVIDUALLY (no stored original to restore
|
|
28
|
+
* to), leaving `models.embedding.default: {}` — an empty shell —
|
|
29
|
+
* persisted to disk.
|
|
30
|
+
* 3. The next boot's config schema validator
|
|
31
|
+
* (`@harperfast/harper`'s `validation/configValidator.js`) resolves the
|
|
32
|
+
* entry's backend via `Joi.alternatives().conditional('.backend', {...,
|
|
33
|
+
* otherwise: unknownBackendEntrySchema})`; with `backend` absent, that
|
|
34
|
+
* falls through to `unknownBackendEntrySchema = Joi.object({backend:
|
|
35
|
+
* string.required()})`, which throws exactly: "Harper config file
|
|
36
|
+
* validation error: 'models.embedding.default.backend' is required" —
|
|
37
|
+
* Harper refuses to boot. Confirmed byte-identical in both @harperfast/
|
|
38
|
+
* harper 5.1.15 and 5.1.17 (this is upstream env-var-config behavior,
|
|
39
|
+
* not a schema difference between versions), and reproduced locally by
|
|
40
|
+
* replaying the exact downgrade-and-revert sequence.
|
|
41
|
+
*
|
|
42
|
+
* No shape flair could put IN HARPER_CONFIG fixes this: the deletion is
|
|
43
|
+
* triggered by an OLDER build never setting the env var in the first place,
|
|
44
|
+
* which is unfixable from the newer build's side. Config that gets torn
|
|
45
|
+
* down whenever a boot doesn't reassert it is fundamentally unsafe for
|
|
46
|
+
* anything a downgrade must survive (flair#695 invariant I: "config files
|
|
47
|
+
* are state too").
|
|
48
|
+
*
|
|
49
|
+
* ─── The fix ────────────────────────────────────────────────────────────
|
|
50
|
+
* Skip Harper's config-file-driven bootstrap path ENTIRELY. Call
|
|
51
|
+
* harper-fabric-embeddings' own `register({logicalName, kind, config})`
|
|
52
|
+
* factory DIRECTLY — the same factory Harper's `bootstrapModels()` would
|
|
53
|
+
* have invoked, and (per that package's own source) the "public path for
|
|
54
|
+
* components and apps to add in-process ... backends":
|
|
55
|
+
* `models.registerBackend(kind, id, backend)` on Harper's process-wide
|
|
56
|
+
* `models` singleton. This is genuinely reassert-only: every boot calls this
|
|
57
|
+
* function again (module-scope side effect, same convention as
|
|
58
|
+
* `migration-boot.ts`), NOTHING is ever written to `harper-config.yaml`, so
|
|
59
|
+
* there is no persisted state for a downgrade to trip over — the class of
|
|
60
|
+
* bug this file fixes cannot recur by construction, not by a shape
|
|
61
|
+
* contract that has to keep being honored.
|
|
62
|
+
*
|
|
63
|
+
* Bonus: this also drops the old absolute-path workaround
|
|
64
|
+
* (`resolveEmbeddingBackendModule`'s `require.resolve` + `pathToFileURL`
|
|
65
|
+
* dance) that HARPER_CONFIG's `backend:` needed because
|
|
66
|
+
* `resolveBackendSpecifier` resolves a bare package name from the Harper
|
|
67
|
+
* INSTANCE ROOT's `node_modules`, not flair's own package dir. Importing
|
|
68
|
+
* harper-fabric-embeddings directly from flair's own code needs no such
|
|
69
|
+
* workaround — plain Node module resolution from flair's own `node_modules`
|
|
70
|
+
* always finds it, uniformly across a local install, Docker, AND a Fabric
|
|
71
|
+
* deploy (where flair runs as a non-root cluster component and
|
|
72
|
+
* `bootstrapModels()` — gated on `isRoot` — was never reachable at all; see
|
|
73
|
+
* the removed comment this replaces in `config.yaml`).
|
|
74
|
+
*
|
|
75
|
+
* ─── Loading mechanism ──────────────────────────────────────────────────
|
|
76
|
+
* Plain (non-Resource) module — same shape as `migration-boot.ts` /
|
|
77
|
+
* `embeddings-provider.ts` / `table-helpers.ts` — so Harper's `jsResource:
|
|
78
|
+
* files: dist/resources/*.js` loader (config.yaml) imports it at boot like
|
|
79
|
+
* every other flat file under `resources/`, running its top-level side
|
|
80
|
+
* effect exactly once per process. No config.yaml wiring needed.
|
|
81
|
+
*
|
|
82
|
+
* Graceful degrade preserved: if harper-fabric-embeddings isn't installed,
|
|
83
|
+
* or `globalThis.models` isn't available (this module loaded outside a real
|
|
84
|
+
* Harper boot), registration is skipped and logged — Harper falls back to
|
|
85
|
+
* keyword-only search, matching the pre-existing degrade contract.
|
|
86
|
+
*/
|
|
87
|
+
import { resolveModelsDir } from "./embeddings-provider.js";
|
|
88
|
+
const LOGICAL_NAME = "default";
|
|
89
|
+
const MODEL_NAME = "nomic-embed-text";
|
|
90
|
+
/**
|
|
91
|
+
* BENCH-ONLY escape hatch — NOT a production feature flag, never documented
|
|
92
|
+
* as an operator setting, and never read anywhere else in this codebase.
|
|
93
|
+
* Mirrors `embeddings-provider.ts`'s `FLAIR_RECALL_HARNESS_FORCE_PREFIX`
|
|
94
|
+
* hatch (see that file's header for the pattern this follows): a bench-only
|
|
95
|
+
* env var, read lazily, that lets `test/bench/recall-harness/run.ts`'s
|
|
96
|
+
* `--model-file <path>` flag override model SELECTION the same way that
|
|
97
|
+
* hatch overrides prefix behavior — without editing this file's hardcoded
|
|
98
|
+
* `MODEL_NAME`/`resolveModelsDir()` call every time a bakeoff needs a
|
|
99
|
+
* different GGUF (e.g. a Q8_0 quant of the same base model).
|
|
100
|
+
*
|
|
101
|
+
* This is not a new capability grafted on — `harper-fabric-embeddings`'
|
|
102
|
+
* `register()` factory already accepts `config.modelPath` as an alternative
|
|
103
|
+
* to `modelName`+`modelsDir` (see `engineOptionsFromConfig()` in
|
|
104
|
+
* `harper-fabric-embeddings`' `dist/index.js`): an absolute path bypasses its
|
|
105
|
+
* built-in model registry and HuggingFace-download resolution entirely. This
|
|
106
|
+
* hatch is the one line that lets a caller reach that existing parameter.
|
|
107
|
+
* `EmbeddingEngine`'s `modelIdentity` (used for nomic-prefix detection, see
|
|
108
|
+
* `engine.js`'s `#applyPrefix`) becomes the file's basename in this path, so
|
|
109
|
+
* prefix behavior is unaffected as long as the GGUF's filename still
|
|
110
|
+
* contains "nomic-embed-text".
|
|
111
|
+
*
|
|
112
|
+
* No production deploy sets this env var, so the unset (overwhelmingly
|
|
113
|
+
* common) case is byte-identical to before this hatch existed.
|
|
114
|
+
*/
|
|
115
|
+
function benchModelPathOverride() {
|
|
116
|
+
return process.env.FLAIR_RECALL_HARNESS_MODEL_PATH || undefined;
|
|
117
|
+
}
|
|
118
|
+
let registered = false;
|
|
119
|
+
/**
|
|
120
|
+
* Register the embedding backend. Idempotent within a process (mirrors
|
|
121
|
+
* `migration-boot.ts`'s `scheduled` guard) — safe to call more than once,
|
|
122
|
+
* only the first call does anything.
|
|
123
|
+
*/
|
|
124
|
+
export async function registerEmbeddingsBackend() {
|
|
125
|
+
if (registered)
|
|
126
|
+
return;
|
|
127
|
+
registered = true;
|
|
128
|
+
try {
|
|
129
|
+
const { register } = await import("harper-fabric-embeddings");
|
|
130
|
+
const modelPath = benchModelPathOverride();
|
|
131
|
+
await register({
|
|
132
|
+
logicalName: LOGICAL_NAME,
|
|
133
|
+
kind: "embedding",
|
|
134
|
+
config: modelPath ? { modelPath } : { modelName: MODEL_NAME, modelsDir: resolveModelsDir() },
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
catch (err) {
|
|
138
|
+
// Not installed, or globalThis.models isn't ready (module loaded outside
|
|
139
|
+
// a real Harper boot, e.g. some future non-Harper import path) — degrade
|
|
140
|
+
// to Harper's keyword-only fallback, the same contract the old
|
|
141
|
+
// HARPER_CONFIG-omitted path preserved.
|
|
142
|
+
console.error(`[embeddings] backend registration skipped: ${err?.message ?? String(err)}`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/** Test-only reset — never used in production (a real process boots once). */
|
|
146
|
+
export function _resetEmbeddingsBackendRegistrationForTests() {
|
|
147
|
+
registered = false;
|
|
148
|
+
}
|
|
149
|
+
void registerEmbeddingsBackend();
|