@tpsdev-ai/flair 0.28.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bridges/builtins/index.js +1 -1
- package/dist/bridges/discover.js +1 -1
- package/dist/bridges/runtime/load-plugin.js +1 -1
- package/dist/bridges/scaffold.js +3 -3
- package/dist/bridges/types.js +1 -1
- package/dist/cli-shim.cjs +1 -1
- package/dist/cli.js +357 -130
- package/dist/deploy.js +29 -19
- package/dist/fabric-upgrade.js +61 -23
- package/dist/fleet-verify.js +3 -3
- package/dist/lib/mcp-enable.js +2 -2
- package/dist/rem/runner.js +2 -2
- package/dist/rem/scheduler.js +226 -17
- package/dist/resources/A2AAdapter.js +1 -1
- package/dist/resources/Admin.js +1 -1
- package/dist/resources/AdminConnectors.js +1 -1
- package/dist/resources/AdminDashboard.js +1 -1
- package/dist/resources/AdminIdp.js +1 -1
- package/dist/resources/AdminInstance.js +1 -1
- package/dist/resources/AdminMemory.js +1 -1
- package/dist/resources/AdminPrincipals.js +1 -1
- package/dist/resources/Agent.js +1 -1
- package/dist/resources/AgentCard.js +1 -1
- package/dist/resources/AgentSeed.js +1 -1
- package/dist/resources/AttentionQuery.js +1 -1
- package/dist/resources/Credential.js +1 -1
- package/dist/resources/Federation.js +1 -1
- package/dist/resources/Instance.js +1 -1
- package/dist/resources/Integration.js +1 -1
- package/dist/resources/MCPClientMetadata.js +1 -1
- package/dist/resources/Memory.js +1 -1
- package/dist/resources/MemoryBootstrap.js +1 -1
- package/dist/resources/MemoryCandidate.js +168 -0
- package/dist/resources/MemoryConsolidate.js +1 -1
- package/dist/resources/MemoryDedupStats.js +1 -1
- package/dist/resources/MemoryFeed.js +1 -1
- package/dist/resources/MemoryGrant.js +1 -1
- package/dist/resources/MemoryMaintenance.js +1 -1
- package/dist/resources/MemoryReflect.js +3 -3
- package/dist/resources/MemoryReindex.js +1 -1
- package/dist/resources/MemoryUsage.js +1 -1
- package/dist/resources/OAuth.js +1 -1
- package/dist/resources/OAuthClient.js +1 -1
- package/dist/resources/OrgEvent.js +1 -1
- package/dist/resources/OrgEventCatchup.js +1 -1
- package/dist/resources/OrgEventMaintenance.js +1 -1
- package/dist/resources/PairingToken.js +1 -1
- package/dist/resources/Peer.js +1 -1
- package/dist/resources/Presence.js +3 -3
- package/dist/resources/RecordUsage.js +16 -2
- package/dist/resources/Relationship.js +1 -1
- package/dist/resources/SemanticSearch.js +1 -1
- package/dist/resources/SkillScan.js +1 -1
- package/dist/resources/Soul.js +1 -1
- package/dist/resources/SoulFeed.js +1 -1
- package/dist/resources/WorkspaceLatest.js +1 -1
- package/dist/resources/WorkspaceState.js +1 -1
- package/dist/resources/XAA.js +1 -1
- package/dist/resources/a2a-url.js +1 -1
- package/dist/resources/agent-auth.js +1 -1
- package/dist/resources/agentcard-fields.js +1 -1
- package/dist/resources/auth-middleware.js +1 -1
- package/dist/resources/embeddings-boot.js +4 -4
- package/dist/resources/embeddings-provider.js +13 -43
- package/dist/resources/federation-cleanup.js +5 -5
- package/dist/resources/federation-nonce-store.js +5 -5
- package/dist/resources/health.js +29 -6
- package/dist/resources/instance-identity.js +1 -1
- package/dist/resources/mcp-client-metadata-fields.js +1 -1
- package/dist/resources/mcp-handler.js +1 -1
- package/dist/resources/mcp-oauth.js +2 -2
- package/dist/resources/mcp-tools.js +9 -1
- package/dist/resources/memory-read-scope.js +1 -1
- package/dist/resources/memory-reflect-lib.js +4 -4
- package/dist/resources/memory-visibility.js +3 -3
- package/dist/resources/migration-boot.js +1 -1
- package/dist/resources/migrations/embedding-stamp.js +3 -3
- package/dist/resources/migrations/graph-heal.js +2 -2
- package/dist/resources/migrations/ledger.js +1 -1
- package/dist/resources/migrations/registry.js +14 -5
- package/dist/resources/migrations/runner.js +64 -11
- package/dist/resources/migrations/synthetic-test-migration.js +1 -1
- package/dist/resources/migrations/visibility-backfill.js +232 -0
- package/dist/resources/models-dir.js +55 -0
- package/dist/resources/record-types.js +30 -2
- package/dist/resources/rerank-provider.js +24 -4
- package/dist/resources/semantic-retrieval-core.js +1 -1
- package/dist/resources/usage-recording.js +73 -7
- package/dist/resources/version.js +33 -0
- package/dist/src/rem/scheduler.js +449 -0
- package/docs/assets/flair-cross-orchestrator.cast +1 -1
- package/docs/bridges.md +3 -3
- package/docs/deployment.md +1 -0
- package/docs/mcp-clients.md +3 -1
- package/docs/n8n.md +1 -2
- package/docs/notes/rem-ux.md +1 -1
- package/docs/rem.md +2 -0
- package/docs/supply-chain-policy.md +4 -4
- package/docs/upgrade.md +2 -2
- package/package.json +4 -4
- package/schemas/memory.graphql +27 -12
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* visibility-backfill.ts — backfills the `visibility` field on Memory rows
|
|
3
|
+
* written before flair#509 (the durability-keyed default-visibility slice),
|
|
4
|
+
* which never stamped `visibility` at all. riskClass 'derived-only':
|
|
5
|
+
* `visibility` here is a derived stamp recomputable at any time from
|
|
6
|
+
* `durability` (which remains on the row, untouched, forever) — never the
|
|
7
|
+
* only copy of anything (invariant I) — and this migration only ever writes
|
|
8
|
+
* into a field that is CURRENTLY EMPTY, so the cheapest posture applies:
|
|
9
|
+
* metadata-only snapshot, no content-hash gate (row-count + stamp
|
|
10
|
+
* convergence only, same as `embedding-stamp`/`graph-heal`).
|
|
11
|
+
*
|
|
12
|
+
* WHY THIS EXISTS: `resources/Memory.ts`'s `defaultVisibilityForDurability`
|
|
13
|
+
* (flair#509) only stamps `visibility` on a WRITE — a row that has sat
|
|
14
|
+
* untouched since before that slice shipped has no `visibility` field at
|
|
15
|
+
* all. `src/cli.ts`'s federation push filter (`isFederationPrivateVisibility`)
|
|
16
|
+
* excludes only `visibility === "private"` — by design, per
|
|
17
|
+
* `resources/memory-visibility.ts`'s migration invariant, an ABSENT
|
|
18
|
+
* `visibility` field is NOT private and must keep syncing exactly as
|
|
19
|
+
* before (so today's single-org fleet never silently stops replicating
|
|
20
|
+
* pre-#509 rows). That is correct while every federation peer IS this same
|
|
21
|
+
* org's own fleet. A cross-organization hub pairing is coming, and at that
|
|
22
|
+
* point an absent-visibility row that should have been private (a
|
|
23
|
+
* `standard`/`ephemeral` note, never intended to leave the instance) would
|
|
24
|
+
* cross an org boundary it never should. This migration closes that gap
|
|
25
|
+
* proactively: once every row carries an explicit `visibility`, "absent"
|
|
26
|
+
* becomes the empty set, and the existing filter (unchanged, zero new
|
|
27
|
+
* behavior in it) is correct as-is for the cross-org case too.
|
|
28
|
+
*
|
|
29
|
+
* THE RULE (flair#509 — reused, not reinvented): `permanent`/`persistent`
|
|
30
|
+
* durability -> `shared`; `standard`/`ephemeral` durability, OR durability
|
|
31
|
+
* absent/unrecognised -> `private` (fail-safe: never widen access on a row
|
|
32
|
+
* whose intent this migration cannot actually determine).
|
|
33
|
+
* `deriveVisibilityFromDurability` below mirrors
|
|
34
|
+
* `resources/Memory.ts`'s (unexported) `defaultVisibilityForDurability`
|
|
35
|
+
* byte-for-byte. Deliberately DUPLICATED rather than imported: importing
|
|
36
|
+
* `resources/Memory.ts` here would drag in its module-top-level
|
|
37
|
+
* `export class Memory extends (databases as any).flair.Memory` — a class
|
|
38
|
+
* EXTENDS that only resolves once a live Harper has already registered the
|
|
39
|
+
* base `Memory` resource, which unit tests (no real Harper; see
|
|
40
|
+
* `test/unit/migrations-visibility-backfill.test.ts`) cannot satisfy. Same
|
|
41
|
+
* reasoning, same shape, as `src/cli.ts`'s `isFederationPrivateVisibility`
|
|
42
|
+
* duplicating `resources/memory-visibility.ts`'s `isPrivateVisibility`
|
|
43
|
+
* rather than importing it (see that file's comment) — both are one-line,
|
|
44
|
+
* pure, and load-bearing enough that a comment pointing at the canonical
|
|
45
|
+
* definition is the safer contract than a shared import with a landmine
|
|
46
|
+
* behind it. If this rule ever changes, both copies must change together.
|
|
47
|
+
*
|
|
48
|
+
* NEVER OVERWRITES an existing `visibility` value — that is an explicit
|
|
49
|
+
* author decision (their own call, even if it disagrees with what this
|
|
50
|
+
* migration would have derived). Only rows where `visibility` is
|
|
51
|
+
* null/undefined are eligible, and `run()` below re-verifies that on a
|
|
52
|
+
* freshly-read record immediately before writing (the actual enforcement
|
|
53
|
+
* of "never overwrite" — see the write-gate comment there), independent of
|
|
54
|
+
* how precisely the candidate-selection query below narrowed things.
|
|
55
|
+
*
|
|
56
|
+
* THE CANDIDATE QUERY — why it looks different from `embedding-stamp`'s:
|
|
57
|
+
* `visibility` is declared in `schemas/memory.graphql` but NOT `@indexed`
|
|
58
|
+
* (unlike `embeddingModel`, which flair#807 needed indexed for its own
|
|
59
|
+
* pending query). Every condition against it is therefore already a
|
|
60
|
+
* full-table-scan filter (`resources/search.ts`'s `filterByType`), whether
|
|
61
|
+
* or not the comparator is negated — the `not_equals`-forces-an-index-bypass
|
|
62
|
+
* mechanism `embedding-stamp.ts` depends on (flair#807) is simply moot for
|
|
63
|
+
* an attribute with no index to be stale in the first place. What DOES
|
|
64
|
+
* still matter, and is the same lesson flair#807 taught: a strict-equals
|
|
65
|
+
* filter (`recordValue === value`) never matches a row whose property was
|
|
66
|
+
* NEVER SET (`recordValue` reads back as `undefined`, and `undefined !==
|
|
67
|
+
* null`), so `{comparator: "equals", value: null}` alone would silently
|
|
68
|
+
* miss every truly-absent row — exactly the rows this migration exists to
|
|
69
|
+
* find. A NEGATED comparator does match them (`undefined !== "private"` is
|
|
70
|
+
* true), so the pending condition here is `visibility not_equals "private"
|
|
71
|
+
* AND visibility not_equals "shared"` — no separate `equals: null` leg
|
|
72
|
+
* needed (unlike `embedding-stamp`, whose write path can leave an
|
|
73
|
+
* intermediate explicit-null state on a failed HTTP regen; this migration's
|
|
74
|
+
* write is a single in-process read-modify-write that either fully
|
|
75
|
+
* succeeds or leaves the row in its exact prior state, so there is no
|
|
76
|
+
* intermediate state to add a leg for). This exact `not_equal`-covers-
|
|
77
|
+
* absent-and-null-and-nothing-else-with-the-value-itself semantics is
|
|
78
|
+
* already relied on, in production, by `resources/memory-read-scope.ts`'s
|
|
79
|
+
* cross-agent read-scope condition on this SAME field (`visibility !=
|
|
80
|
+
* 'private'` — "which INCLUDES records missing the field entirely") — this
|
|
81
|
+
* migration's query is not a new risk, it is the same proven mechanic.
|
|
82
|
+
*
|
|
83
|
+
* A garbage third value (something other than "private"/"shared"/absent)
|
|
84
|
+
* would also match this AND-of-two-`not_equals` condition — Harper's query
|
|
85
|
+
* algebra has no "attribute is absent" primitive to exclude that case at
|
|
86
|
+
* the query level. No code path in this codebase ever writes such a value
|
|
87
|
+
* today (confirmed by grep — every write site either omits `visibility`,
|
|
88
|
+
* defaults it via `defaultVisibilityForDurability`, or passes an operator-
|
|
89
|
+
* supplied string straight through with no allowlist, so nothing rules it
|
|
90
|
+
* out categorically either). The write-gate in `run()` is what actually
|
|
91
|
+
* protects against this, not the query: a row is only ever written to if a
|
|
92
|
+
* freshly-read `.get()` shows `visibility` is STILL null/undefined at write
|
|
93
|
+
* time. A genuinely garbage-valued row would be pulled in as a candidate,
|
|
94
|
+
* re-checked, found to already carry an explicit (if unexpected) value, and
|
|
95
|
+
* skipped — untouched, exactly like any other already-set row. Worst case
|
|
96
|
+
* this makes `countPending()` overcount relative to what `run()` will ever
|
|
97
|
+
* actually touch, which would show up as a real, visible completion-gate
|
|
98
|
+
* halt (`resources/migrations/runner.ts`'s "completion gate failed:
|
|
99
|
+
* rowsRemaining=N") rather than a silent skip or a false success — halt-
|
|
100
|
+
* don't-brick (invariant II), and worth surfacing loudly if it ever
|
|
101
|
+
* happens, since a garbage `visibility` value is itself a bug worth seeing.
|
|
102
|
+
*
|
|
103
|
+
* THE WRITE PATH — deliberately NOT `embedding-stamp`'s loopback-HTTP
|
|
104
|
+
* `PUT /Memory/:id` trick. That mechanism exists ONLY because
|
|
105
|
+
* `embedding-stamp` needs `resources/Memory.ts`'s SUBCLASS override (the
|
|
106
|
+
* `content.content && !content.embedding` regen branch) to actually fire,
|
|
107
|
+
* and the raw `databases.flair.Memory` reference (what this migration and
|
|
108
|
+
* that one both get via their table accessor) resolves to the RAW
|
|
109
|
+
* underlying table, silently skipping every subclass override — see that
|
|
110
|
+
* file's module doc for the full, empirically-confirmed mechanism. This
|
|
111
|
+
* migration needs NO subclass business logic (no dedup, no regen, no
|
|
112
|
+
* provenance stamping, no rate limiting) — it is a narrow, single-field
|
|
113
|
+
* patch, which is exactly the shape `resources/table-helpers.ts`'s
|
|
114
|
+
* `patchRecord` documents and this codebase already uses directly against
|
|
115
|
+
* the RAW table for the same kind of narrow metadata write:
|
|
116
|
+
* `resources/auth-middleware.ts`'s `backfillEmbedding()` (patches
|
|
117
|
+
* `embedding`), `resources/MemoryReflect.ts` and `resources/Memory.ts`
|
|
118
|
+
* itself (both patch `lastReflected` this exact way). Per that file's own
|
|
119
|
+
* warning, Harper's raw `.put()` is FULL RECORD REPLACEMENT — a partial
|
|
120
|
+
* object silently deletes every field not included — so `run()` below
|
|
121
|
+
* always spreads the freshly-read `existing` record before adding
|
|
122
|
+
* `visibility`, the same read-modify-write shape `patchRecord` implements
|
|
123
|
+
* (inlined here rather than calling that helper directly only because
|
|
124
|
+
* `run()` already needs its own `.get()` for the write-gate check above,
|
|
125
|
+
* and re-using that same read avoids a second, redundant round-trip).
|
|
126
|
+
*
|
|
127
|
+
* NO `recheckPending()`: that hook exists specifically to catch a stale
|
|
128
|
+
* SECONDARY INDEX diverging from live values (flair#807, `embedding-stamp`
|
|
129
|
+
* doc above) — a failure mode that requires an index to go stale in the
|
|
130
|
+
* first place. `visibility` has no index at all (every read here is
|
|
131
|
+
* already a live-record scan), so there is no analogous divergence for a
|
|
132
|
+
* safety net to guard against. Same reasoning `graph-heal.ts` (also
|
|
133
|
+
* `derived-only`, also unindexed-field-adjacent) already applies by also
|
|
134
|
+
* omitting it.
|
|
135
|
+
*/
|
|
136
|
+
import { databases } from "harper";
|
|
137
|
+
function defaultMemoryTable() {
|
|
138
|
+
return databases.flair.Memory;
|
|
139
|
+
}
|
|
140
|
+
export const VISIBILITY_BACKFILL_ID = "visibility-backfill";
|
|
141
|
+
/**
|
|
142
|
+
* flair#509's rule, reused verbatim (see module doc for why this is a
|
|
143
|
+
* deliberate duplicate of `resources/Memory.ts`'s
|
|
144
|
+
* `defaultVisibilityForDurability` rather than an import). Total function —
|
|
145
|
+
* every input maps to exactly one of the two valid outputs, and the
|
|
146
|
+
* fail-safe branch (`private`) is the ELSE of the allowlist check, so any
|
|
147
|
+
* absent/null/unrecognised/wrong-typed `durability` value falls into it by
|
|
148
|
+
* construction, never needs its own case.
|
|
149
|
+
*/
|
|
150
|
+
export function deriveVisibilityFromDurability(durability) {
|
|
151
|
+
return durability === "permanent" || durability === "persistent" ? "shared" : "private";
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Rows this migration still needs to touch: `visibility` is neither
|
|
155
|
+
* "private" nor "shared" — see module doc for why this AND-of-two-
|
|
156
|
+
* `not_equals` correctly matches both a truly-absent property and an
|
|
157
|
+
* explicit `null`, without a separate `equals: null` leg.
|
|
158
|
+
*/
|
|
159
|
+
function pendingCondition() {
|
|
160
|
+
return [
|
|
161
|
+
{ attribute: "visibility", comparator: "not_equals", value: "private" },
|
|
162
|
+
{ attribute: "visibility", comparator: "not_equals", value: "shared" },
|
|
163
|
+
];
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* `getTable` is injectable so unit tests exercise this migration's full
|
|
167
|
+
* detect/countPending/run logic against an in-memory fake table (matching
|
|
168
|
+
* `embedding-stamp.ts`'s DI style) — no real Harper needed. Unlike
|
|
169
|
+
* `embedding-stamp`, there is no separate HTTP mechanism to inject: the
|
|
170
|
+
* write IS the table's own `.put()`, already covered by the same fake.
|
|
171
|
+
*/
|
|
172
|
+
export function createVisibilityBackfillMigration(getTable = defaultMemoryTable) {
|
|
173
|
+
return {
|
|
174
|
+
id: VISIBILITY_BACKFILL_ID,
|
|
175
|
+
riskClass: "derived-only",
|
|
176
|
+
affectsTables: ["Memory"],
|
|
177
|
+
async detect() {
|
|
178
|
+
const table = getTable();
|
|
179
|
+
for await (const _row of table.search({ conditions: pendingCondition(), limit: 1 })) {
|
|
180
|
+
return true;
|
|
181
|
+
}
|
|
182
|
+
return false;
|
|
183
|
+
},
|
|
184
|
+
async countPending() {
|
|
185
|
+
const table = getTable();
|
|
186
|
+
let n = 0;
|
|
187
|
+
for await (const _row of table.search({ conditions: pendingCondition() }))
|
|
188
|
+
n++;
|
|
189
|
+
return n;
|
|
190
|
+
},
|
|
191
|
+
async run(batchSize) {
|
|
192
|
+
const table = getTable();
|
|
193
|
+
const candidates = [];
|
|
194
|
+
for await (const row of table.search({ conditions: pendingCondition(), limit: batchSize })) {
|
|
195
|
+
candidates.push(row);
|
|
196
|
+
}
|
|
197
|
+
const touchedIds = [];
|
|
198
|
+
for (const row of candidates) {
|
|
199
|
+
const id = String(row.id ?? "");
|
|
200
|
+
if (!id)
|
|
201
|
+
continue;
|
|
202
|
+
const existing = await table.get(id);
|
|
203
|
+
if (!existing)
|
|
204
|
+
continue; // deleted since the search above — nothing to fix
|
|
205
|
+
// The write-gate: this is what actually enforces "never overwrite an
|
|
206
|
+
// existing visibility value", independent of how precisely the
|
|
207
|
+
// candidate query above narrowed things (see module doc — a
|
|
208
|
+
// garbage third value would still be pulled in as a candidate).
|
|
209
|
+
// Re-checks the FRESHLY-READ record, not the (possibly stale)
|
|
210
|
+
// search-result row, and doubles as the idempotency/concurrent-
|
|
211
|
+
// writer guard `embedding-stamp.ts` gets from its own analogous
|
|
212
|
+
// pre-write check.
|
|
213
|
+
if (existing.visibility !== undefined && existing.visibility !== null)
|
|
214
|
+
continue;
|
|
215
|
+
const derived = deriveVisibilityFromDurability(existing.durability);
|
|
216
|
+
// Never-widen invariant, asserted, not just tested: the only two
|
|
217
|
+
// values `deriveVisibilityFromDurability` can ever produce are
|
|
218
|
+
// "private"/"shared" (see its own doc), so this can never fire
|
|
219
|
+
// today — it exists to fail loudly (halting this migration via the
|
|
220
|
+
// runner's mid-batch-throw path, never a silent bad write) if a
|
|
221
|
+
// future edit to that function ever widens its return type or is
|
|
222
|
+
// bypassed via a type-unsafe call.
|
|
223
|
+
if (derived !== "private" && derived !== "shared") {
|
|
224
|
+
throw new Error(`visibility-backfill: derived an invalid visibility for row ${id} — refusing to write`);
|
|
225
|
+
}
|
|
226
|
+
await table.put({ ...existing, visibility: derived });
|
|
227
|
+
touchedIds.push(id);
|
|
228
|
+
}
|
|
229
|
+
return { processed: touchedIds.length, touchedIds };
|
|
230
|
+
},
|
|
231
|
+
};
|
|
232
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* models-dir.ts — the ONE place that decides where model GGUFs live.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from embeddings-provider.ts (flair#815) so the reranker
|
|
5
|
+
* (rerank-provider.ts) can share the exact same resolution WITHOUT importing
|
|
6
|
+
* embeddings-provider: several unit-isolated tests `mock.module()` the whole
|
|
7
|
+
* embeddings-provider module (with only the named exports THEY consume), and
|
|
8
|
+
* bun's module cache is process-wide — so any new cross-module import of
|
|
9
|
+
* embeddings-provider from another resource makes those mocks incomplete and
|
|
10
|
+
* kills unrelated test files at module load. This module is tiny, pure
|
|
11
|
+
* node-stdlib, and never mocked, so both providers (and any future model
|
|
12
|
+
* consumer) can depend on it safely.
|
|
13
|
+
*/
|
|
14
|
+
import { existsSync } from "node:fs";
|
|
15
|
+
import { homedir } from "node:os";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
/**
|
|
18
|
+
* Resolve the directory model GGUFs live in / download into — shared by the
|
|
19
|
+
* embedding engine (embeddings-boot.ts's `register()` options) and the
|
|
20
|
+
* cross-encoder reranker (rerank-provider.ts, flair#815 — it used to hardcode
|
|
21
|
+
* `<cwd>/models`, silently failing open wherever Harper's cwd wasn't the
|
|
22
|
+
* models location).
|
|
23
|
+
*
|
|
24
|
+
* Resolution order (everything writable, never the read-only package dir):
|
|
25
|
+
* 1. FLAIR_MODELS_DIR — explicit operator/docker override.
|
|
26
|
+
* 2. <ROOTPATH>/models — Harper's data dir (Flair passes ROOTPATH =
|
|
27
|
+
* ~/.flair/data when it spawns Harper). User-
|
|
28
|
+
* owned and writable even on sudo-global installs.
|
|
29
|
+
* 3. <cwd>/models — ONLY if a model already lives there. Backward
|
|
30
|
+
* compat for existing writable installs that
|
|
31
|
+
* downloaded into the package dir before this fix;
|
|
32
|
+
* never used as a download target on fresh installs.
|
|
33
|
+
* 4. ~/.flair/data/models — last-resort default when ROOTPATH is unset.
|
|
34
|
+
*
|
|
35
|
+
* The chosen dir is always writable, so the embeddings engine can download the
|
|
36
|
+
* model on first use without hitting EACCES on a root-owned package dir.
|
|
37
|
+
*
|
|
38
|
+
* Tested independently (test/unit/embeddings-models-dir.test.ts, via
|
|
39
|
+
* embeddings-provider's re-export) as the single documented source of truth
|
|
40
|
+
* for this default.
|
|
41
|
+
*/
|
|
42
|
+
export function resolveModelsDir() {
|
|
43
|
+
const override = process.env.FLAIR_MODELS_DIR;
|
|
44
|
+
if (override)
|
|
45
|
+
return override;
|
|
46
|
+
const rootPath = process.env.ROOTPATH;
|
|
47
|
+
if (rootPath)
|
|
48
|
+
return join(rootPath, "models");
|
|
49
|
+
// Backward compat: a prior (writable) install may have the model cached in
|
|
50
|
+
// the package dir. Reuse it rather than re-downloading — but only if present.
|
|
51
|
+
const cwdModels = join(process.cwd(), "models");
|
|
52
|
+
if (existsSync(cwdModels))
|
|
53
|
+
return cwdModels;
|
|
54
|
+
return join(homedir(), ".flair", "data", "models");
|
|
55
|
+
}
|
|
@@ -127,8 +127,8 @@
|
|
|
127
127
|
*
|
|
128
128
|
* `remEligible` — MUST be `false` for every entry in v1 (typed as the
|
|
129
129
|
* literal `false`, not `boolean`, so a stray `true` is a compile error, not
|
|
130
|
-
* just a runtime policy mistake). REM's nightly gather step
|
|
131
|
-
*
|
|
130
|
+
* just a runtime policy mistake). REM's nightly gather step (see docs/rem.md)
|
|
131
|
+
* is hardcoded to `GET /Memory?agentId=…`
|
|
132
132
|
* — single table, single agent, by construction. There is no multi-table
|
|
133
133
|
* distillation input path to opt into yet; the field exists so a future
|
|
134
134
|
* REM generalization doesn't require a breaking schema change to every
|
|
@@ -315,6 +315,34 @@ export const RECORD_TYPES = {
|
|
|
315
315
|
federation: "included",
|
|
316
316
|
mcp: { toolPrefix: "soul", readVerbs: ["get"], writeVerbs: ["store"] },
|
|
317
317
|
},
|
|
318
|
+
// MemoryCandidate — flair#849. A candidate is an unreviewed draft
|
|
319
|
+
// distillation staged by the FLAIR-NIGHTLY-REM cycle (docs/rem.md);
|
|
320
|
+
// `readScope: "owner-only"` (not "open-within-org") is deliberate: a
|
|
321
|
+
// candidate must not be org-readable before a human/agent reviewer
|
|
322
|
+
// explicitly promotes or rejects it — this is the standard-review-bar
|
|
323
|
+
// choice per this file's header doc's review-gate tiering (an
|
|
324
|
+
// "open-within-org" or "none" entry would additionally require Sherlock's
|
|
325
|
+
// explicit security sign-off, which this entry does not carry). No
|
|
326
|
+
// `embedding` — the claim text has no dedicated recall surface (it isn't
|
|
327
|
+
// Memory.content). No `mcp` — no MCP tool exposes this table today.
|
|
328
|
+
MemoryCandidate: {
|
|
329
|
+
table: "MemoryCandidate",
|
|
330
|
+
ownerField: "agentId",
|
|
331
|
+
identity: "gated",
|
|
332
|
+
readScope: "owner-only",
|
|
333
|
+
attribution: { post: "validate-truthy", put: "validate-truthy" },
|
|
334
|
+
// No provenance field on the schema (schemas/memory.graphql) — nobody
|
|
335
|
+
// stamps buildProvenance() for this table.
|
|
336
|
+
provenance: false,
|
|
337
|
+
remEligible: false,
|
|
338
|
+
// Not in Federation.ts's push table map nor src/cli.ts's
|
|
339
|
+
// runFederationSyncOnce hardcoded list (`["Memory", "Soul", "Agent",
|
|
340
|
+
// "Relationship"]`) — per this file's header doc's federation section,
|
|
341
|
+
// a new type's registration must not silently inherit whatever that
|
|
342
|
+
// filter does or doesn't already exclude, so this is "excluded" by the
|
|
343
|
+
// same reasoning, not by omission.
|
|
344
|
+
federation: "excluded",
|
|
345
|
+
},
|
|
318
346
|
};
|
|
319
347
|
// ─── Composite MCP tools (the second, and only other, reviewed chokepoint) ─
|
|
320
348
|
//
|
|
@@ -70,10 +70,16 @@
|
|
|
70
70
|
* No Ollama (no logprobs, no rerank endpoint — verified), no network hop, no
|
|
71
71
|
* new auth boundary.
|
|
72
72
|
*
|
|
73
|
-
* GGUF files are NOT committed; they are provisioned into models
|
|
74
|
-
* embedding GGUF
|
|
73
|
+
* GGUF files are NOT committed; they are provisioned into the models dir
|
|
74
|
+
* alongside the embedding GGUF — resolved via the SAME `resolveModelsDir()`
|
|
75
|
+
* the embedding engine uses (FLAIR_MODELS_DIR → <ROOTPATH>/models →
|
|
76
|
+
* <cwd>/models → ~/.flair/data/models; flair#815 — this file used to hardcode
|
|
77
|
+
* <cwd>/models, so any deployment whose cwd wasn't the models location failed
|
|
78
|
+
* init and silently fell open to vector order). See docs/rerank-provisioning.md
|
|
79
|
+
* for download sources.
|
|
75
80
|
*/
|
|
76
81
|
import { join } from "node:path";
|
|
82
|
+
import { resolveModelsDir } from "./models-dir.js";
|
|
77
83
|
// Known reranker models → GGUF filename + inference mode. jina is the DEFAULT
|
|
78
84
|
// (working — see file header, flair#811): its rank-pooling path completes
|
|
79
85
|
// inside Harper. qwen3 is EXPERIMENTAL: its generative path is validated
|
|
@@ -181,6 +187,20 @@ function resolveModelKey() {
|
|
|
181
187
|
return requested;
|
|
182
188
|
return DEFAULT_MODEL;
|
|
183
189
|
}
|
|
190
|
+
/**
|
|
191
|
+
* Absolute path the reranker GGUF is expected at: `<models dir>/<file>`,
|
|
192
|
+
* where the models dir is the shared `resolveModelsDir()` (models-dir.ts,
|
|
193
|
+
* same resolution the embedding engine uses) — the single documented source
|
|
194
|
+
* of truth (FLAIR_MODELS_DIR override, ROOTPATH/models, cwd/models backward
|
|
195
|
+
* compat, ~/.flair/data/models default).
|
|
196
|
+
* flair#815: this used to be a hardcoded `<cwd>/models/<file>`, which broke
|
|
197
|
+
* (fail-open, silently) in any deployment where Harper's cwd wasn't the
|
|
198
|
+
* models location — including the recall harness's ephemeral Harpers.
|
|
199
|
+
* Exported for unit testing (see test/unit/rerank-provider.test.ts).
|
|
200
|
+
*/
|
|
201
|
+
export function resolveRerankModelPath(file) {
|
|
202
|
+
return join(resolveModelsDir(), file);
|
|
203
|
+
}
|
|
184
204
|
/**
|
|
185
205
|
* Decide whether `ensureInit()` needs to (re)run the engine init sequence —
|
|
186
206
|
* the flair#811 point-3 fix. Pure so the decision matrix is directly
|
|
@@ -234,9 +254,9 @@ async function ensureInit() {
|
|
|
234
254
|
const spec = MODELS[_modelKey];
|
|
235
255
|
_mode = spec.mode;
|
|
236
256
|
const { existsSync } = await import("node:fs");
|
|
237
|
-
const modelPath =
|
|
257
|
+
const modelPath = resolveRerankModelPath(spec.file);
|
|
238
258
|
if (!existsSync(modelPath)) {
|
|
239
|
-
throw new Error(`reranker GGUF not found:
|
|
259
|
+
throw new Error(`reranker GGUF not found: ${modelPath} (provision per docs/rerank-provisioning.md; set FLAIR_MODELS_DIR to override the models dir)`);
|
|
240
260
|
}
|
|
241
261
|
// Dynamic import — deferred to avoid Harper 5.x VM linker race (same reason
|
|
242
262
|
// embeddings-provider.ts defers harper-fabric-embeddings).
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
// before calling in). Never exposes which internal leg (BM25+RRF hybrid vs.
|
|
37
37
|
// legacy HNSW-only vs. keyword-only fallback) produced a given result — the
|
|
38
38
|
// output shape is identical regardless of `hybrid`.
|
|
39
|
-
import { databases } from "
|
|
39
|
+
import { databases } from "harper";
|
|
40
40
|
import { withDetachedTxn } from "./table-helpers.js";
|
|
41
41
|
import { wrapUntrusted } from "./content-safety.js";
|
|
42
42
|
import { cosineSimilarity } from "./dedup.js";
|
|
@@ -26,10 +26,28 @@
|
|
|
26
26
|
* RecordUsage.post()'s loop, parameterized so Memory.ts can drive it from a
|
|
27
27
|
* write body instead of a dedicated POST body. It NEVER reads the ledger for
|
|
28
28
|
* authority — it only writes contributions; `usedMemoryIds` must never enter
|
|
29
|
-
* an access/scope/attribution/dedup decision (flair#744 slice A invariant 3
|
|
29
|
+
* an access/scope/attribution/dedup decision (flair#744 slice A invariant 3;
|
|
30
|
+
* the flair#775 read-scope gate below is the REVERSE direction — the writer's
|
|
31
|
+
* scope vets the cited ids, the cited ids never widen anything).
|
|
32
|
+
*
|
|
33
|
+
* recordCitations() additionally validates each cited id against the
|
|
34
|
+
* WRITER's own read-scope before crediting (flair#775, a Kern+Sherlock
|
|
35
|
+
* binding condition on the locked design): citing a memory the writer cannot
|
|
36
|
+
* read is silently dropped, uniformly with a nonexistent id — never an
|
|
37
|
+
* error, never a distinguishable code path (an error, or any response/shape
|
|
38
|
+
* difference, would leak "that id exists but you can't see it").
|
|
39
|
+
* POST /RecordUsage is DELIBERATELY not scope-gated — its module doc
|
|
40
|
+
* establishes usage feedback as a cross-agent contract (agent B reports
|
|
41
|
+
* using agent A's memory regardless of A's visibility setting);
|
|
42
|
+
* citation-on-write is a separate write surface with a narrower threat
|
|
43
|
+
* model (the ids ride along on the WRITER's own memory-creation call, so
|
|
44
|
+
* crediting an unreadable id would let a writer both probe for and boost
|
|
45
|
+
* memories it cannot see). So the scope gate lives HERE, not retrofitted
|
|
46
|
+
* onto recordUsageContribution() — do not "unify" the two surfaces.
|
|
30
47
|
*/
|
|
31
|
-
import { databases } from "
|
|
48
|
+
import { databases } from "harper";
|
|
32
49
|
import { withDetachedTxn } from "./table-helpers.js";
|
|
50
|
+
import { resolveReadScope } from "./memory-read-scope.js";
|
|
33
51
|
/**
|
|
34
52
|
* Per-call cap on ids credited in one batch — shared by RecordUsage.post()'s
|
|
35
53
|
* validated `memoryIds` body (rejects a batch over the cap with a 400) and
|
|
@@ -106,6 +124,17 @@ export async function recordUsageContribution(ctx, agentId, memoryId, attributio
|
|
|
106
124
|
return; // deleted between the checks above and now — no-op
|
|
107
125
|
await withDetachedTxn(ctx, () => databases.flair.Memory.put({ ...fresh, usageCount: (fresh.usageCount ?? 0) + 1 }));
|
|
108
126
|
}
|
|
127
|
+
/**
|
|
128
|
+
* Default record fetch for recordCitations()'s read-scope gate — the RAW
|
|
129
|
+
* Memory table (bypassing the Memory RESOURCE class's own read wrapper; the
|
|
130
|
+
* same trusted-internal-caller pattern recordUsageContribution() uses) so
|
|
131
|
+
* `scope.isAllowed` runs against the raw stored record. Never throws: a
|
|
132
|
+
* fetch failure reads as "not found", which the gate silently drops —
|
|
133
|
+
* identical to a nonexistent id.
|
|
134
|
+
*/
|
|
135
|
+
async function fetchMemoryForScopeCheck(ctx, memoryId) {
|
|
136
|
+
return withDetachedTxn(ctx, () => databases.flair.Memory.get(memoryId)).catch(() => null);
|
|
137
|
+
}
|
|
109
138
|
/**
|
|
110
139
|
* Batch citation helper — credits every id in `usedMemoryIds` through
|
|
111
140
|
* `recordFn` (the real `recordUsageContribution` by default), one contribution
|
|
@@ -129,16 +158,30 @@ export async function recordUsageContribution(ctx, agentId, memoryId, attributio
|
|
|
129
158
|
* validated request body, so an oversized list is trimmed rather than
|
|
130
159
|
* rejected (unlike RecordUsage.post()'s validated `memoryIds`, which
|
|
131
160
|
* 400s over the same cap).
|
|
161
|
+
* - Each cited id is validated against the WRITER's read scope before it
|
|
162
|
+
* is credited (flair#775 slice 1, K&S binding condition — see the module
|
|
163
|
+
* doc above): the scope is resolved ONCE per batch via resolveReadScope
|
|
164
|
+
* (the same single source every cross-agent Memory read path uses), then
|
|
165
|
+
* each id gets one raw fetch + one in-process `scope.isAllowed` check.
|
|
166
|
+
* Not-found and out-of-scope take the SAME silent-drop branch — there is
|
|
167
|
+
* no structurally distinguishable code path, error, or response
|
|
168
|
+
* difference between them that a caller could use to probe whether
|
|
169
|
+
* another agent's private id exists.
|
|
132
170
|
* - Each id is credited independently: one id throwing never stops the
|
|
133
171
|
* rest, and the failure is logged server-side, never surfaced to the
|
|
134
172
|
* caller (the write already committed by the time this runs).
|
|
135
173
|
*
|
|
136
|
-
* `agentId` passed to `recordFn`
|
|
137
|
-
* auth context, never anything derived from
|
|
138
|
-
* caller-supplied input (flair#744 slice A
|
|
139
|
-
* behalf of another identity).
|
|
174
|
+
* `agentId` passed to `recordFn` (and to the scope resolution) is ALWAYS
|
|
175
|
+
* `auth.agentId` — the resolved auth context, never anything derived from
|
|
176
|
+
* `usedMemoryIds` or any other caller-supplied input (flair#744 slice A
|
|
177
|
+
* invariant 4: no forging on behalf of another identity).
|
|
178
|
+
*
|
|
179
|
+
* `recordFn` / `fetchFn` / `scopeFn` are unit-test injection seams
|
|
180
|
+
* (test/unit/usage-recording.test.ts) — production callers pass none of
|
|
181
|
+
* them and always get the real recordUsageContribution / raw-table fetch /
|
|
182
|
+
* resolveReadScope.
|
|
140
183
|
*/
|
|
141
|
-
export async function recordCitations(ctx, auth, usedMemoryIds, now, recordFn = recordUsageContribution) {
|
|
184
|
+
export async function recordCitations(ctx, auth, usedMemoryIds, now, recordFn = recordUsageContribution, fetchFn = fetchMemoryForScopeCheck, scopeFn = resolveReadScope) {
|
|
142
185
|
if (auth.kind !== "agent")
|
|
143
186
|
return;
|
|
144
187
|
if (!Array.isArray(usedMemoryIds) ||
|
|
@@ -147,8 +190,31 @@ export async function recordCitations(ctx, auth, usedMemoryIds, now, recordFn =
|
|
|
147
190
|
return;
|
|
148
191
|
}
|
|
149
192
|
const ids = [...new Set(usedMemoryIds)].slice(0, MAX_USAGE_IDS_PER_CALL);
|
|
193
|
+
// Resolve the WRITER's read scope ONCE per batch. Fail CLOSED: if scope
|
|
194
|
+
// resolution itself fails, drop the whole batch rather than credit
|
|
195
|
+
// unvetted ids — citations are advisory signal, so losing a batch is
|
|
196
|
+
// strictly safer than crediting an id the writer may not be able to read.
|
|
197
|
+
let scope;
|
|
198
|
+
try {
|
|
199
|
+
scope = await scopeFn(auth.agentId);
|
|
200
|
+
}
|
|
201
|
+
catch (err) {
|
|
202
|
+
console.error("recordCitations: read-scope resolution failed — batch dropped (no-op)", { err });
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
150
205
|
for (const id of ids) {
|
|
151
206
|
try {
|
|
207
|
+
// flair#775 slice 1 read-scope gate. The raw fetch + in-process
|
|
208
|
+
// predicate is deliberately ONE branch for both "doesn't exist" and
|
|
209
|
+
// "exists but out of the writer's read scope" — uniform silent drop
|
|
210
|
+
// (see the module doc). recordUsageContribution's own existence
|
|
211
|
+
// re-check makes this a second point-lookup of the same record;
|
|
212
|
+
// accepted — keeping the shared ledger core byte-identical for
|
|
213
|
+
// RecordUsage.post() is worth two point-lookups on a ≤20-id advisory
|
|
214
|
+
// batch.
|
|
215
|
+
const record = await fetchFn(ctx, id);
|
|
216
|
+
if (!record || !scope.isAllowed(record))
|
|
217
|
+
continue;
|
|
152
218
|
await recordFn(ctx, auth.agentId, id, undefined, now);
|
|
153
219
|
}
|
|
154
220
|
catch (err) {
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* version.ts — shared runtime-version resolver.
|
|
3
|
+
*
|
|
4
|
+
* Reads the running @tpsdev-ai/flair version from the bundled package.json.
|
|
5
|
+
* `process.env.npm_package_version` is only populated inside `npm run`, so
|
|
6
|
+
* reading package.json relative to THIS running module is the only way to
|
|
7
|
+
* report the version of the code that's actually executing.
|
|
8
|
+
*
|
|
9
|
+
* Extracted from the duplicated copies in resources/Presence.ts and
|
|
10
|
+
* resources/AdminInstance.ts (flair#831). Those modules still carry their
|
|
11
|
+
* own copies for now — migrating them is a follow-up.
|
|
12
|
+
*/
|
|
13
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
14
|
+
import { join, dirname } from "node:path";
|
|
15
|
+
import { fileURLToPath } from "node:url";
|
|
16
|
+
export function resolveVersion() {
|
|
17
|
+
try {
|
|
18
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
19
|
+
const candidates = [
|
|
20
|
+
join(here, "..", "..", "package.json"),
|
|
21
|
+
join(here, "..", "package.json"),
|
|
22
|
+
];
|
|
23
|
+
for (const p of candidates) {
|
|
24
|
+
if (existsSync(p)) {
|
|
25
|
+
const pkg = JSON.parse(readFileSync(p, "utf-8"));
|
|
26
|
+
if (pkg.version)
|
|
27
|
+
return pkg.version;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
catch { /* fall through */ }
|
|
32
|
+
return process.env.npm_package_version ?? "dev";
|
|
33
|
+
}
|