@tpsdev-ai/flair 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +194 -377
- package/dist/cli.js +1355 -281
- package/dist/deploy.js +212 -24
- package/dist/fabric-upgrade.js +16 -1
- package/dist/federation/scheduler.js +500 -0
- package/dist/install/clients.js +111 -53
- package/dist/lib/mcp-spec.js +128 -0
- package/dist/lib/safe-snapshot-extract.js +231 -0
- package/dist/lib/scheduler-platform.js +128 -0
- package/dist/lib/xml-escape.js +54 -0
- package/dist/rem/scheduler.js +35 -87
- package/dist/rem/snapshot.js +13 -0
- package/dist/replication-convergence.js +505 -0
- package/dist/resources/MemoryBootstrap.js +7 -8
- package/dist/resources/SemanticSearch.js +17 -45
- package/dist/resources/abstention.js +1 -1
- package/dist/resources/embeddings-boot.js +10 -12
- package/dist/resources/embeddings-provider.js +10 -7
- package/dist/resources/health.js +24 -19
- package/dist/resources/in-process.js +225 -0
- package/dist/resources/mcp-tools.js +23 -17
- package/dist/resources/migration-boot.js +80 -10
- package/dist/resources/migrations/data-dir.js +205 -0
- package/dist/resources/migrations/progress.js +33 -0
- package/dist/resources/migrations/runner.js +29 -2
- package/dist/resources/migrations/state.js +13 -2
- package/dist/resources/models-dir.js +18 -9
- package/dist/resources/semantic-retrieval-core.js +5 -4
- package/dist/src/lib/scheduler-platform.js +128 -0
- package/dist/src/lib/xml-escape.js +54 -0
- package/dist/src/rem/scheduler.js +35 -87
- package/docs/deploying-on-fabric.md +267 -0
- package/docs/deployment.md +5 -0
- package/docs/embedding-in-a-harper-app.md +299 -0
- package/docs/federation.md +61 -4
- package/docs/integrations.md +3 -0
- package/docs/mcp-clients.md +16 -7
- package/docs/quickstart.md +80 -54
- package/docs/releasing.md +72 -38
- package/docs/supply-chain-policy.md +36 -0
- package/docs/troubleshooting.md +24 -0
- package/docs/upgrade.md +98 -3
- package/package.json +1 -11
- package/templates/bin/flair-federation-sync.sh.tmpl +28 -0
- package/templates/launchd/dev.flair.federation.sync.plist.tmpl +47 -0
- package/templates/systemd/flair-federation-sync.service.tmpl +21 -0
- package/templates/systemd/flair-federation-sync.timer.tmpl +16 -0
- package/dist/resources/rerank-provider.js +0 -569
- package/docs/rerank-provisioning.md +0 -101
|
@@ -98,19 +98,17 @@ const MODEL_NAME = "nomic-embed-text";
|
|
|
98
98
|
* table for the exact contract this PR bumps to).
|
|
99
99
|
*
|
|
100
100
|
* "mean" is nomic-embed-text-v1.5's actual pooling type — NOT assumed from
|
|
101
|
-
* the model's reputation, directly confirmed against the shipped GGUF
|
|
102
|
-
* `
|
|
103
|
-
* ~/.flair/data/models/nomic-embed-text-v1.5.Q4_K_M.gguf` reports
|
|
104
|
-
* `"nomic-bert": { pooling_type: 1 }`, and llama.cpp's
|
|
101
|
+
* the model's reputation, directly confirmed against the shipped GGUF by
|
|
102
|
+
* reading its metadata: `"nomic-bert": { pooling_type: 1 }`, and llama.cpp's
|
|
105
103
|
* `enum llama_pooling_type` maps 1 -> `LLAMA_POOLING_TYPE_MEAN` (0 = none,
|
|
106
|
-
* 1 = mean, 2 = cls, 3 = last, 4 = rank).
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
104
|
+
* 1 = mean, 2 = cls, 3 = last, 4 = rank). (That reading was originally taken
|
|
105
|
+
* with `node-llama-cpp inspect gguf`, which flair no longer depends on —
|
|
106
|
+
* flair#893. Any GGUF inspector reports the same metadata field; install the
|
|
107
|
+
* CLI ad hoc if the check needs repeating.) nomic-embed-text is a
|
|
108
|
+
* NomicBertModel architecture, not Qwen3 (last-token), and flair registers no
|
|
109
|
+
* Qwen3-class embedding model today. If a Qwen3-class (last-token-pooling)
|
|
110
|
+
* embedding model is ever registered here, it must declare `pooling: "last"`,
|
|
111
|
+
* not "mean".
|
|
114
112
|
*
|
|
115
113
|
* Applies to the bench-only `modelPath` override too (`benchModelPathOverride()`
|
|
116
114
|
* below): `FLAIR_RECALL_HARNESS_MODEL_PATH` lets an operator point this
|
|
@@ -152,7 +152,7 @@ const EMBEDDING_PREFIXES_ENABLED = true; // Phase 2 prefix flip ON — re-baseli
|
|
|
152
152
|
* embedding call, so this lets it override the ONE thing that matters
|
|
153
153
|
* (whether `models.embed` receives `inputType`, and whether `getModelId()`
|
|
154
154
|
* stamps the suffix) via the env var it already forwards to `startHarper()`
|
|
155
|
-
* (the same mechanism `FLAIR_HYBRID_RETRIEVAL
|
|
155
|
+
* (the same mechanism `FLAIR_HYBRID_RETRIEVAL` uses).
|
|
156
156
|
*
|
|
157
157
|
* Bidirectional, not two separate force-on/force-off hatches: THE GATE has
|
|
158
158
|
* flipped direction once already (off→on, this PR) and may again in the
|
|
@@ -236,12 +236,15 @@ async function getModelsApi() {
|
|
|
236
236
|
/**
|
|
237
237
|
* Resolve the directory the embeddings model lives in / downloads into.
|
|
238
238
|
*
|
|
239
|
-
* Implementation
|
|
240
|
-
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
*
|
|
244
|
-
*
|
|
239
|
+
* Implementation lives in `resources/models-dir.ts` (extracted in flair#815)
|
|
240
|
+
* so any model consumer can share the exact same resolution without importing
|
|
241
|
+
* this module — several unit-isolated tests `mock.module()` this file
|
|
242
|
+
* wholesale, and an incomplete stub then breaks every file that imports it.
|
|
243
|
+
* See models-dir.ts's header for why that hazard makes the tiny standalone
|
|
244
|
+
* module worth keeping even now that embeddings are its only consumer.
|
|
245
|
+
* Re-exported here so existing consumers keep their import site:
|
|
246
|
+
* `resources/embeddings-boot.ts` (flair#694), which builds the `modelsDir` it
|
|
247
|
+
* passes to harper-fabric-embeddings' `register()`, and
|
|
245
248
|
* test/unit/embeddings-models-dir.test.ts, which pins the resolution order.
|
|
246
249
|
*/
|
|
247
250
|
export { resolveModelsDir } from "./models-dir.js";
|
package/dist/resources/health.js
CHANGED
|
@@ -3,9 +3,9 @@ import { promises as fsp, existsSync, readFileSync } from "node:fs";
|
|
|
3
3
|
import { homedir, platform } from "node:os";
|
|
4
4
|
import { join, dirname } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
|
-
import { getRerankStatus } from "./rerank-provider.js";
|
|
7
6
|
import { allowVerified, resolveAgentAuth } from "./agent-auth.js";
|
|
8
7
|
import { getMigrationStatusSnapshot } from "./migrations/status.js";
|
|
8
|
+
import { resolveMigrationDataDirForRead } from "./migrations/data-dir.js";
|
|
9
9
|
import { REM_DEDUP_STATS_PATH } from "./dedup-cluster.js";
|
|
10
10
|
const db = databases;
|
|
11
11
|
const redactHome = (p) => {
|
|
@@ -560,12 +560,21 @@ export class HealthDetail extends Resource {
|
|
|
560
560
|
// "pre-flight integrity check in progress" state the K&S verdict calls
|
|
561
561
|
// for; a halted migration surfaces here (with `reason`) AND as a
|
|
562
562
|
// warning below so it's visible without a separate lookup.
|
|
563
|
+
//
|
|
564
|
+
// flair#812: the dataDir is resolved by the shared, READ-ONLY resolver
|
|
565
|
+
// (resources/migrations/data-dir.ts) rather than the old
|
|
566
|
+
// `HDB_ROOT ?? homedir()/.flair/data` expression duplicated here — that
|
|
567
|
+
// env var is never set by anything, so the left branch was dead and the
|
|
568
|
+
// path silently disagreed with wherever the boot cycle actually wrote.
|
|
569
|
+
// `lastCycleError` is surfaced so `flair doctor` can name WHY a cycle
|
|
570
|
+
// didn't run, not merely that it didn't.
|
|
563
571
|
try {
|
|
564
|
-
const migrationsDataDir =
|
|
572
|
+
const migrationsDataDir = resolveMigrationDataDirForRead();
|
|
565
573
|
const snapshot = getMigrationStatusSnapshot(migrationsDataDir);
|
|
566
574
|
stats.migrations = {
|
|
567
575
|
cyclePhase: snapshot.cyclePhase,
|
|
568
576
|
lastCycleAt: snapshot.lastCycleAt ?? null,
|
|
577
|
+
lastCycleError: snapshot.lastCycleError ?? null,
|
|
569
578
|
migrations: snapshot.migrations,
|
|
570
579
|
};
|
|
571
580
|
for (const m of snapshot.migrations) {
|
|
@@ -573,13 +582,25 @@ export class HealthDetail extends Resource {
|
|
|
573
582
|
warnings.push({ level: "warn", message: `migration '${m.id}' ${m.state}${m.reason ? `: ${m.reason}` : ""} — see \`flair doctor\`` });
|
|
574
583
|
}
|
|
575
584
|
}
|
|
585
|
+
// The boot trigger sets `scheduled` synchronously at module load, so
|
|
586
|
+
// `idle` here means resources/migration-boot.js never loaded — the
|
|
587
|
+
// instance is running a build whose migration trigger is absent or
|
|
588
|
+
// failed to import, and NO migration will ever run on it.
|
|
589
|
+
if (snapshot.cyclePhase === "idle" && snapshot.migrations.length > 0) {
|
|
590
|
+
warnings.push({
|
|
591
|
+
level: "warn",
|
|
592
|
+
message: "migration boot cycle never fired on this instance (cyclePhase=idle) — no migration will run until this is resolved; see `flair doctor`",
|
|
593
|
+
});
|
|
594
|
+
}
|
|
576
595
|
}
|
|
577
596
|
catch {
|
|
578
597
|
stats.migrations = null;
|
|
579
598
|
}
|
|
580
599
|
// ── Disk ──
|
|
600
|
+
// flair#812: same shared read-only resolution as the migrations section
|
|
601
|
+
// above, so `disk.dataDir` names the directory migrations actually use.
|
|
581
602
|
try {
|
|
582
|
-
const dataDir =
|
|
603
|
+
const dataDir = resolveMigrationDataDirForRead();
|
|
583
604
|
const snapshotDir = join(homedir(), ".flair", "snapshots");
|
|
584
605
|
const dirSize = async (root, maxDepth = 6) => {
|
|
585
606
|
if (!(await exists(root)))
|
|
@@ -658,22 +679,6 @@ export class HealthDetail extends Resource {
|
|
|
658
679
|
catch {
|
|
659
680
|
stats.bridges = null;
|
|
660
681
|
}
|
|
661
|
-
// ── Reranker ──
|
|
662
|
-
// Cross-encoder rerank stage. Reports flag/model/mode + live counters so we
|
|
663
|
-
// can see if it's silently degrading (fallbackCount climbing) in prod.
|
|
664
|
-
try {
|
|
665
|
-
const rr = getRerankStatus();
|
|
666
|
-
stats.rerank = rr;
|
|
667
|
-
if (rr.enabled && rr.state === "failed") {
|
|
668
|
-
warnings.push({ level: "warn", message: `reranker enabled but unavailable: ${rr.error ?? "init failed"} — recall falling back to vector order` });
|
|
669
|
-
}
|
|
670
|
-
if (rr.enabled && rr.rerankCount > 0 && rr.fallbackCount > rr.rerankCount) {
|
|
671
|
-
warnings.push({ level: "warn", message: `reranker falling back more than reranking (${rr.fallbackCount} fallbacks / ${rr.rerankCount} reranks) — check latency budget / topN` });
|
|
672
|
-
}
|
|
673
|
-
}
|
|
674
|
-
catch {
|
|
675
|
-
stats.rerank = null;
|
|
676
|
-
}
|
|
677
682
|
// ── Warnings ──
|
|
678
683
|
stats.warnings = warnings;
|
|
679
684
|
// ── Process info ──
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ─── The in-process call seam (one incantation, one place) ───────────────────
|
|
3
|
+
*
|
|
4
|
+
* How code running INSIDE the Harper process invokes a flair resource AS a
|
|
5
|
+
* specific agent — flair's own native MCP handler (resources/mcp-tools.ts), and
|
|
6
|
+
* any co-located application component that loads flair as a sub-component and
|
|
7
|
+
* calls it directly instead of over HTTP.
|
|
8
|
+
*
|
|
9
|
+
* Three things an in-process caller has to get right, none of which is visible
|
|
10
|
+
* from a resource's own source and none of which used to fail in a way that
|
|
11
|
+
* points at the cause:
|
|
12
|
+
*
|
|
13
|
+
* 1. **Identity.** Every flair resource resolves its caller through
|
|
14
|
+
* resources/agent-auth.ts's `resolveAgentAuth()`, whose FIRST hop is the
|
|
15
|
+
* `tpsAgent` annotation the HTTP auth middleware stamps on a verified
|
|
16
|
+
* request. There is no HTTP request in-process, so the caller supplies that
|
|
17
|
+
* annotation itself — {@link agentContext}.
|
|
18
|
+
*
|
|
19
|
+
* 2. **Not accidentally becoming an administrator.** A context with no usable
|
|
20
|
+
* agent id resolves to flair's trusted `internal` verdict, which is
|
|
21
|
+
* UNFILTERED. See the safety-design block below: this module's API shape is
|
|
22
|
+
* built around making that unreachable by accident.
|
|
23
|
+
*
|
|
24
|
+
* 3. **Collection binding.** A resource's `post()` — the create path — only
|
|
25
|
+
* works on a resource instance Harper has marked as a COLLECTION. That mark
|
|
26
|
+
* is a PRIVATE field (`#isCollection`) set inside Harper's own
|
|
27
|
+
* `getResource()`; the public `isCollection` is a GETTER WITH NO SETTER on
|
|
28
|
+
* `Resource.prototype`. So the obvious `new Cls(undefined, ctx)` +
|
|
29
|
+
* `h.isCollection = true` does not "not work" quietly — under ESM (always
|
|
30
|
+
* strict mode) the assignment throws
|
|
31
|
+
* `TypeError: Cannot set property isCollection ... which has only a getter`,
|
|
32
|
+
* and dropping the assignment instead yields
|
|
33
|
+
* `405 The <X> does not have a post method implemented` from Harper's base
|
|
34
|
+
* `Resource.post()`. {@link collectionResource} is the one supported way in:
|
|
35
|
+
* hand `getResource()` an `{ isCollection: true }` option and let Harper set
|
|
36
|
+
* its own private field.
|
|
37
|
+
*
|
|
38
|
+
* flair itself got (3) wrong in four MCP tool paths before this module existed —
|
|
39
|
+
* the same mistake a consumer reading only the resource classes would make.
|
|
40
|
+
*
|
|
41
|
+
* ─── SAFETY DESIGN: the dangerous thing has to look dangerous ────────────────
|
|
42
|
+
*
|
|
43
|
+
* MEASURED, against the real resolver:
|
|
44
|
+
*
|
|
45
|
+
* resolveAgentAuth({ request: { tpsAgent: undefined } }) -> { kind: "internal" }
|
|
46
|
+
* resolveAgentAuth({ request: { tpsAgent: "" } }) -> { kind: "internal" }
|
|
47
|
+
* allowAdmin({ request: { tpsAgent: undefined } }) -> true
|
|
48
|
+
*
|
|
49
|
+
* `resolveAgentAuth` tests `tpsAgent` for TRUTHINESS, so a missing or empty id
|
|
50
|
+
* is indistinguishable from "no identity was supplied at all" — and that is the
|
|
51
|
+
* trusted verdict. The consequence is that the most ordinary application bug
|
|
52
|
+
* there is — `agentContext(session.agentId)` where the field is undefined, a
|
|
53
|
+
* typo'd property, a lookup that found nothing — would not fail closed. It
|
|
54
|
+
* would silently grant unfiltered cross-agent reads and writes, and pass the
|
|
55
|
+
* admin-only gate on admin-only resources. No error, no 403, no log line.
|
|
56
|
+
*
|
|
57
|
+
* That is the same defect class as a check that cannot fail: **the failure mode
|
|
58
|
+
* of "I forgot the id" must never be "you are now an administrator."** So:
|
|
59
|
+
*
|
|
60
|
+
* - {@link agentContext} THROWS on a missing, empty or blank id. It is always
|
|
61
|
+
* a programming error, and an exception is the only outcome that cannot be
|
|
62
|
+
* mistaken for success.
|
|
63
|
+
* - {@link agentContext} takes NO options. It is structurally incapable of
|
|
64
|
+
* producing an admin context, so spreading a caller-influenced object into
|
|
65
|
+
* its arguments cannot escalate.
|
|
66
|
+
* - Admin is a SEPARATE, NAMED export ({@link adminContext}), and so is the
|
|
67
|
+
* unfiltered maintenance verdict ({@link internalContext}). Both are
|
|
68
|
+
* greppable: `git grep -n "adminContext\|internalContext"` enumerates every
|
|
69
|
+
* privileged call site in a codebase.
|
|
70
|
+
* - {@link collectionResource} REQUIRES a context and throws without one, so
|
|
71
|
+
* the privileged path can never be reached by leaving an argument off. The
|
|
72
|
+
* dangerous path is now the LONGEST path, not the shortest.
|
|
73
|
+
*
|
|
74
|
+
* These guards are runtime, not types, on purpose: an embedding application may
|
|
75
|
+
* be plain JavaScript, where a `string` parameter annotation buys exactly
|
|
76
|
+
* nothing. test/integration/in-process-agents.test.ts pins both the guards and
|
|
77
|
+
* the hazard they exist for, against a real Harper.
|
|
78
|
+
*
|
|
79
|
+
* ── Deliberately dependency-free ─────────────────────────────────────────────
|
|
80
|
+
* No `import { databases } from "harper"` (see resources/memory-visibility.ts
|
|
81
|
+
* for why that import is load-bearing to avoid): these helpers are pure shape,
|
|
82
|
+
* and an embedding app may want them before any table is resolved.
|
|
83
|
+
*/
|
|
84
|
+
/**
|
|
85
|
+
* Thrown when an in-process call is built in a way that would have silently
|
|
86
|
+
* escalated. Its own type so a caller can distinguish "I wired this wrong" from
|
|
87
|
+
* a resource's business-logic refusal (which arrives as a `Response`, not a
|
|
88
|
+
* throw).
|
|
89
|
+
*/
|
|
90
|
+
export class InProcessContextError extends Error {
|
|
91
|
+
constructor(message) {
|
|
92
|
+
super(message);
|
|
93
|
+
this.name = "InProcessContextError";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/** Reject anything that would resolve to the trusted `internal` verdict, or to a nonsense id. */
|
|
97
|
+
function requireAgentId(agentId, fnName) {
|
|
98
|
+
if (typeof agentId !== "string" || agentId.trim() === "") {
|
|
99
|
+
const got = agentId === undefined ? "undefined"
|
|
100
|
+
: agentId === null ? "null"
|
|
101
|
+
: typeof agentId !== "string" ? `a ${typeof agentId}`
|
|
102
|
+
: agentId === "" ? "an empty string"
|
|
103
|
+
: "a blank string";
|
|
104
|
+
throw new InProcessContextError(`${fnName}() requires a non-empty agent id, but received ${got}. ` +
|
|
105
|
+
"This is refused rather than defaulted because flair resolves a missing or empty " +
|
|
106
|
+
"tpsAgent to its trusted `internal` verdict, which reads and writes UNFILTERED across " +
|
|
107
|
+
"every agent and passes the admin-only gate — so returning a context here would have " +
|
|
108
|
+
"turned a missing id into silent administrator access. Resolve the agent id from your " +
|
|
109
|
+
"own server-side state before calling, and never from request data.");
|
|
110
|
+
}
|
|
111
|
+
return agentId;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* The resource context that makes an in-process call act as ONE SPECIFIC agent.
|
|
115
|
+
* This is the call an application makes; the other two constructors below are
|
|
116
|
+
* privileged and deliberately harder to type.
|
|
117
|
+
*
|
|
118
|
+
* **`agentId` is asserted, not proven.** flair reads it and acts as that agent:
|
|
119
|
+
* no signature, no lookup against the `Agent` table, no registration
|
|
120
|
+
* requirement. That is right for a caller already inside the trust boundary —
|
|
121
|
+
* it could write the raw table anyway — and it is exactly why the id must come
|
|
122
|
+
* from YOUR OWN server-side state (the session you authenticated, the job
|
|
123
|
+
* record you dequeued) and **never from request data**. An agent id that
|
|
124
|
+
* reaches here from a body field, a query param, or a header you did not verify
|
|
125
|
+
* yourself is privilege escalation with no error and no trace.
|
|
126
|
+
*
|
|
127
|
+
* Throws {@link InProcessContextError} on a missing, empty or blank id — see the
|
|
128
|
+
* safety-design block at the top of this file for why that is not merely
|
|
129
|
+
* defensive.
|
|
130
|
+
*
|
|
131
|
+
* Takes no options, and in particular no way to ask for admin: use
|
|
132
|
+
* {@link adminContext} for that, so the escalation is a word you had to type.
|
|
133
|
+
*
|
|
134
|
+
* ```ts
|
|
135
|
+
* const Memory = server.resources.get("Memory").Resource;
|
|
136
|
+
* const h = await collectionResource(Memory, agentContext("planner"));
|
|
137
|
+
* const { id } = await h.post({ agentId: "planner", content: "…" });
|
|
138
|
+
* ```
|
|
139
|
+
*/
|
|
140
|
+
export function agentContext(agentId) {
|
|
141
|
+
return {
|
|
142
|
+
request: {
|
|
143
|
+
tpsAgent: requireAgentId(agentId, "agentContext"),
|
|
144
|
+
tpsAgentIsAdmin: false,
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Like {@link agentContext}, but with flair-admin authority: unfiltered
|
|
150
|
+
* cross-agent reads, and writes attributed to agents other than this one.
|
|
151
|
+
*
|
|
152
|
+
* Asserted, never checked — nothing validates that the named agent is actually
|
|
153
|
+
* an admin principal. **Treat a call to this exactly as you would a root
|
|
154
|
+
* shell:** provisioning and maintenance only, never a request handler's
|
|
155
|
+
* default, and never with an id or a flag derived from anything a caller
|
|
156
|
+
* supplied.
|
|
157
|
+
*
|
|
158
|
+
* It is a separate export rather than an option so that (a) no options object
|
|
159
|
+
* spread into {@link agentContext} can escalate, and (b) every privileged call
|
|
160
|
+
* site in a codebase is findable by name.
|
|
161
|
+
*
|
|
162
|
+
* Throws {@link InProcessContextError} on a missing, empty or blank id.
|
|
163
|
+
*/
|
|
164
|
+
export function adminContext(agentId) {
|
|
165
|
+
return {
|
|
166
|
+
request: {
|
|
167
|
+
tpsAgent: requireAgentId(agentId, "adminContext"),
|
|
168
|
+
tpsAgentIsAdmin: true,
|
|
169
|
+
},
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* The TRUSTED, UNATTRIBUTED, UNFILTERED context — flair's `internal` verdict.
|
|
174
|
+
* Reads see every agent's private records; writes are owned by nobody.
|
|
175
|
+
*
|
|
176
|
+
* This exists for work that is genuinely infrastructure rather than an agent's:
|
|
177
|
+
* provisioning a principal, a migration, a maintenance sweep. It is the same
|
|
178
|
+
* verdict a context-less call used to fall into by accident; naming it means an
|
|
179
|
+
* application can no longer reach it by forgetting an argument, and means
|
|
180
|
+
* `git grep internalContext` enumerates every place flair or an embedder took
|
|
181
|
+
* that authority deliberately.
|
|
182
|
+
*
|
|
183
|
+
* There is no id to validate: the verdict comes precisely from the ABSENCE of
|
|
184
|
+
* an identity annotation, so the returned object carries none. The marker field
|
|
185
|
+
* is inert — it documents intent at a debugger breakpoint and is read by
|
|
186
|
+
* nothing.
|
|
187
|
+
*
|
|
188
|
+
* ```ts
|
|
189
|
+
* // Registering a principal — infrastructure, not an agent's own write.
|
|
190
|
+
* const h = await collectionResource(Agent, internalContext());
|
|
191
|
+
* await h.post({ id, name: id, publicKey });
|
|
192
|
+
* ```
|
|
193
|
+
*/
|
|
194
|
+
export function internalContext() {
|
|
195
|
+
return { request: {}, __flairInternal: true };
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* A resource instance bound to `context` and marked as a COLLECTION, so its
|
|
199
|
+
* `post()` (the create path) works — see this module's header for why that mark
|
|
200
|
+
* cannot be applied from outside Harper.
|
|
201
|
+
*
|
|
202
|
+
* `context` is REQUIRED. Passing nothing used to yield the unfiltered
|
|
203
|
+
* `internal` verdict, which made the privileged path the shortest one to type;
|
|
204
|
+
* it now throws. Pass {@link agentContext} for an agent's own work,
|
|
205
|
+
* {@link adminContext} or {@link internalContext} when the authority is
|
|
206
|
+
* genuinely intended.
|
|
207
|
+
*
|
|
208
|
+
* Reads do not need this — `Cls.get(id, context)` and `Cls.search(query,
|
|
209
|
+
* context)` (Harper's static resource methods) already thread the context and
|
|
210
|
+
* start their own transaction.
|
|
211
|
+
*/
|
|
212
|
+
export async function collectionResource(Cls, context) {
|
|
213
|
+
if (context == null || typeof context !== "object") {
|
|
214
|
+
throw new InProcessContextError("collectionResource() requires an explicit context, but received " +
|
|
215
|
+
(context === undefined ? "undefined" : context === null ? "null" : `a ${typeof context}`) +
|
|
216
|
+
". Omitting it would resolve to flair's trusted `internal` verdict — unfiltered reads and " +
|
|
217
|
+
"unattributed writes across every agent — so the privileged path is never the one you get " +
|
|
218
|
+
"by leaving an argument off. Pass agentContext(id) to act as an agent, or internalContext() " +
|
|
219
|
+
"if this really is infrastructure work.");
|
|
220
|
+
}
|
|
221
|
+
// `{}` is a sufficient RequestTarget here: getResource() reads only `.id` off
|
|
222
|
+
// it (undefined ⇒ Harper mints the primary key), and takes the collection
|
|
223
|
+
// mark from the options argument.
|
|
224
|
+
return (await Cls.getResource({}, context, { isCollection: true }));
|
|
225
|
+
}
|
|
@@ -36,6 +36,7 @@
|
|
|
36
36
|
* slice.
|
|
37
37
|
*/
|
|
38
38
|
import { resolveVersion } from "./version.js";
|
|
39
|
+
import { agentContext, adminContext, collectionResource } from "./in-process.js";
|
|
39
40
|
const H = {};
|
|
40
41
|
const LOADERS = {
|
|
41
42
|
SemanticSearch: async () => (await import("./SemanticSearch.js")).SemanticSearch,
|
|
@@ -78,16 +79,22 @@ export function __setHandlers(overrides) {
|
|
|
78
79
|
* agent, never anonymous, never a header re-verify.
|
|
79
80
|
*/
|
|
80
81
|
function delegationContext(agent) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
82
|
+
// Shape single-sourced from resources/in-process.ts — the same context an
|
|
83
|
+
// embedding Harper app builds to act as one of its agents — plus the
|
|
84
|
+
// `x-tps-agent` header shim the delegated handlers' own header-reading paths
|
|
85
|
+
// expect.
|
|
86
|
+
//
|
|
87
|
+
// The admin branch is spelled out rather than passed as a flag: adminContext()
|
|
88
|
+
// is the greppable name for "this call carries flair-admin authority". Both
|
|
89
|
+
// constructors THROW on an empty agent id, which is the outcome we want here —
|
|
90
|
+
// a token that resolved to no principal must fail the tool call, never fall
|
|
91
|
+
// through to flair's unfiltered `internal` verdict.
|
|
92
|
+
const ctx = agent.isAdmin ? adminContext(agent.agentId) : agentContext(agent.agentId);
|
|
93
|
+
ctx.request.headers = {
|
|
94
|
+
get: (k) => (k.toLowerCase() === "x-tps-agent" ? agent.agentId : undefined),
|
|
90
95
|
};
|
|
96
|
+
ctx.user = undefined;
|
|
97
|
+
return ctx;
|
|
91
98
|
}
|
|
92
99
|
/**
|
|
93
100
|
* Unwrap a handler return value into a plain object/string for the MCP result.
|
|
@@ -128,8 +135,7 @@ async function memorySearch(agent, args) {
|
|
|
128
135
|
}
|
|
129
136
|
async function memoryStore(agent, args) {
|
|
130
137
|
const Cls = await handler("Memory");
|
|
131
|
-
const h =
|
|
132
|
-
h.isCollection = true;
|
|
138
|
+
const h = await collectionResource(Cls, delegationContext(agent));
|
|
133
139
|
// agentId is the RESOLVED agent — Memory.post also re-checks ownership via
|
|
134
140
|
// resolveAgentAuth, so a mismatched body agentId would 403 anyway; we set it
|
|
135
141
|
// to the verified id so the write is correctly owned.
|
|
@@ -204,8 +210,10 @@ async function memoryUpdate(agent, args) {
|
|
|
204
210
|
// version's provenance records which client authored this update.
|
|
205
211
|
if (agent.clientId)
|
|
206
212
|
record.claimedClient = agent.clientId;
|
|
207
|
-
|
|
208
|
-
|
|
213
|
+
// A create needs a COLLECTION-bound instance (see resources/in-process.ts);
|
|
214
|
+
// `h` above is the by-id handle the get()/put() branches use.
|
|
215
|
+
const coll = await collectionResource(Cls, delegationContext(agent));
|
|
216
|
+
return unwrap(await coll.post(record));
|
|
209
217
|
}
|
|
210
218
|
const merged = { ...existing, content, updatedAt: new Date().toISOString() };
|
|
211
219
|
delete merged.embedding;
|
|
@@ -281,8 +289,7 @@ async function soulGet(agent, args) {
|
|
|
281
289
|
}
|
|
282
290
|
async function workspaceSet(agent, args) {
|
|
283
291
|
const Cls = await handler("WorkspaceState");
|
|
284
|
-
const h =
|
|
285
|
-
h.isCollection = true;
|
|
292
|
+
const h = await collectionResource(Cls, delegationContext(agent));
|
|
286
293
|
// No agentId in the body — WorkspaceState.post attributes the record to the
|
|
287
294
|
// authenticated identity (from the context), never the body. Same no-forge
|
|
288
295
|
// contract as the flair-mcp stdio tool.
|
|
@@ -304,8 +311,7 @@ async function workspaceSet(agent, args) {
|
|
|
304
311
|
}
|
|
305
312
|
async function orgEvent(agent, args) {
|
|
306
313
|
const Cls = await handler("OrgEvent");
|
|
307
|
-
const h =
|
|
308
|
-
h.isCollection = true;
|
|
314
|
+
const h = await collectionResource(Cls, delegationContext(agent));
|
|
309
315
|
// No authorId in the body — OrgEvent.post attributes to the authenticated
|
|
310
316
|
// identity, never the body (no forging as another agent).
|
|
311
317
|
const body = { kind: args?.kind, summary: args?.summary };
|
|
@@ -25,20 +25,43 @@
|
|
|
25
25
|
* `runMigrationCycle` itself never throws (see runner.ts's module doc) —
|
|
26
26
|
* the `.catch()` below is pure defense-in-depth so a bug there can never
|
|
27
27
|
* take down the process either.
|
|
28
|
+
*
|
|
29
|
+
* ─── flair#812: this path must never fail silently ────────────────────────
|
|
30
|
+
* `runMigrationCycle` REPORTS why a cycle didn't run (`{ ran: false, reason
|
|
31
|
+
* }`) — it does not log it, because it is a library. This file, the only
|
|
32
|
+
* caller, previously DISCARDED that value, which is what turned a
|
|
33
|
+
* recoverable environment problem into an invisible one: on a provisioned
|
|
34
|
+
* install where `~/.flair/data` isn't creatable, the runner's very first
|
|
35
|
+
* step (`acquireMigrationLock` → `mkdirSync`) threw `EACCES`, the runner
|
|
36
|
+
* caught it and returned `reason: "lock error: ..."`, and nothing anywhere
|
|
37
|
+
* said a word. `.migrations/state.json` was never written, no
|
|
38
|
+
* `[flair-migrations]` marker ever appeared, and EVERY migration — shipped
|
|
39
|
+
* and future — was skipped on that instance forever.
|
|
40
|
+
*
|
|
41
|
+
* So: the data dir is now resolved to a candidate PROVEN writable before
|
|
42
|
+
* the cycle is handed one (resources/migrations/data-dir.ts), and every
|
|
43
|
+
* non-benign outcome — an unresolvable data dir, tables that never became
|
|
44
|
+
* ready, a runner-reported failure, an unexpected throw — is BOTH logged
|
|
45
|
+
* with a `[flair-migrations]` marker AND recorded as a `failed` progress
|
|
46
|
+
* entry per registered migration. That progress entry is what makes the
|
|
47
|
+
* condition visible where an operator will actually meet it:
|
|
48
|
+
* `/HealthDetail` (with a warning), `flair doctor`'s Migrations section
|
|
49
|
+
* (counted as an issue) and `flair quality`'s `instance.migrationsClean`.
|
|
50
|
+
*
|
|
51
|
+
* The one deliberately-quiet outcome is `single-flight` — another thread or
|
|
52
|
+
* process holds the lock and is running the cycle. That is the guard doing
|
|
53
|
+
* its job, not a failure, and Harper boots N worker threads that each load
|
|
54
|
+
* this module; logging it would emit N-1 scary lines on every healthy boot.
|
|
28
55
|
*/
|
|
29
56
|
import { databases } from "harper";
|
|
30
|
-
import { homedir } from "node:os";
|
|
31
57
|
import { existsSync, readFileSync } from "node:fs";
|
|
32
58
|
import { join, dirname } from "node:path";
|
|
33
59
|
import { fileURLToPath } from "node:url";
|
|
34
60
|
import { buildRegistry } from "./migrations/registry.js";
|
|
35
61
|
import { runMigrationCycle } from "./migrations/runner.js";
|
|
36
|
-
import { seedIdleProgress } from "./migrations/progress.js";
|
|
62
|
+
import { markIdleMigrationsFailed, seedIdleProgress, setCyclePhase } from "./migrations/progress.js";
|
|
63
|
+
import { describeUnresolvableDataDir, resolveWritableMigrationDataDir, } from "./migrations/data-dir.js";
|
|
37
64
|
import { getMode } from "./embeddings-provider.js";
|
|
38
|
-
/** Same dataDir resolution as resources/health.ts's disk section. */
|
|
39
|
-
export function resolveMigrationDataDir() {
|
|
40
|
-
return process.env.HDB_ROOT ?? join(homedir(), ".flair", "data");
|
|
41
|
-
}
|
|
42
65
|
/** Same "resolve the running package's own version" idiom as resources/health.ts. */
|
|
43
66
|
export function resolveRunningVersion() {
|
|
44
67
|
try {
|
|
@@ -103,6 +126,26 @@ async function waitForEmbeddingsSettled(maxWaitMs = 8_500, intervalMs = 150) {
|
|
|
103
126
|
await new Promise((r) => setTimeout(r, intervalMs));
|
|
104
127
|
}
|
|
105
128
|
}
|
|
129
|
+
/**
|
|
130
|
+
* Records a boot-path failure everywhere an operator might look: the
|
|
131
|
+
* process log (with the `[flair-migrations]` marker the runbook greps for),
|
|
132
|
+
* the cycle status (`lastCycleError`, surfaced by `/HealthDetail` and named
|
|
133
|
+
* by `flair doctor`), and a `failed` entry for each migration that never
|
|
134
|
+
* started — because a boot path that cannot run is not a per-migration
|
|
135
|
+
* problem, it is an instance-wide one, and `flair doctor` / `flair quality`
|
|
136
|
+
* read per-migration state.
|
|
137
|
+
*
|
|
138
|
+
* `failed` (not `halted`) is deliberate for these: nothing was attempted
|
|
139
|
+
* against the corpus, so there is no halted work to resume. And the marking
|
|
140
|
+
* deliberately touches only migrations still `idle` — see
|
|
141
|
+
* `markIdleMigrationsFailed` for why overwriting a terminal state would be
|
|
142
|
+
* a downgrade rather than extra information.
|
|
143
|
+
*/
|
|
144
|
+
function reportBootFailure(registry, reason) {
|
|
145
|
+
console.error(`[flair-migrations] ${reason}`);
|
|
146
|
+
setCyclePhase("done", reason);
|
|
147
|
+
markIdleMigrationsFailed(registry.list().map((m) => m.id), reason);
|
|
148
|
+
}
|
|
106
149
|
let scheduled = false;
|
|
107
150
|
export function scheduleMigrationBoot() {
|
|
108
151
|
if (scheduled)
|
|
@@ -110,30 +153,57 @@ export function scheduleMigrationBoot() {
|
|
|
110
153
|
scheduled = true;
|
|
111
154
|
const registry = buildRegistry();
|
|
112
155
|
seedIdleProgress(registry.list().map((m) => m.id));
|
|
156
|
+
// Proof-of-life, set SYNCHRONOUSLY at module load: from here on,
|
|
157
|
+
// `cyclePhase === "idle"` means this module never loaded at all — the
|
|
158
|
+
// one hypothesis flair#812 could not otherwise rule out from the outside.
|
|
159
|
+
setCyclePhase("scheduled");
|
|
113
160
|
setImmediate(() => {
|
|
114
161
|
void (async () => {
|
|
115
162
|
const ready = await waitForTablesReady();
|
|
116
163
|
if (!ready) {
|
|
117
|
-
|
|
164
|
+
reportBootFailure(registry, "Memory/Relationship tables never became ready within 30s — this boot's migration cycle was skipped (it retries on the next restart)");
|
|
118
165
|
return;
|
|
119
166
|
}
|
|
120
167
|
await waitForEmbeddingsSettled();
|
|
168
|
+
// Resolve a data dir PROVEN writable before handing one to the runner
|
|
169
|
+
// — the runner's first act is to take a file lock there, and a
|
|
170
|
+
// failure at that point is reported back rather than thrown, which is
|
|
171
|
+
// precisely how flair#812 stayed invisible. See data-dir.ts.
|
|
172
|
+
const resolved = resolveWritableMigrationDataDir();
|
|
173
|
+
if (!resolved.dataDir) {
|
|
174
|
+
reportBootFailure(registry, describeUnresolvableDataDir(resolved.tried));
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
121
177
|
try {
|
|
122
|
-
await runMigrationCycle({
|
|
178
|
+
const result = await runMigrationCycle({
|
|
123
179
|
registry,
|
|
124
180
|
getTable,
|
|
125
|
-
dataDir:
|
|
181
|
+
dataDir: resolved.dataDir,
|
|
126
182
|
runningVersion: resolveRunningVersion(),
|
|
127
183
|
});
|
|
184
|
+
// `nothing pending` is the healthy no-op; `single-flight` is the
|
|
185
|
+
// lock guard working as designed on a multi-threaded boot. Anything
|
|
186
|
+
// else is a cycle that WANTED to run and couldn't, and must be loud.
|
|
187
|
+
if (!result.ran && result.reason && !isBenignSkip(result.reason)) {
|
|
188
|
+
reportBootFailure(registry, `migration cycle did not run: ${result.reason}`);
|
|
189
|
+
}
|
|
128
190
|
}
|
|
129
191
|
catch (err) {
|
|
130
192
|
// Defense-in-depth only — runMigrationCycle is documented to never
|
|
131
193
|
// throw. A boot-path exception must never surface here regardless.
|
|
132
|
-
|
|
194
|
+
reportBootFailure(registry, `unexpected error from runMigrationCycle: ${err?.message ?? String(err)}`);
|
|
133
195
|
}
|
|
134
196
|
})();
|
|
135
197
|
});
|
|
136
198
|
}
|
|
199
|
+
/**
|
|
200
|
+
* The two `{ ran: false }` reasons that are NOT failures: nothing was
|
|
201
|
+
* pending, or another holder is already running the cycle. Matched on the
|
|
202
|
+
* prefixes runner.ts constructs (`"nothing pending"`, `"single-flight: …"`).
|
|
203
|
+
*/
|
|
204
|
+
export function isBenignSkip(reason) {
|
|
205
|
+
return reason === "nothing pending" || reason.startsWith("single-flight:");
|
|
206
|
+
}
|
|
137
207
|
// Test-only reset so a unit/integration test can re-trigger scheduling
|
|
138
208
|
// within the same process (never used in production — a real process only
|
|
139
209
|
// ever boots once).
|