@tpsdev-ai/flair 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +194 -377
- package/dist/cli.js +1355 -281
- package/dist/deploy.js +212 -24
- package/dist/fabric-upgrade.js +16 -1
- package/dist/federation/scheduler.js +500 -0
- package/dist/install/clients.js +111 -53
- package/dist/lib/mcp-spec.js +128 -0
- package/dist/lib/safe-snapshot-extract.js +231 -0
- package/dist/lib/scheduler-platform.js +128 -0
- package/dist/lib/xml-escape.js +54 -0
- package/dist/rem/scheduler.js +35 -87
- package/dist/rem/snapshot.js +13 -0
- package/dist/replication-convergence.js +505 -0
- package/dist/resources/MemoryBootstrap.js +7 -8
- package/dist/resources/SemanticSearch.js +17 -45
- package/dist/resources/abstention.js +1 -1
- package/dist/resources/embeddings-boot.js +10 -12
- package/dist/resources/embeddings-provider.js +10 -7
- package/dist/resources/health.js +24 -19
- package/dist/resources/in-process.js +225 -0
- package/dist/resources/mcp-tools.js +23 -17
- package/dist/resources/migration-boot.js +80 -10
- package/dist/resources/migrations/data-dir.js +205 -0
- package/dist/resources/migrations/progress.js +33 -0
- package/dist/resources/migrations/runner.js +29 -2
- package/dist/resources/migrations/state.js +13 -2
- package/dist/resources/models-dir.js +18 -9
- package/dist/resources/semantic-retrieval-core.js +5 -4
- package/dist/src/lib/scheduler-platform.js +128 -0
- package/dist/src/lib/xml-escape.js +54 -0
- package/dist/src/rem/scheduler.js +35 -87
- package/docs/deploying-on-fabric.md +267 -0
- package/docs/deployment.md +5 -0
- package/docs/embedding-in-a-harper-app.md +299 -0
- package/docs/federation.md +61 -4
- package/docs/integrations.md +3 -0
- package/docs/mcp-clients.md +16 -7
- package/docs/quickstart.md +80 -54
- package/docs/releasing.md +72 -38
- package/docs/supply-chain-policy.md +36 -0
- package/docs/troubleshooting.md +24 -0
- package/docs/upgrade.md +98 -3
- package/package.json +1 -11
- package/templates/bin/flair-federation-sync.sh.tmpl +28 -0
- package/templates/launchd/dev.flair.federation.sync.plist.tmpl +47 -0
- package/templates/systemd/flair-federation-sync.service.tmpl +21 -0
- package/templates/systemd/flair-federation-sync.timer.tmpl +16 -0
- package/dist/resources/rerank-provider.js +0 -569
- package/docs/rerank-provisioning.md +0 -101
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* data-dir.ts — resolves the directory the migration subsystem owns
|
|
3
|
+
* (`<dataDir>/.migrations/{state.json,lock,snapshots,exports}`), and proves
|
|
4
|
+
* it is actually usable BEFORE the boot cycle commits to it.
|
|
5
|
+
*
|
|
6
|
+
* ─── Why this file exists (flair#812) ──────────────────────────────────────
|
|
7
|
+
* The previous resolution was a single expression, duplicated in
|
|
8
|
+
* `resources/migration-boot.ts` and `resources/health.ts`:
|
|
9
|
+
*
|
|
10
|
+
* process.env.HDB_ROOT ?? join(homedir(), ".flair", "data")
|
|
11
|
+
*
|
|
12
|
+
* Both halves of that were wrong in a way that only shows up off the
|
|
13
|
+
* developer/default shape:
|
|
14
|
+
*
|
|
15
|
+
* 1. `HDB_ROOT` IS NEVER SET BY ANYTHING. It is not a Harper process env
|
|
16
|
+
* var — Harper's own root-path env var is `ROOTPATH`
|
|
17
|
+
* (`harper`'s `utility/common_utils.ts` reads
|
|
18
|
+
* `process.env['ROOTPATH']`; `HDB_ROOT` appears only as a legacy *config
|
|
19
|
+
* key* alias in `utility/hdbTerms.ts`, and Harper never reads or writes
|
|
20
|
+
* it on `process.env`). flair's own spawner (`src/cli.ts`) exports
|
|
21
|
+
* `ROOTPATH`, never `HDB_ROOT`, and `resources/models-dir.ts` already
|
|
22
|
+
* reads `ROOTPATH` for exactly this reason. So the `??` left branch was
|
|
23
|
+
* dead code and the effective resolution was UNCONDITIONALLY
|
|
24
|
+
* `~/.flair/data`, regardless of where Harper's real root actually is.
|
|
25
|
+
*
|
|
26
|
+
* 2. `~/.flair/data` is right only by coincidence — it is `flair init`'s
|
|
27
|
+
* default data dir, so on a default local install the two happen to
|
|
28
|
+
* coincide. On a PROVISIONED install (a service-managed spoke, a
|
|
29
|
+
* container, a Harper Fabric component deployment) the process's
|
|
30
|
+
* `homedir()` is whatever the service account has: possibly read-only,
|
|
31
|
+
* possibly nonexistent, and in general nothing to do with the instance's
|
|
32
|
+
* data.
|
|
33
|
+
*
|
|
34
|
+
* When `~/.flair/data` is not creatable, EVERY consumer of the old
|
|
35
|
+
* expression failed — and, critically, failed SILENTLY: the very first
|
|
36
|
+
* thing `runMigrationCycle` does is `acquireMigrationLock`, whose
|
|
37
|
+
* `mkdirSync(dirname(lockPath), { recursive: true, mode: 0o700 })` throws
|
|
38
|
+
* `EACCES`/`ENOENT`, which the runner catches and turns into
|
|
39
|
+
* `{ ran: false, reason: "lock error: ..." }` — a value `migration-boot.ts`
|
|
40
|
+
* then discarded. No log line, no state file, no health signal, and every
|
|
41
|
+
* migration ever shipped skipped forever on that instance. (Contrast
|
|
42
|
+
* `resources/embeddings-boot.ts`, loaded by the SAME `jsResource` glob:
|
|
43
|
+
* it touches no filesystem path of its own, which is exactly why embeddings
|
|
44
|
+
* kept working on the affected shapes while migrations did not.)
|
|
45
|
+
*
|
|
46
|
+
* ─── The resolution ─────────────────────────────────────────────────────
|
|
47
|
+
* An ordered candidate list, first USABLE one wins:
|
|
48
|
+
*
|
|
49
|
+
* 1. `FLAIR_MIGRATION_DATA_DIR` — explicit operator override. The escape
|
|
50
|
+
* hatch for any deployment shape whose data dir this module cannot
|
|
51
|
+
* infer; naming it in the failure message below is what makes an
|
|
52
|
+
* unresolvable instance actionable rather than merely reported.
|
|
53
|
+
* 2. `HDB_ROOT` — retained purely for compatibility with anything that
|
|
54
|
+
* may have started setting it because flair once read it. Never set by
|
|
55
|
+
* Harper or by flair.
|
|
56
|
+
* 3. `homedir()/.flair/data` — the historical default, and `flair init`'s
|
|
57
|
+
* own data dir. DELIBERATELY still ahead of `ROOTPATH`: on every
|
|
58
|
+
* currently-working install this is where the state file already
|
|
59
|
+
* lives, and reordering would silently relocate it (costing a
|
|
60
|
+
* re-detect pass and orphaning the existing audit record) on instances
|
|
61
|
+
* that have no problem at all. This fix is about instances where the
|
|
62
|
+
* cycle cannot run, not about relocating ones where it can.
|
|
63
|
+
* 4. `ROOTPATH` — Harper's real root path. The rescue candidate: it is
|
|
64
|
+
* writable by definition on a running instance (Harper is writing its
|
|
65
|
+
* own databases there), so a shape whose `homedir()` is unusable still
|
|
66
|
+
* gets a stable, per-instance location instead of nothing.
|
|
67
|
+
* 5. The Harper root INFERRED from this module's own location, but only
|
|
68
|
+
* when the running layout is unambiguously a deployed component (see
|
|
69
|
+
* `deployedComponentRootPath`). `ROOTPATH` is an env var, and a
|
|
70
|
+
* deployment that configures `rootPath` in Harper's config file
|
|
71
|
+
* instead of the environment leaves candidate 4 empty — this covers
|
|
72
|
+
* that case without guessing.
|
|
73
|
+
*
|
|
74
|
+
* "Usable" is probed by DOING THE REAL OPERATION the runner would do —
|
|
75
|
+
* create `<dir>/.migrations` at 0700 and check it is writable — not by a
|
|
76
|
+
* proxy check that could disagree with it. The probe is idempotent and, on
|
|
77
|
+
* a healthy instance, is satisfied by the first candidate without touching
|
|
78
|
+
* the others.
|
|
79
|
+
*
|
|
80
|
+
* If NO candidate is usable, `resolveWritableMigrationDataDir` returns
|
|
81
|
+
* `dataDir: null` WITH the per-candidate reasons, and the boot path turns
|
|
82
|
+
* that into a loud console error plus a `failed` progress entry per
|
|
83
|
+
* registered migration — visible in `/HealthDetail`, `flair doctor` and
|
|
84
|
+
* `flair quality`'s `instance.migrationsClean`. An instance that cannot run
|
|
85
|
+
* migrations now says so; that silence was the actual defect.
|
|
86
|
+
*/
|
|
87
|
+
import { accessSync, constants, existsSync, mkdirSync } from "node:fs";
|
|
88
|
+
import { homedir } from "node:os";
|
|
89
|
+
import { basename, dirname, join } from "node:path";
|
|
90
|
+
import { fileURLToPath } from "node:url";
|
|
91
|
+
/** Explicit operator override for the migration data dir (see module doc). */
|
|
92
|
+
export const MIGRATION_DATA_DIR_ENV = "FLAIR_MIGRATION_DATA_DIR";
|
|
93
|
+
/** The subdirectory the migration subsystem owns inside whichever dataDir wins. */
|
|
94
|
+
export const MIGRATIONS_SUBDIR = ".migrations";
|
|
95
|
+
/**
|
|
96
|
+
* Harper's root path, inferred from where THIS module is running, but ONLY
|
|
97
|
+
* when the layout is unambiguously a deployed component:
|
|
98
|
+
*
|
|
99
|
+
* <rootPath>/components/<name>/dist/resources/migrations/data-dir.js
|
|
100
|
+
*
|
|
101
|
+
* i.e. the directory two levels above `dist/` must itself be named
|
|
102
|
+
* `components` — Harper's `componentsRoot` convention, and the layout
|
|
103
|
+
* `deploy_component` produces. That check is what keeps this from firing on
|
|
104
|
+
* a source checkout or an npm install, where the same arithmetic would
|
|
105
|
+
* point at an arbitrary parent directory. Returns null whenever the layout
|
|
106
|
+
* doesn't match, so a non-match contributes no candidate rather than a
|
|
107
|
+
* guess.
|
|
108
|
+
*
|
|
109
|
+
* `moduleUrl` is a parameter purely so a unit test can exercise both the
|
|
110
|
+
* matching and non-matching layouts without relocating the built file.
|
|
111
|
+
*/
|
|
112
|
+
export function deployedComponentRootPath(moduleUrl = import.meta.url) {
|
|
113
|
+
try {
|
|
114
|
+
const here = dirname(fileURLToPath(moduleUrl)); // <component>/dist/resources/migrations
|
|
115
|
+
const componentDir = dirname(dirname(dirname(here))); // <component>
|
|
116
|
+
const componentsRoot = dirname(componentDir); // .../components
|
|
117
|
+
if (basename(componentsRoot) !== "components")
|
|
118
|
+
return null;
|
|
119
|
+
return dirname(componentsRoot);
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Ordered candidates for the migration data dir. See the module doc for why
|
|
127
|
+
* this order is what it is — in particular, why `homedir()/.flair/data`
|
|
128
|
+
* stays ahead of `ROOTPATH`.
|
|
129
|
+
*/
|
|
130
|
+
export function migrationDataDirCandidates(env = process.env, moduleUrl = import.meta.url) {
|
|
131
|
+
const out = [];
|
|
132
|
+
const add = (p) => {
|
|
133
|
+
const trimmed = p?.trim();
|
|
134
|
+
if (trimmed && !out.includes(trimmed))
|
|
135
|
+
out.push(trimmed);
|
|
136
|
+
};
|
|
137
|
+
add(env[MIGRATION_DATA_DIR_ENV]);
|
|
138
|
+
add(env.HDB_ROOT);
|
|
139
|
+
add(join(homedir(), ".flair", "data"));
|
|
140
|
+
add(env.ROOTPATH);
|
|
141
|
+
add(deployedComponentRootPath(moduleUrl));
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Probes a candidate by performing the exact operation the runner's lock
|
|
146
|
+
* acquisition performs (`mkdir -p <dir>/.migrations` at 0700), then
|
|
147
|
+
* confirming the result is writable. Idempotent: on an already-working
|
|
148
|
+
* instance this is a no-op stat/mkdir against a directory that already
|
|
149
|
+
* exists.
|
|
150
|
+
*/
|
|
151
|
+
export function probeMigrationDataDir(dir) {
|
|
152
|
+
const owned = join(dir, MIGRATIONS_SUBDIR);
|
|
153
|
+
try {
|
|
154
|
+
mkdirSync(owned, { recursive: true, mode: 0o700 });
|
|
155
|
+
accessSync(owned, constants.W_OK | constants.X_OK);
|
|
156
|
+
return { dir, ok: true };
|
|
157
|
+
}
|
|
158
|
+
catch (err) {
|
|
159
|
+
return { dir, ok: false, reason: err?.message ?? String(err) };
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Resolves a migration data dir that is proven writable, or reports why not.
|
|
164
|
+
* Used by the boot path (`resources/migration-boot.ts`), which must not
|
|
165
|
+
* commit to a directory the runner will then fail to lock.
|
|
166
|
+
*/
|
|
167
|
+
export function resolveWritableMigrationDataDir(env = process.env) {
|
|
168
|
+
const tried = [];
|
|
169
|
+
for (const candidate of migrationDataDirCandidates(env)) {
|
|
170
|
+
const probe = probeMigrationDataDir(candidate);
|
|
171
|
+
tried.push(probe);
|
|
172
|
+
if (probe.ok)
|
|
173
|
+
return { dataDir: candidate, tried };
|
|
174
|
+
}
|
|
175
|
+
return { dataDir: null, tried };
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Operator-facing explanation of a total resolution failure — carries the
|
|
179
|
+
* actor (each path tried), the state (why each was rejected) and the remedy
|
|
180
|
+
* (the override env var), so the message is actionable at 3am rather than
|
|
181
|
+
* merely accurate.
|
|
182
|
+
*/
|
|
183
|
+
export function describeUnresolvableDataDir(tried) {
|
|
184
|
+
const detail = tried.length
|
|
185
|
+
? tried.map((t) => `${t.dir} (${t.reason ?? "unusable"})`).join("; ")
|
|
186
|
+
: "no candidate paths at all";
|
|
187
|
+
return (`no writable migration data directory — tried ${detail}. ` +
|
|
188
|
+
`Migrations cannot run on this instance until one exists: point ${MIGRATION_DATA_DIR_ENV} ` +
|
|
189
|
+
`at a writable directory (or make one of the paths above writable by the account this ` +
|
|
190
|
+
`instance runs as) and restart.`);
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* READ-ONLY counterpart for `/HealthDetail`, which must never create
|
|
194
|
+
* directories as a side effect of a GET. Prefers whichever candidate
|
|
195
|
+
* already carries a `.migrations` directory (i.e. the one a boot actually
|
|
196
|
+
* chose), falling back to the first candidate so the field is never empty.
|
|
197
|
+
*/
|
|
198
|
+
export function resolveMigrationDataDirForRead(env = process.env) {
|
|
199
|
+
const candidates = migrationDataDirCandidates(env);
|
|
200
|
+
for (const candidate of candidates) {
|
|
201
|
+
if (existsSync(join(candidate, MIGRATIONS_SUBDIR)))
|
|
202
|
+
return candidate;
|
|
203
|
+
}
|
|
204
|
+
return candidates[0] ?? join(homedir(), ".flair", "data");
|
|
205
|
+
}
|
|
@@ -25,6 +25,39 @@ export function seedIdleProgress(ids) {
|
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* Marks every migration that never got off the starting line as `failed`
|
|
30
|
+
* with `reason` — and ONLY those (flair#812).
|
|
31
|
+
*
|
|
32
|
+
* The boot path calls this when a cycle couldn't run, so that a condition
|
|
33
|
+
* which is really instance-wide (no writable data dir, tables never ready,
|
|
34
|
+
* the lock itself unobtainable) shows up per-migration, which is the shape
|
|
35
|
+
* `flair doctor` and `flair quality`'s `instance.migrationsClean` actually
|
|
36
|
+
* read.
|
|
37
|
+
*
|
|
38
|
+
* Restricting it to `idle` entries is load-bearing, not tidiness: a cycle
|
|
39
|
+
* can fail AFTER some migrations already reached a terminal state — the
|
|
40
|
+
* pre-hash path, for instance, halts each candidate with its own precise
|
|
41
|
+
* reason and THEN reports cycle failure. Overwriting those would replace a
|
|
42
|
+
* specific `halted` (which the runner retries on the next boot, per its
|
|
43
|
+
* documented contract) with a blanket `failed`, and would flip migrations
|
|
44
|
+
* that genuinely completed this cycle to failed. Only an entry still `idle`
|
|
45
|
+
* is one nothing has said anything about yet.
|
|
46
|
+
*
|
|
47
|
+
* Returns the ids actually marked, so a caller can tell "nothing had
|
|
48
|
+
* started" from "the cycle died partway".
|
|
49
|
+
*/
|
|
50
|
+
export function markIdleMigrationsFailed(ids, reason) {
|
|
51
|
+
const marked = [];
|
|
52
|
+
for (const id of ids) {
|
|
53
|
+
const current = progressById.get(id);
|
|
54
|
+
if (current && current.state !== "idle")
|
|
55
|
+
continue;
|
|
56
|
+
progressById.set(id, { id, rowsDone: 0, rowsRemaining: 0, state: "failed", reason });
|
|
57
|
+
marked.push(id);
|
|
58
|
+
}
|
|
59
|
+
return marked;
|
|
60
|
+
}
|
|
28
61
|
export function _resetProgressForTests() {
|
|
29
62
|
progressById.clear();
|
|
30
63
|
cycleStatus = { phase: "idle" };
|
|
@@ -151,7 +151,21 @@ async function runCycleLocked(deps, lock) {
|
|
|
151
151
|
const candidates = [];
|
|
152
152
|
for (const migration of deps.registry.list()) {
|
|
153
153
|
if (isShortCircuited(state, migration.id, deps.runningVersion)) {
|
|
154
|
-
|
|
154
|
+
// flair#812: a short-circuit is an ASSERTION READ OFF A FILE, not an
|
|
155
|
+
// observation of the corpus — and that file is hand-editable (the
|
|
156
|
+
// documented remediation for a stuck migration is to correct it by
|
|
157
|
+
// hand). Reporting it as plain "completed" made a hand-written entry
|
|
158
|
+
// indistinguishable from a migration this boot actually verified.
|
|
159
|
+
// The skip itself is unchanged (it is the documented cheap path); it
|
|
160
|
+
// now just says what it is, so `flair doctor` never presents an
|
|
161
|
+
// unverified claim as a verified one.
|
|
162
|
+
setMigrationProgress({
|
|
163
|
+
id: migration.id,
|
|
164
|
+
rowsDone: 0,
|
|
165
|
+
rowsRemaining: 0,
|
|
166
|
+
state: "completed",
|
|
167
|
+
reason: `recorded complete at ${deps.runningVersion} in ${deps.statePath} — not re-verified this boot`,
|
|
168
|
+
});
|
|
155
169
|
continue;
|
|
156
170
|
}
|
|
157
171
|
setMigrationProgress({ id: migration.id, rowsDone: 0, rowsRemaining: 0, state: "checking" });
|
|
@@ -160,12 +174,18 @@ async function runCycleLocked(deps, lock) {
|
|
|
160
174
|
pending = await migration.detect();
|
|
161
175
|
}
|
|
162
176
|
catch (err) {
|
|
177
|
+
// flair#812: a throwing detect() removes this migration from the
|
|
178
|
+
// candidate set, so a cycle where EVERY detect() throws used to reach
|
|
179
|
+
// the `nothing pending` return and look like a clean no-op. Log it
|
|
180
|
+
// with the standard marker as well as recording it in progress.
|
|
181
|
+
const reason = `detect() threw: ${err?.message ?? String(err)}`;
|
|
182
|
+
console.error(`[flair-migrations] ${migration.id}: ${reason}`);
|
|
163
183
|
setMigrationProgress({
|
|
164
184
|
id: migration.id,
|
|
165
185
|
rowsDone: 0,
|
|
166
186
|
rowsRemaining: 0,
|
|
167
187
|
state: "failed",
|
|
168
|
-
reason
|
|
188
|
+
reason,
|
|
169
189
|
});
|
|
170
190
|
continue;
|
|
171
191
|
}
|
|
@@ -176,6 +196,13 @@ async function runCycleLocked(deps, lock) {
|
|
|
176
196
|
candidates.push(migration);
|
|
177
197
|
}
|
|
178
198
|
if (candidates.length === 0) {
|
|
199
|
+
// flair#812: mark the cycle done even on the no-op path. "Nothing was
|
|
200
|
+
// pending" and "the cycle never ran" are opposite conclusions that used
|
|
201
|
+
// to produce an IDENTICAL `/HealthDetail` reading (`cyclePhase: "idle",
|
|
202
|
+
// lastCycleAt: null`), which is exactly why a totally-skipped boot cycle
|
|
203
|
+
// looked healthy for months. A cycle that reached this line has done its
|
|
204
|
+
// job, and now says so.
|
|
205
|
+
setCyclePhase("done");
|
|
179
206
|
return { ran: false, reason: "nothing pending" };
|
|
180
207
|
}
|
|
181
208
|
// ── ONE shared async pre-hash at cycle start (K&S: computed after ready,
|
|
@@ -30,11 +30,22 @@ export function readMigrationState(path) {
|
|
|
30
30
|
if (!existsSync(path))
|
|
31
31
|
return {};
|
|
32
32
|
const raw = JSON.parse(readFileSync(path, "utf-8"));
|
|
33
|
-
|
|
33
|
+
if (raw && typeof raw === "object")
|
|
34
|
+
return raw;
|
|
35
|
+
console.warn(`[flair-migrations] ${path} is not a JSON object — ignoring it and re-detecting every migration this boot`);
|
|
36
|
+
return {};
|
|
34
37
|
}
|
|
35
|
-
catch {
|
|
38
|
+
catch (err) {
|
|
36
39
|
// Corrupt/unreadable state — treat as "nothing known yet", never throw.
|
|
37
40
|
// Worst case this costs one extra (cheap, bounded) detect() call.
|
|
41
|
+
//
|
|
42
|
+
// flair#812: but SAY SO. This file is hand-edited in practice (the
|
|
43
|
+
// documented remediation for a stuck migration), and a fat-fingered
|
|
44
|
+
// edit silently discarding every recorded outcome is precisely the
|
|
45
|
+
// class of quiet failure this issue is about. The behaviour is
|
|
46
|
+
// unchanged and still safe — only the silence is fixed.
|
|
47
|
+
console.warn(`[flair-migrations] could not read ${path} (${err?.message ?? String(err)}) — ` +
|
|
48
|
+
`treating migration state as empty and re-detecting every migration this boot`);
|
|
38
49
|
return {};
|
|
39
50
|
}
|
|
40
51
|
}
|
|
@@ -1,25 +1,34 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* models-dir.ts — the ONE place that decides where model GGUFs live.
|
|
3
3
|
*
|
|
4
|
-
* Extracted from embeddings-provider.ts (flair#815) so
|
|
5
|
-
*
|
|
4
|
+
* Extracted from embeddings-provider.ts (flair#815) so a second model
|
|
5
|
+
* consumer could share the exact same resolution WITHOUT importing
|
|
6
6
|
* embeddings-provider: several unit-isolated tests `mock.module()` the whole
|
|
7
7
|
* embeddings-provider module (with only the named exports THEY consume), and
|
|
8
8
|
* bun's module cache is process-wide — so any new cross-module import of
|
|
9
9
|
* embeddings-provider from another resource makes those mocks incomplete and
|
|
10
10
|
* kills unrelated test files at module load. This module is tiny, pure
|
|
11
|
-
* node-stdlib, and never mocked, so
|
|
12
|
-
*
|
|
11
|
+
* node-stdlib, and never mocked, so any model consumer can depend on it
|
|
12
|
+
* safely.
|
|
13
|
+
*
|
|
14
|
+
* The second consumer that motivated the split (the cross-encoder reranker)
|
|
15
|
+
* has since been removed (flair#893), leaving the embedding engine as the
|
|
16
|
+
* only caller. The module stays split anyway, and deliberately: the
|
|
17
|
+
* `mock.module()` hazard above is a property of embeddings-provider, not of
|
|
18
|
+
* the reranker, so folding this back in would re-arm it for whatever the next
|
|
19
|
+
* model consumer turns out to be. It is also the single documented source of
|
|
20
|
+
* truth for the models dir, independently tested as such.
|
|
13
21
|
*/
|
|
14
22
|
import { existsSync } from "node:fs";
|
|
15
23
|
import { homedir } from "node:os";
|
|
16
24
|
import { join } from "node:path";
|
|
17
25
|
/**
|
|
18
|
-
* Resolve the directory model GGUFs live in / download into —
|
|
19
|
-
* embedding engine (embeddings-boot.ts's `register()` options) and the
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* models location
|
|
26
|
+
* Resolve the directory model GGUFs live in / download into — used by the
|
|
27
|
+
* embedding engine (embeddings-boot.ts's `register()` options), and the
|
|
28
|
+
* resolution any future model consumer should use rather than re-deriving its
|
|
29
|
+
* own. flair#815 exists because a consumer that hardcoded `<cwd>/models`
|
|
30
|
+
* silently failed open wherever Harper's cwd wasn't the models location; the
|
|
31
|
+
* fix was to give the decision exactly one home, which is this function.
|
|
23
32
|
*
|
|
24
33
|
* Resolution order (everything writable, never the read-only package dir):
|
|
25
34
|
* 1. FLAIR_MODELS_DIR — explicit operator/docker override.
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
// Extracted from resources/SemanticSearch.ts's post() (Kern-approved
|
|
4
4
|
// refactor, flair#695). Before this module existed,
|
|
5
5
|
// SemanticSearch.post() was one function entangling auth resolution,
|
|
6
|
-
// rate-limiting, HNSW/BM25 retrieval, post-retrieval filtering,
|
|
7
|
-
//
|
|
6
|
+
// rate-limiting, HNSW/BM25 retrieval, post-retrieval filtering, AND
|
|
7
|
+
// retrievalCount/lastRetrieved hit-tracking side
|
|
8
8
|
// effects — so the ONLY way for MemoryBootstrap (resources/MemoryBootstrap.ts)
|
|
9
9
|
// to get bounded, HNSW-pushed-down candidates was to duplicate the retrieval
|
|
10
10
|
// logic or trip the side effects (an internal bootstrap call spuriously
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
// the HNSW leg query construction (sort/select/conditions/limit), the BM25 +
|
|
17
17
|
// union-RRF hybrid fusion, the per-record temporal/expiry/supersede filters,
|
|
18
18
|
// and the scope.isAllowed() defense-in-depth re-check. It does NOT own: auth
|
|
19
|
-
// resolution, rate-limiting,
|
|
19
|
+
// resolution, rate-limiting, or the retrievalCount/lastRetrieved
|
|
20
20
|
// hit-tracking side effects — those stay in SemanticSearch.post()'s wrapper
|
|
21
21
|
// (resources/SemanticSearch.ts) so an internal caller (bootstrap) never trips
|
|
22
22
|
// them.
|
|
@@ -32,7 +32,8 @@
|
|
|
32
32
|
// Returns results AFTER all filters, sorted best-first by `_score` — bounded
|
|
33
33
|
// ONLY by the `limit` the caller chose to push down (the core never
|
|
34
34
|
// multiplies `limit` internally; any overfetch policy — SemanticSearch's
|
|
35
|
-
// CANDIDATE_MULTIPLIER,
|
|
35
|
+
// CANDIDATE_MULTIPLIER, MemoryBootstrap's K formula — is the CALLER's
|
|
36
|
+
// decision, made
|
|
36
37
|
// before calling in). Never exposes which internal leg (BM25+RRF hybrid vs.
|
|
37
38
|
// legacy HNSW-only vs. keyword-only fallback) produced a given result — the
|
|
38
39
|
// output shape is identical regardless of `hybrid`.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Platform-scheduler primitives shared by every Flair-managed background job.
|
|
3
|
+
*
|
|
4
|
+
* Two modules install user-session scheduler entries today:
|
|
5
|
+
* - src/rem/scheduler.ts (`flair rem nightly enable`)
|
|
6
|
+
* - src/federation/scheduler.ts (`flair federation sync enable`)
|
|
7
|
+
*
|
|
8
|
+
* They differ in what they schedule and how often; they do NOT differ in how
|
|
9
|
+
* you ask launchd/systemd whether a job is really loaded, how you read the
|
|
10
|
+
* answer, or how you render a template into a unit file. This module exists so
|
|
11
|
+
* there is exactly ONE implementation of those — same reason src/lib/xml-escape.ts
|
|
12
|
+
* exists (#918): the second copy of a subtle rule is where it goes wrong.
|
|
13
|
+
*
|
|
14
|
+
* `interpretActiveResult()` in particular encodes a production lesson
|
|
15
|
+
* (flair#850) that took a real outage to learn. It must not be re-derived.
|
|
16
|
+
*/
|
|
17
|
+
import { existsSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
|
|
18
|
+
import { resolve, dirname } from "node:path";
|
|
19
|
+
import { platform } from "node:os";
|
|
20
|
+
import { spawnSync } from "node:child_process";
|
|
21
|
+
/**
|
|
22
|
+
* 30s ceiling on launchctl/systemctl invocations so a hung service manager
|
|
23
|
+
* can't block the CLI indefinitely. Sherlock #415 follow-up.
|
|
24
|
+
*/
|
|
25
|
+
export const SPAWN_TIMEOUT_MS = 30_000;
|
|
26
|
+
/**
|
|
27
|
+
* Status-check spawns (launchctl print / systemctl is-active) return
|
|
28
|
+
* near-instantly when the service manager is reachable, and fail fast (no
|
|
29
|
+
* hang) when it isn't (e.g. "Failed to connect to bus"). A short ceiling
|
|
30
|
+
* keeps status commands and the Health endpoint responsive even when the
|
|
31
|
+
* query is inconclusive.
|
|
32
|
+
*/
|
|
33
|
+
export const STATUS_CHECK_TIMEOUT_MS = 5_000;
|
|
34
|
+
export function detectPlatform(label, override) {
|
|
35
|
+
if (override)
|
|
36
|
+
return override;
|
|
37
|
+
const p = platform();
|
|
38
|
+
if (p === "darwin")
|
|
39
|
+
return "darwin";
|
|
40
|
+
if (p === "linux")
|
|
41
|
+
return "linux";
|
|
42
|
+
throw new Error(`unsupported platform for ${label}: ${p} (only darwin and linux)`);
|
|
43
|
+
}
|
|
44
|
+
export function spawnReport(cmd, timeoutMs = SPAWN_TIMEOUT_MS) {
|
|
45
|
+
const r = spawnSync(cmd[0], cmd.slice(1), {
|
|
46
|
+
encoding: "buffer",
|
|
47
|
+
timeout: timeoutMs,
|
|
48
|
+
});
|
|
49
|
+
return {
|
|
50
|
+
code: r.status,
|
|
51
|
+
stdout: r.stdout?.toString("utf-8") ?? "",
|
|
52
|
+
stderr: r.stderr?.toString("utf-8") ?? "",
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Interprets the result of a `launchctl print` / `systemctl --user is-active`
|
|
57
|
+
* probe. Shared by the sync (CLI) and async (server) callers of both
|
|
58
|
+
* schedulers.
|
|
59
|
+
*
|
|
60
|
+
* - true — the service manager confirms the job is loaded/active.
|
|
61
|
+
* - false — confirmed NOT active. This includes the "no session bus" case
|
|
62
|
+
* (flair#850): when `systemctl --user` can't reach a bus, it fails
|
|
63
|
+
* before printing a status word ("Failed to connect to bus: No medium
|
|
64
|
+
* found") — but nothing CAN be running without a bus, so `false` is the
|
|
65
|
+
* honest answer, not "unknown".
|
|
66
|
+
* - null — genuinely inconclusive (the command itself couldn't run at
|
|
67
|
+
* all — e.g. the binary is missing — with no output to interpret).
|
|
68
|
+
*/
|
|
69
|
+
export function interpretActiveResult(plat, code, stdout, stderr) {
|
|
70
|
+
const noOutput = !stdout.trim() && !stderr.trim();
|
|
71
|
+
if (plat === "darwin") {
|
|
72
|
+
if (code === 0)
|
|
73
|
+
return true;
|
|
74
|
+
if (code === null && noOutput)
|
|
75
|
+
return null; // spawn itself failed — inconclusive
|
|
76
|
+
return false; // launchctl ran and reported not-loaded
|
|
77
|
+
}
|
|
78
|
+
const out = stdout.trim();
|
|
79
|
+
if (out === "active" || out === "activating")
|
|
80
|
+
return true;
|
|
81
|
+
if (out === "inactive" || out === "failed" || out === "unknown")
|
|
82
|
+
return false;
|
|
83
|
+
if (code === null && noOutput)
|
|
84
|
+
return null; // spawn itself failed — inconclusive
|
|
85
|
+
return false; // covers the no-bus case: empty stdout, nonzero/failed exit
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Human remedy text for a failed scheduler-load attempt (flair#850). Covers
|
|
89
|
+
* the one root cause traced so far: a missing systemd user session bus,
|
|
90
|
+
* which blocks `systemctl --user` entirely in ssh-without-lingering,
|
|
91
|
+
* container, and CI contexts. Returns null when the failure doesn't match a
|
|
92
|
+
* known pattern — the caller already prints the raw stderr, so the operator
|
|
93
|
+
* still has something to go on.
|
|
94
|
+
*
|
|
95
|
+
* `enableCommand` is the caller's own enable invocation, named in the remedy
|
|
96
|
+
* so the operator is told to re-run the command they actually ran.
|
|
97
|
+
*/
|
|
98
|
+
export function describeLoadFailure(plat, loadResult, enableCommand) {
|
|
99
|
+
const stderr = loadResult.stderr || "";
|
|
100
|
+
if (plat === "linux" && /failed to connect to bus/i.test(stderr)) {
|
|
101
|
+
return ("No systemd user session bus is available in this session (common over ssh without lingering, " +
|
|
102
|
+
"in containers, or under CI). Fix: enable lingering for this user — `loginctl enable-linger <user>` " +
|
|
103
|
+
`— then re-run \`${enableCommand}\`.`);
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
/** Single-pass `{{KEY}}` substitution, with a per-value escape hook. */
|
|
108
|
+
export function renderTemplateWith(text, subs, escape) {
|
|
109
|
+
return text.replace(/\{\{([A-Z_]+)\}\}/g, (_, key) => {
|
|
110
|
+
const value = subs[key];
|
|
111
|
+
if (value === undefined)
|
|
112
|
+
throw new Error(`unknown template placeholder: ${key}`);
|
|
113
|
+
return escape(String(value));
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
export function readTemplate(rootDir, relativePath) {
|
|
117
|
+
const full = resolve(rootDir, relativePath);
|
|
118
|
+
if (!existsSync(full)) {
|
|
119
|
+
throw new Error(`template not found: ${full}`);
|
|
120
|
+
}
|
|
121
|
+
return readFileSync(full, "utf-8");
|
|
122
|
+
}
|
|
123
|
+
export function writeFileWithDir(path, contents, mode = 0o600) {
|
|
124
|
+
const dir = dirname(path);
|
|
125
|
+
if (!existsSync(dir))
|
|
126
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
127
|
+
writeFileSync(path, contents, { mode });
|
|
128
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* XML entity escaping for the launchd plists this CLI generates.
|
|
3
|
+
*
|
|
4
|
+
* A launchd plist is XML, so ANY value interpolated into one has to be
|
|
5
|
+
* escaped or the document is malformed — and `launchctl load` rejects a
|
|
6
|
+
* malformed plist outright, so the service silently never registers.
|
|
7
|
+
*
|
|
8
|
+
* Two independent writers generate plists, and both go through here:
|
|
9
|
+
* - buildLaunchdPlist() in src/cli.ts (the Harper service, `flair init`)
|
|
10
|
+
* - renderPlistTemplate() in src/rem/scheduler.ts (`flair rem nightly enable`)
|
|
11
|
+
*
|
|
12
|
+
* This module exists so there is exactly ONE implementation to get right.
|
|
13
|
+
* Anything that interpolates into plist XML imports it rather than
|
|
14
|
+
* hand-rolling a `.replace()` chain at the call site — a local chain is how
|
|
15
|
+
* this class of bug got in (and how it stayed partial: the original one
|
|
16
|
+
* covered three of the five entities).
|
|
17
|
+
*/
|
|
18
|
+
/** The five XML predefined entities, in the order they must be replaced. */
|
|
19
|
+
const XML_ESCAPES = [
|
|
20
|
+
// `&` MUST be first: replacing it after `<` would turn the `&` of an
|
|
21
|
+
// already-emitted `<` into `&lt;` and corrupt the value.
|
|
22
|
+
[/&/g, "&"],
|
|
23
|
+
[/</g, "<"],
|
|
24
|
+
[/>/g, ">"],
|
|
25
|
+
[/"/g, """],
|
|
26
|
+
[/'/g, "'"],
|
|
27
|
+
];
|
|
28
|
+
/**
|
|
29
|
+
* Escape the five XML predefined entities for use inside an XML text node.
|
|
30
|
+
*
|
|
31
|
+
* Safe to apply to any string, including one with no special characters.
|
|
32
|
+
* The result round-trips exactly through unescapeXml().
|
|
33
|
+
*/
|
|
34
|
+
export function escapeXml(s) {
|
|
35
|
+
let out = s;
|
|
36
|
+
for (const [pattern, replacement] of XML_ESCAPES)
|
|
37
|
+
out = out.replace(pattern, replacement);
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Inverse of escapeXml(), for reading a value back out of a document we wrote.
|
|
42
|
+
*
|
|
43
|
+
* `&` MUST be decoded LAST, mirroring escapeXml()'s ordering: decoding it
|
|
44
|
+
* first would turn `&lt;` (the escaping of a literal "<") into `<`
|
|
45
|
+
* instead of back into `<`.
|
|
46
|
+
*/
|
|
47
|
+
export function unescapeXml(s) {
|
|
48
|
+
return s
|
|
49
|
+
.replace(/</g, "<")
|
|
50
|
+
.replace(/>/g, ">")
|
|
51
|
+
.replace(/"/g, '"')
|
|
52
|
+
.replace(/'/g, "'")
|
|
53
|
+
.replace(/&/g, "&");
|
|
54
|
+
}
|