@tpsdev-ai/flair 0.44.13 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.yaml +35 -2
- package/dist/cli.js +410 -21
- package/dist/doctor-client.js +266 -0
- package/dist/federation/scheduler.js +114 -9
- package/dist/hook-install.js +150 -1
- package/dist/lib/mcp-enable.js +59 -19
- package/dist/lib/safe-snapshot-extract.js +20 -2
- package/dist/lib/scheduler-platform.js +210 -1
- package/dist/rem/scheduler.js +113 -13
- package/dist/rem/snapshot.js +14 -13
- package/dist/resources/Memory.js +32 -1
- package/dist/resources/MemoryBootstrap.js +88 -51
- package/dist/resources/MemoryFeed.js +56 -1
- package/dist/resources/MemoryMaintenance.js +8 -2
- package/dist/resources/abstention.js +12 -9
- package/dist/resources/auth-middleware.js +11 -3
- package/dist/resources/bm25.js +7 -3
- package/dist/resources/mcp-oauth-flag.js +20 -0
- package/dist/resources/mcp-oauth.js +6 -1
- package/dist/resources/memory-visibility.js +48 -0
- package/dist/resources/semantic-retrieval-core.js +99 -19
- package/dist/src/lib/scheduler-platform.js +210 -1
- package/dist/src/rem/scheduler.js +113 -13
- package/docs/notes/mcp-oauth-model2.md +10 -3
- package/docs/quickstart-fabric.md +42 -3
- package/package.json +1 -1
- package/templates/bin/flair-federation-sync.sh.tmpl +8 -1
- package/templates/bin/flair-rem-nightly.sh.tmpl +8 -1
package/dist/lib/mcp-enable.js
CHANGED
|
@@ -306,7 +306,6 @@ export function buildMcpOAuthConfigBlock(params) {
|
|
|
306
306
|
const provider = params.idpProvider;
|
|
307
307
|
const envPrefix = `OAUTH_${provider.toUpperCase()}`;
|
|
308
308
|
const cimdAllowedHosts = params.cimdAllowedHosts ?? DEFAULT_CIMD_ALLOWED_HOSTS;
|
|
309
|
-
const enabled = params.enabled ?? true;
|
|
310
309
|
return {
|
|
311
310
|
"@harperfast/oauth": {
|
|
312
311
|
package: "@harperfast/oauth",
|
|
@@ -317,9 +316,29 @@ export function buildMcpOAuthConfigBlock(params) {
|
|
|
317
316
|
},
|
|
318
317
|
},
|
|
319
318
|
mcp: {
|
|
320
|
-
|
|
319
|
+
// flair#1152: whole-token env reference — same flag flair's in-process
|
|
320
|
+
// route gates on. ASYMMETRY (load-bearing, measured on oauth 2.5.0):
|
|
321
|
+
// the component's coerceConfigBoolean accepts ONLY "true"/"false" and
|
|
322
|
+
// DELETES any other string (unresolved placeholder, "1", "yes",
|
|
323
|
+
// garbage) so its disabled default applies; flair's mcpOAuthEnabled()
|
|
324
|
+
// (resources/mcp-oauth-flag.ts) accepts 1/true/yes/on. So "true" is
|
|
325
|
+
// the one value that enables BOTH; "1"/"yes"/"on" flip flair's /mcp
|
|
326
|
+
// handler on while the component AS stays off (fail-closed broken-on:
|
|
327
|
+
// all 401, no AS advertised); garbage/unset disable both. On oauth
|
|
328
|
+
// <2.5.0 there is NO normalization and an unresolved placeholder is a
|
|
329
|
+
// truthy string (fail-open) — which is why the resolved-version
|
|
330
|
+
// assertion in mcp-oauth-boot-safety.test.ts exists. If component
|
|
331
|
+
// `enabled` semantics ever change, or it ever drives flair handler
|
|
332
|
+
// registration directly, re-derive this table before shipping.
|
|
333
|
+
enabled: "${FLAIR_MCP_OAUTH}",
|
|
321
334
|
issuer: "${FLAIR_MCP_ISSUER}",
|
|
322
|
-
|
|
335
|
+
// flair#1180: NO `resource` key — the component's resolveResource()
|
|
336
|
+
// derives `<issuer>/mcp` at request time when it is absent, identical
|
|
337
|
+
// to flair's in-process derivation. The old composite
|
|
338
|
+
// "${FLAIR_MCP_ISSUER}/mcp" never interpolated (whole-token-only
|
|
339
|
+
// expansion) and failed every connect with invalid_target. Escape
|
|
340
|
+
// hatch: an operator needing a non-standard resource sets an explicit
|
|
341
|
+
// LITERAL absolute URL in config.yaml (never a composite).
|
|
323
342
|
accessTokenTtl: REQUIRED_ACCESS_TOKEN_TTL,
|
|
324
343
|
// Explicit fail-closed disable — see the doc comment above and the
|
|
325
344
|
// module header for why an omitted block is NOT equivalent to this.
|
|
@@ -336,14 +355,24 @@ export function buildMcpOAuthConfigBlock(params) {
|
|
|
336
355
|
};
|
|
337
356
|
}
|
|
338
357
|
// ─── Local config.yaml update (flair#1136) ──────────────────────────────────
|
|
358
|
+
/** The whole-token env reference `flair mcp enable` writes as mcp.enabled
|
|
359
|
+
* (flair#1152). The on/off choice lives in the environment (the secrets
|
|
360
|
+
* bundle stages FLAIR_MCP_OAUTH=true — see buildSecretsBundle for why it
|
|
361
|
+
* must be "true"), never as a literal in the config file. */
|
|
362
|
+
export const MCP_ENABLED_ENV_REFERENCE = "${FLAIR_MCP_OAUTH}";
|
|
339
363
|
/**
|
|
340
|
-
*
|
|
341
|
-
* `{ ok: false }` with a reason when the file can't be
|
|
364
|
+
* Set mcp.enabled in a local component config.yaml to the flair#1152 shape.
|
|
365
|
+
* Best-effort: returns `{ ok: false }` with a reason when the file can't be
|
|
366
|
+
* found or parsed.
|
|
367
|
+
*
|
|
368
|
+
* `enabled: true` writes the WHOLE-TOKEN env reference ${FLAIR_MCP_OAUTH}
|
|
369
|
+
* (never a literal `true` — the env var, staged to `true` by the secrets bundle,
|
|
370
|
+
* carries the choice; a legacy literal `true` found in the file is normalized
|
|
371
|
+
* to the reference). `enabled: false` writes literal `false` — decisively off
|
|
372
|
+
* regardless of environment.
|
|
342
373
|
*
|
|
343
374
|
* Looks for config.yaml at `explicitPath`, then `./config.yaml`, then
|
|
344
|
-
* `~/.flair/config.yaml`.
|
|
345
|
-
* with `mcp:\n enabled: true` (exact string match — avoids a YAML parser
|
|
346
|
-
* dependency for a single boolean flip).
|
|
375
|
+
* `~/.flair/config.yaml`.
|
|
347
376
|
*/
|
|
348
377
|
export function updateLocalConfigMcpEnabled(enabled, explicitPath) {
|
|
349
378
|
const candidates = explicitPath
|
|
@@ -356,11 +385,15 @@ export function updateLocalConfigMcpEnabled(enabled, explicitPath) {
|
|
|
356
385
|
break;
|
|
357
386
|
}
|
|
358
387
|
}
|
|
388
|
+
// The value the file should carry for this call (flair#1152): the env
|
|
389
|
+
// reference when enabling, literal false when disabling.
|
|
390
|
+
const target = enabled ? MCP_ENABLED_ENV_REFERENCE : false;
|
|
391
|
+
const targetLabel = enabled ? `${MCP_ENABLED_ENV_REFERENCE} (env-referenced)` : "false";
|
|
359
392
|
if (!configPath) {
|
|
360
393
|
return {
|
|
361
394
|
ok: false,
|
|
362
395
|
detail: `local config.yaml not found (tried: ${candidates.join(", ")}). ` +
|
|
363
|
-
`Set mcp.enabled: ${
|
|
396
|
+
`Set mcp.enabled: ${targetLabel} in your component config.yaml manually, then restart.`,
|
|
364
397
|
};
|
|
365
398
|
}
|
|
366
399
|
let raw;
|
|
@@ -388,7 +421,7 @@ export function updateLocalConfigMcpEnabled(enabled, explicitPath) {
|
|
|
388
421
|
return {
|
|
389
422
|
ok: false,
|
|
390
423
|
detail: `@harperfast/oauth block not found in ${configPath}. ` +
|
|
391
|
-
`Ensure the component block is present with mcp.enabled: ${
|
|
424
|
+
`Ensure the component block is present with mcp.enabled: ${targetLabel}.`,
|
|
392
425
|
};
|
|
393
426
|
}
|
|
394
427
|
const mcp = oauth.mcp;
|
|
@@ -396,15 +429,15 @@ export function updateLocalConfigMcpEnabled(enabled, explicitPath) {
|
|
|
396
429
|
return {
|
|
397
430
|
ok: false,
|
|
398
431
|
detail: `mcp key not found under @harperfast/oauth in ${configPath}. ` +
|
|
399
|
-
`Ensure the mcp block is present with enabled: ${
|
|
432
|
+
`Ensure the mcp block is present with enabled: ${targetLabel}.`,
|
|
400
433
|
};
|
|
401
434
|
}
|
|
402
435
|
const current = mcp.enabled;
|
|
403
|
-
if (current ===
|
|
404
|
-
return { ok: true, detail: `mcp.enabled already ${
|
|
436
|
+
if (current === target) {
|
|
437
|
+
return { ok: true, detail: `mcp.enabled already ${targetLabel} in ${configPath}` };
|
|
405
438
|
}
|
|
406
439
|
// Mutate the parsed document and re-emit.
|
|
407
|
-
mcp.enabled =
|
|
440
|
+
mcp.enabled = target;
|
|
408
441
|
const updated = yaml.dump(doc, { lineWidth: -1, noCompatMode: true });
|
|
409
442
|
try {
|
|
410
443
|
writeFileSync(configPath, updated, { encoding: "utf-8" });
|
|
@@ -412,7 +445,7 @@ export function updateLocalConfigMcpEnabled(enabled, explicitPath) {
|
|
|
412
445
|
catch (err) {
|
|
413
446
|
return { ok: false, detail: `cannot write ${configPath}: ${err.message}` };
|
|
414
447
|
}
|
|
415
|
-
return { ok: true, detail: `mcp.enabled set to ${
|
|
448
|
+
return { ok: true, detail: `mcp.enabled set to ${targetLabel} in ${configPath}` };
|
|
416
449
|
}
|
|
417
450
|
/** The exact callback URL to hand the operator when they create the IdP
|
|
418
451
|
* OAuth app ("with the exact GitHub callback URL printed"). */
|
|
@@ -425,7 +458,14 @@ export function idpCallbackUrl(issuer, idpProvider) {
|
|
|
425
458
|
export function buildSecretsBundle(params) {
|
|
426
459
|
const envPrefix = `OAUTH_${params.idpProvider.toUpperCase()}`;
|
|
427
460
|
return {
|
|
428
|
-
|
|
461
|
+
// "true" is the ONLY value both readers of this flag accept (flair#1152,
|
|
462
|
+
// measured against oauth 2.5.0): flair's strict mcpOAuthEnabled() takes
|
|
463
|
+
// 1/true/yes/on, but the component's coerceConfigBoolean takes ONLY
|
|
464
|
+
// "true"/"false" and DELETES anything else (disabled default applies).
|
|
465
|
+
// Staging "1" here would flip flair's /mcp handler ON while the
|
|
466
|
+
// component's AS stays OFF — fail-closed but broken-on (every request
|
|
467
|
+
// 401s, no AS is advertised). Keep this "true".
|
|
468
|
+
FLAIR_MCP_OAUTH: "true",
|
|
429
469
|
FLAIR_MCP_ISSUER: params.issuer.replace(/\/+$/, ""),
|
|
430
470
|
FLAIR_MCP_SIGNING_KEY_PEM: params.signingKeyPem,
|
|
431
471
|
[`${envPrefix}_CLIENT_ID`]: params.idpClientId,
|
|
@@ -1052,9 +1092,9 @@ export async function enableMcp(params, deps = {}) {
|
|
|
1052
1092
|
currentStep = "fabric-operator-deploy";
|
|
1053
1093
|
const msg = [
|
|
1054
1094
|
`Fabric deployment detected (${new URL(params.instance).hostname}).`,
|
|
1055
|
-
`The @harperfast/oauth block ships in config.yaml with mcp.enabled:
|
|
1056
|
-
`To activate:
|
|
1057
|
-
`
|
|
1095
|
+
`The @harperfast/oauth block ships in config.yaml with mcp.enabled: \${FLAIR_MCP_OAUTH} (env-referenced, flair#1152) — no config edit is needed.`,
|
|
1096
|
+
`To activate: apply the staged secrets (FLAIR_MCP_OAUTH=true among them) to the instance's environment (Fabric env), then restart the instance.`,
|
|
1097
|
+
`Deploys can no longer revert the choice — it lives in the environment, not the packed file.`,
|
|
1058
1098
|
`Then re-run \`flair mcp enable\` — earlier steps are idempotent and will be reused.`,
|
|
1059
1099
|
].join(" ");
|
|
1060
1100
|
push(false, msg);
|
|
@@ -68,6 +68,18 @@ function hasDotDotSegment(p) {
|
|
|
68
68
|
* `linkpath` is the entry's link target, when it has one.
|
|
69
69
|
*/
|
|
70
70
|
export function checkSnapshotEntry(entryPath, type, linkpath, resolvedTargetDir) {
|
|
71
|
+
// flair#901 fail-closed shape checks: an entry this check cannot READ is an
|
|
72
|
+
// entry it must not PASS. An empty path used to resolve to the target dir
|
|
73
|
+
// itself (inside, so ok), and a link entry with no linkpath skipped the
|
|
74
|
+
// link branches entirely — both silently approved exactly when the input
|
|
75
|
+
// was least trustworthy (tampered archive, or an upstream property rename
|
|
76
|
+
// surviving to runtime).
|
|
77
|
+
if (!entryPath) {
|
|
78
|
+
return { ok: false, reason: "entry has an empty or unreadable path — refusing an entry this check cannot classify" };
|
|
79
|
+
}
|
|
80
|
+
if ((type === "Link" || type === "SymbolicLink") && !linkpath) {
|
|
81
|
+
return { ok: false, reason: `${type === "Link" ? "hard link" : "symlink"} "${entryPath}" has no readable link target — refusing an entry this check cannot classify` };
|
|
82
|
+
}
|
|
71
83
|
if (looksAbsolute(entryPath)) {
|
|
72
84
|
return { ok: false, reason: `entry has an absolute path ("${entryPath}"), which would write outside the target directory` };
|
|
73
85
|
}
|
|
@@ -154,14 +166,20 @@ export async function validateSnapshotArchive(opts) {
|
|
|
154
166
|
const violations = [];
|
|
155
167
|
await tarList({
|
|
156
168
|
file: opts.file,
|
|
169
|
+
// Typed against tar's ReadEntry (flair#901) — a property rename upstream
|
|
170
|
+
// now fails compilation instead of coercing to ""/undefined at runtime.
|
|
171
|
+
// The coercions below remain as the runtime belt, and they no longer
|
|
172
|
+
// default to safe: checkSnapshotEntry refuses an empty path and a
|
|
173
|
+
// link-typed entry with no link target.
|
|
157
174
|
onReadEntry: (entry) => {
|
|
158
|
-
const
|
|
175
|
+
const entryPath = typeof entry.path === "string" ? entry.path : "";
|
|
176
|
+
const verdict = checkSnapshotEntry(entryPath, entry.type, typeof entry.linkpath === "string" && entry.linkpath !== "" ? entry.linkpath : undefined, resolvedTargetDir);
|
|
159
177
|
// `verdict.ok === false` rather than `!verdict.ok`: an explicit
|
|
160
178
|
// comparison against the literal discriminant is what narrows the union
|
|
161
179
|
// to its failure member, so `reason` is known to exist here. The union
|
|
162
180
|
// is deliberately shaped so a caller cannot read a reason off a success.
|
|
163
181
|
if (verdict.ok === false) {
|
|
164
|
-
violations.push({ entryPath
|
|
182
|
+
violations.push({ entryPath, reason: verdict.reason });
|
|
165
183
|
}
|
|
166
184
|
},
|
|
167
185
|
});
|
|
@@ -13,9 +13,15 @@
|
|
|
13
13
|
*
|
|
14
14
|
* `interpretActiveResult()` in particular encodes a production lesson
|
|
15
15
|
* (flair#850) that took a real outage to learn. It must not be re-derived.
|
|
16
|
+
* flair#1231 extended it one layer deeper: a load command exiting 0 proves the
|
|
17
|
+
* service manager ACCEPTED the job, not that the job can RUN — two fleet
|
|
18
|
+
* incidents (a stripped exec bit, a missing log directory) both passed the
|
|
19
|
+
* load check and died on the first real run, invisibly. The rule now encoded
|
|
20
|
+
* in `verifyFirstRun()`: success may not be claimed until the thing the
|
|
21
|
+
* operator asked for has been observed to happen once.
|
|
16
22
|
*/
|
|
17
23
|
import { existsSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
|
|
18
|
-
import { resolve, dirname } from "node:path";
|
|
24
|
+
import { resolve, dirname, isAbsolute } from "node:path";
|
|
19
25
|
import { platform } from "node:os";
|
|
20
26
|
import { spawnSync } from "node:child_process";
|
|
21
27
|
/**
|
|
@@ -126,3 +132,206 @@ export function writeFileWithDir(path, contents, mode = 0o600) {
|
|
|
126
132
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
127
133
|
writeFileSync(path, contents, { mode });
|
|
128
134
|
}
|
|
135
|
+
// ─── node binary resolution (flair#1231) ────────────────────────────────────
|
|
136
|
+
/**
|
|
137
|
+
* Resolves the ABSOLUTE path to the node binary at enable time, so the shim
|
|
138
|
+
* can `exec "<node>" "<script>"` with ZERO PATH lookups at run time.
|
|
139
|
+
*
|
|
140
|
+
* Why this exists: the shims switched from `exec "{{FLAIR_BIN}}"` (which
|
|
141
|
+
* required an exec bit that tarball extraction strips — the #1231 regression)
|
|
142
|
+
* to running the CLI under node, which needs read permission only. But a bare
|
|
143
|
+
* `exec node …` would introduce a run-time PATH lookup the old absolute-path
|
|
144
|
+
* form never had: whatever PATH the service manager's environment carries
|
|
145
|
+
* would pick the `node` that runs with the operator's credentials. So the
|
|
146
|
+
* node path is resolved HERE, once, from the enabling process's own
|
|
147
|
+
* environment, and baked into the shim — symmetric with how FLAIR_BIN is
|
|
148
|
+
* already handled.
|
|
149
|
+
*
|
|
150
|
+
* Resolution order:
|
|
151
|
+
* 1. `explicit` — caller/test override.
|
|
152
|
+
* 2. `process.execPath` when the enabling runtime IS node (the published
|
|
153
|
+
* CLI's case): absolute, known-good, already trusted to run this code.
|
|
154
|
+
* 3. `command -v node` in the enabling shell environment (dev/test under
|
|
155
|
+
* bun): the one deliberate PATH consultation, made at enable time by the
|
|
156
|
+
* operator's own session, never later by the service manager.
|
|
157
|
+
* Nothing resolvable ⇒ throw — enable must fail loudly rather than bake a
|
|
158
|
+
* run-time lookup into the shim.
|
|
159
|
+
*/
|
|
160
|
+
export function resolveNodeBin(explicit) {
|
|
161
|
+
if (explicit)
|
|
162
|
+
return explicit;
|
|
163
|
+
if (!process.versions.bun && process.execPath && isAbsolute(process.execPath)) {
|
|
164
|
+
return process.execPath;
|
|
165
|
+
}
|
|
166
|
+
const r = spawnReport(["/bin/sh", "-c", "command -v node"], STATUS_CHECK_TIMEOUT_MS);
|
|
167
|
+
const found = r.stdout.trim().split("\n")[0]?.trim() ?? "";
|
|
168
|
+
if (r.code === 0 && found && isAbsolute(found) && existsSync(found))
|
|
169
|
+
return found;
|
|
170
|
+
throw new Error("unable to resolve an absolute path to a node binary (not running under node, and `command -v node` " +
|
|
171
|
+
"found nothing). The scheduler shim runs `<node> <flair-script>` with the node path baked in at " +
|
|
172
|
+
"enable time — refusing to install a shim that would resolve `node` from the service manager's PATH " +
|
|
173
|
+
"at run time. Install node (or put it on PATH for this shell) and re-run enable.");
|
|
174
|
+
}
|
|
175
|
+
// ─── first-run verification (flair#1231) ────────────────────────────────────
|
|
176
|
+
// A load/bootstrap command exiting 0 proves the service manager accepted the
|
|
177
|
+
// job — not that the job can run. The only vantage that exercises the real
|
|
178
|
+
// failure modes (launchd spawn error 209 from a missing log dir, exit 126
|
|
179
|
+
// from a stripped exec bit) is the service manager itself, so the first run
|
|
180
|
+
// is triggered and observed THROUGH it, never via a bare spawn of the shim.
|
|
181
|
+
/** Poll cadence for darwin `launchctl print` first-run polling. */
|
|
182
|
+
export const FIRST_RUN_POLL_INTERVAL_MS = 150;
|
|
183
|
+
/** Total budget for first-run verification on both platforms. */
|
|
184
|
+
export const FIRST_RUN_BUDGET_MS = 12_000;
|
|
185
|
+
/** Synchronous sleep without spawning anything. */
|
|
186
|
+
function sleepSync(ms) {
|
|
187
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Parses `launchctl print <domain>/<label>` for run state. `last exit code`
|
|
191
|
+
* is absent (or "(never exited)") until a run has completed, and `pid =` /
|
|
192
|
+
* `state = running` are present only while one is in flight.
|
|
193
|
+
*/
|
|
194
|
+
export function parseLaunchdPrintExit(output) {
|
|
195
|
+
const running = /^\s*state\s*=\s*(?:running|spawn)/m.test(output) || /^\s*pid\s*=\s*\d+/m.test(output);
|
|
196
|
+
const m = /last exit (?:code|status)\s*=\s*(-?\d+)/.exec(output);
|
|
197
|
+
return { running, lastExitCode: m ? Number(m[1]) : null };
|
|
198
|
+
}
|
|
199
|
+
/** Parses `systemctl --user show <unit> --property=ExecMainStatus,Result`. */
|
|
200
|
+
export function parseSystemdShowExit(output) {
|
|
201
|
+
const m = /^ExecMainStatus=(-?\d+)\s*$/m.exec(output);
|
|
202
|
+
const r = /^Result=(\S+)\s*$/m.exec(output);
|
|
203
|
+
return { execMainStatus: m ? Number(m[1]) : null, result: r ? r[1] : null };
|
|
204
|
+
}
|
|
205
|
+
/** Reads the last lines of a log file for failure diagnostics. Never throws. */
|
|
206
|
+
export function readLogTail(path, maxLines = 12, maxChars = 1500) {
|
|
207
|
+
let text;
|
|
208
|
+
try {
|
|
209
|
+
text = readFileSync(path, "utf-8");
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
return { exists: false, empty: false, tail: "" };
|
|
213
|
+
}
|
|
214
|
+
const trimmed = text.trimEnd();
|
|
215
|
+
if (!trimmed)
|
|
216
|
+
return { exists: true, empty: true, tail: "" };
|
|
217
|
+
let tail = trimmed.split("\n").slice(-maxLines).join("\n");
|
|
218
|
+
if (tail.length > maxChars)
|
|
219
|
+
tail = tail.slice(-maxChars);
|
|
220
|
+
return { exists: true, empty: false, tail };
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Names the failure class for a recorded exit status, so the report can lead
|
|
224
|
+
* with actor+state instead of a bare number.
|
|
225
|
+
*/
|
|
226
|
+
export function describeExitCode(code) {
|
|
227
|
+
if (code === null)
|
|
228
|
+
return "no exit status recorded";
|
|
229
|
+
if (code === 126)
|
|
230
|
+
return "exit 126 — found but not runnable (permission denied / exec format)";
|
|
231
|
+
if (code === 127)
|
|
232
|
+
return "exit 127 — command not found";
|
|
233
|
+
if (code === 209)
|
|
234
|
+
return "exit 209 — launchd could not spawn the job (a missing/unwritable log directory produces this)";
|
|
235
|
+
return `exit ${code}`;
|
|
236
|
+
}
|
|
237
|
+
function spawnedNothing(r) {
|
|
238
|
+
return r.code === null && !r.stdout.trim() && !r.stderr.trim();
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Triggers the job's first run through the service manager and reads back how
|
|
242
|
+
* it ended (flair#1231). Call ONLY after the load/bootstrap command exited 0 —
|
|
243
|
+
* a load failure is its own failure mode with its own remedy, and layering a
|
|
244
|
+
* kickstart on top of it would blur which actor failed.
|
|
245
|
+
*
|
|
246
|
+
* darwin: `launchctl kickstart -k` returns immediately (it does NOT block for
|
|
247
|
+
* exit), so the recorded exit status is POLLED out of `launchctl print` until
|
|
248
|
+
* a completed run is visible or the budget lapses. linux: `systemctl --user
|
|
249
|
+
* start` on a oneshot blocks until the run exits, so a single
|
|
250
|
+
* `systemctl --user show` read afterwards suffices.
|
|
251
|
+
*
|
|
252
|
+
* "Can't tell" is its own state: a missing/unreachable service manager yields
|
|
253
|
+
* outcome "manager-unavailable", distinct from "run-failed" — the remedy
|
|
254
|
+
* points at the service manager, not at the job.
|
|
255
|
+
*/
|
|
256
|
+
export function verifyFirstRun(opts) {
|
|
257
|
+
const run = opts.hooks?.run ?? ((cmd, timeoutMs) => spawnReport(cmd, timeoutMs));
|
|
258
|
+
const sleep = opts.hooks?.sleep ?? sleepSync;
|
|
259
|
+
const now = opts.hooks?.now ?? Date.now;
|
|
260
|
+
const pollIntervalMs = opts.pollIntervalMs ?? FIRST_RUN_POLL_INTERVAL_MS;
|
|
261
|
+
const budgetMs = opts.budgetMs ?? FIRST_RUN_BUDGET_MS;
|
|
262
|
+
const finish = (outcome, exitCode, detail) => {
|
|
263
|
+
const log = outcome === "success"
|
|
264
|
+
? { exists: false, empty: false, tail: "" } // no diagnostics needed on success
|
|
265
|
+
: readLogTail(opts.stderrLogPath);
|
|
266
|
+
return {
|
|
267
|
+
verified: outcome === "success",
|
|
268
|
+
outcome,
|
|
269
|
+
exitCode,
|
|
270
|
+
detail,
|
|
271
|
+
logPath: opts.stderrLogPath,
|
|
272
|
+
stderrTail: log.tail,
|
|
273
|
+
logEmpty: log.exists && log.empty,
|
|
274
|
+
budgetMs,
|
|
275
|
+
};
|
|
276
|
+
};
|
|
277
|
+
if (opts.plat === "darwin") {
|
|
278
|
+
const target = opts.darwinTarget;
|
|
279
|
+
if (!target)
|
|
280
|
+
throw new Error("verifyFirstRun: darwinTarget is required on darwin");
|
|
281
|
+
const kickCmd = ["launchctl", "kickstart", "-k", target];
|
|
282
|
+
const kick = run(kickCmd, SPAWN_TIMEOUT_MS);
|
|
283
|
+
if (spawnedNothing(kick)) {
|
|
284
|
+
return finish("manager-unavailable", null, `launchctl could not be run (${kickCmd.join(" ")})`);
|
|
285
|
+
}
|
|
286
|
+
if (kick.code !== 0) {
|
|
287
|
+
return finish("start-failed", null, `${kickCmd.join(" ")} → code ${kick.code}${kick.stderr.trim() ? `: ${kick.stderr.trim()}` : ""}`);
|
|
288
|
+
}
|
|
289
|
+
const deadline = now() + budgetMs;
|
|
290
|
+
// Poll: kickstart returned immediately, so watch `launchctl print` until a
|
|
291
|
+
// COMPLETED run (not running + a recorded exit code) is visible.
|
|
292
|
+
for (;;) {
|
|
293
|
+
const printCmd = ["launchctl", "print", target];
|
|
294
|
+
const r = run(printCmd, STATUS_CHECK_TIMEOUT_MS);
|
|
295
|
+
if (spawnedNothing(r)) {
|
|
296
|
+
return finish("manager-unavailable", null, `launchctl could not be run (${printCmd.join(" ")})`);
|
|
297
|
+
}
|
|
298
|
+
if (r.code === 0) {
|
|
299
|
+
const { running, lastExitCode } = parseLaunchdPrintExit(r.stdout);
|
|
300
|
+
if (!running && lastExitCode !== null) {
|
|
301
|
+
return lastExitCode === 0
|
|
302
|
+
? finish("success", 0, `${printCmd.join(" ")} → last exit code = 0`)
|
|
303
|
+
: finish("run-failed", lastExitCode, `${printCmd.join(" ")} → last exit code = ${lastExitCode}`);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
if (now() >= deadline) {
|
|
307
|
+
return finish("timeout", null, `no completed run visible in ${printCmd.join(" ")} within ${Math.round(budgetMs / 1000)}s`);
|
|
308
|
+
}
|
|
309
|
+
sleep(pollIntervalMs);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
// linux
|
|
313
|
+
const unit = opts.linuxServiceUnit;
|
|
314
|
+
if (!unit)
|
|
315
|
+
throw new Error("verifyFirstRun: linuxServiceUnit is required on linux");
|
|
316
|
+
const startCmd = ["systemctl", "--user", "start", unit];
|
|
317
|
+
const start = run(startCmd, budgetMs);
|
|
318
|
+
if (spawnedNothing(start)) {
|
|
319
|
+
return finish("manager-unavailable", null, `systemctl could not be run (${startCmd.join(" ")})`);
|
|
320
|
+
}
|
|
321
|
+
if (/failed to connect to bus/i.test(start.stderr)) {
|
|
322
|
+
return finish("manager-unavailable", null, `${startCmd.join(" ")} → ${start.stderr.trim()}`);
|
|
323
|
+
}
|
|
324
|
+
if (start.code === null) {
|
|
325
|
+
return finish("timeout", null, `${startCmd.join(" ")} did not return within ${Math.round(budgetMs / 1000)}s`);
|
|
326
|
+
}
|
|
327
|
+
const showCmd = ["systemctl", "--user", "show", unit, "--property=ExecMainStatus,Result"];
|
|
328
|
+
const show = run(showCmd, STATUS_CHECK_TIMEOUT_MS);
|
|
329
|
+
const parsed = parseSystemdShowExit(show.stdout);
|
|
330
|
+
if (start.code === 0) {
|
|
331
|
+
// A blocking start of a oneshot exits 0 only when the run succeeded; the
|
|
332
|
+
// show read supplies the recorded status for the report.
|
|
333
|
+
return finish("success", parsed.execMainStatus ?? 0, `${startCmd.join(" ")} → ok`);
|
|
334
|
+
}
|
|
335
|
+
const resultTxt = parsed.result ? `, Result=${parsed.result}` : "";
|
|
336
|
+
return finish("run-failed", parsed.execMainStatus, `${startCmd.join(" ")} → code ${start.code}${resultTxt}${start.stderr.trim() ? `: ${start.stderr.trim()}` : ""}`);
|
|
337
|
+
}
|
package/dist/rem/scheduler.js
CHANGED
|
@@ -13,13 +13,13 @@
|
|
|
13
13
|
* No daemon code lives here — the scheduler invokes the shim, the shim
|
|
14
14
|
* invokes `flair rem nightly run-once`, the runner module does the work.
|
|
15
15
|
*/
|
|
16
|
-
import { existsSync, chmodSync, rmSync } from "node:fs";
|
|
16
|
+
import { existsSync, mkdirSync, chmodSync, rmSync } from "node:fs";
|
|
17
17
|
import { resolve, dirname } from "node:path";
|
|
18
18
|
import { homedir } from "node:os";
|
|
19
19
|
import { spawn } from "node:child_process";
|
|
20
20
|
import { fileURLToPath } from "node:url";
|
|
21
21
|
import { escapeXml } from "../lib/xml-escape.js";
|
|
22
|
-
import { detectPlatform as detectPlatformFor, spawnReport, readTemplate as readTemplateFrom, renderTemplateWith, writeFileWithDir, interpretActiveResult, describeLoadFailure as describeLoadFailureFor, STATUS_CHECK_TIMEOUT_MS, } from "../lib/scheduler-platform.js";
|
|
22
|
+
import { detectPlatform as detectPlatformFor, spawnReport, readTemplate as readTemplateFrom, renderTemplateWith, writeFileWithDir, interpretActiveResult, describeLoadFailure as describeLoadFailureFor, describeExitCode, resolveNodeBin, verifyFirstRun, STATUS_CHECK_TIMEOUT_MS, } from "../lib/scheduler-platform.js";
|
|
23
23
|
// Re-exported so this module's public surface is unchanged by the extraction
|
|
24
24
|
// into src/lib/scheduler-platform.ts (a second scheduler — `flair federation
|
|
25
25
|
// sync enable` — needs the identical launchctl/systemctl interpretation, and
|
|
@@ -87,15 +87,16 @@ function validateSchedule(hour, minute) {
|
|
|
87
87
|
throw new Error(`minute must be an integer 0-59, got ${minute}`);
|
|
88
88
|
}
|
|
89
89
|
}
|
|
90
|
-
function buildSubstitutions(opts, shimPath, flairBin) {
|
|
90
|
+
function buildSubstitutions(opts, shimPath, flairBin, nodeBin) {
|
|
91
91
|
validateSchedule(opts.hour, opts.minute);
|
|
92
92
|
if (!/^[a-zA-Z0-9_-]+$/.test(opts.agentId)) {
|
|
93
93
|
throw new Error(`invalid agent id: ${opts.agentId}`);
|
|
94
94
|
}
|
|
95
95
|
return {
|
|
96
96
|
FLAIR_BIN: flairBin,
|
|
97
|
+
NODE_BIN: nodeBin,
|
|
97
98
|
SHIM_PATH: shimPath,
|
|
98
|
-
HOME: homedir(),
|
|
99
|
+
HOME: opts.homeOverride ?? homedir(),
|
|
99
100
|
AGENT_ID: opts.agentId,
|
|
100
101
|
FLAIR_URL: opts.flairUrl,
|
|
101
102
|
HOUR: String(opts.hour),
|
|
@@ -183,10 +184,14 @@ export function describeLoadFailure(plat, loadResult) {
|
|
|
183
184
|
* succeeded) is unit-testable without spawning a real launchctl/systemctl or
|
|
184
185
|
* parsing CLI argv.
|
|
185
186
|
*
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
*
|
|
189
|
-
*
|
|
187
|
+
* flair#1231 deepened the #850 rule by one layer: activation exiting 0 proves
|
|
188
|
+
* the service manager ACCEPTED the job, not that the job can run — a stripped
|
|
189
|
+
* exec bit and a missing log directory both passed activation and killed the
|
|
190
|
+
* first real run invisibly. So the ✅ headline is now additionally gated on
|
|
191
|
+
* `firstRunVerified`: success may not be claimed until the thing the operator
|
|
192
|
+
* asked for — a REM run through the service manager — has been observed to
|
|
193
|
+
* happen once. A missing `loadResult`/`firstRun` (test-only skipLoad shape)
|
|
194
|
+
* therefore withholds the headline too, instead of being treated as success.
|
|
190
195
|
*/
|
|
191
196
|
export function formatEnableReport(r, input) {
|
|
192
197
|
const { hour, minute, agentId, flairUrl } = input;
|
|
@@ -212,6 +217,57 @@ export function formatEnableReport(r, input) {
|
|
|
212
217
|
lines.push(` Nothing is scheduled until activation succeeds. Check anytime with: flair rem nightly status`);
|
|
213
218
|
return { lines, ok: false };
|
|
214
219
|
}
|
|
220
|
+
if (!r.firstRunVerified) {
|
|
221
|
+
const fr = r.firstRun;
|
|
222
|
+
const headline = fr?.outcome === "run-failed"
|
|
223
|
+
? `⚠️ REM nightly scheduler installed but the first run FAILED (${describeExitCode(fr.exitCode)})`
|
|
224
|
+
: fr?.outcome === "timeout"
|
|
225
|
+
? `⚠️ REM nightly scheduler installed but the first run did not complete within ${Math.round(fr.budgetMs / 1000)}s — cannot confirm it works`
|
|
226
|
+
: fr?.outcome === "manager-unavailable"
|
|
227
|
+
? `⚠️ REM nightly scheduler installed but the service manager is unreachable — cannot verify the first run`
|
|
228
|
+
: fr?.outcome === "start-failed"
|
|
229
|
+
? `⚠️ REM nightly scheduler installed but the first run could not be started`
|
|
230
|
+
: `⚠️ REM nightly scheduler installed but the first run was never verified`;
|
|
231
|
+
const lines = [
|
|
232
|
+
headline,
|
|
233
|
+
` Schedule: ${scheduleTime} local time`,
|
|
234
|
+
` Scheduler: ${r.schedulerPath}`,
|
|
235
|
+
` Shim: ${r.shimPath}`,
|
|
236
|
+
` Agent: ${agentId}`,
|
|
237
|
+
` Flair URL: ${flairUrl}`,
|
|
238
|
+
];
|
|
239
|
+
if (r.loadResult)
|
|
240
|
+
lines.push(` Load: ${r.loadCommand.join(" ")} → ok`);
|
|
241
|
+
if (fr) {
|
|
242
|
+
lines.push(` First run: ${fr.detail}`);
|
|
243
|
+
if (fr.stderrTail) {
|
|
244
|
+
lines.push(` Log tail (${fr.logPath}):`);
|
|
245
|
+
for (const l of fr.stderrTail.split("\n"))
|
|
246
|
+
lines.push(` ${l}`);
|
|
247
|
+
}
|
|
248
|
+
else if (fr.logEmpty) {
|
|
249
|
+
lines.push(` Log file ${fr.logPath} exists but is EMPTY — the run died before writing anything.`);
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
lines.push(` No log file at ${fr.logPath}.`);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
lines.push("");
|
|
256
|
+
if (fr?.outcome === "timeout") {
|
|
257
|
+
lines.push(` The run may legitimately still be going (a REM cycle can be slow). Check the log above and`);
|
|
258
|
+
lines.push(` \`flair rem nightly status\`; no cycle has been CONFIRMED to work yet.`);
|
|
259
|
+
}
|
|
260
|
+
else if (fr?.outcome === "manager-unavailable") {
|
|
261
|
+
lines.push(` The scheduler files are installed, but launchctl/systemctl could not be consulted, so whether`);
|
|
262
|
+
lines.push(` the nightly cycle runs is UNKNOWN. Fix the service manager for this session, then re-run \`flair rem nightly enable\`.`);
|
|
263
|
+
}
|
|
264
|
+
else {
|
|
265
|
+
lines.push(` No REM cycle has run. Fix the cause above, then re-run \`flair rem nightly enable\`.`);
|
|
266
|
+
}
|
|
267
|
+
lines.push("");
|
|
268
|
+
lines.push(` Check anytime with: flair rem nightly status`);
|
|
269
|
+
return { lines, ok: false };
|
|
270
|
+
}
|
|
215
271
|
const lines = [
|
|
216
272
|
`✅ REM nightly scheduler enabled (${r.platform})`,
|
|
217
273
|
` Schedule: ${scheduleTime} local time`,
|
|
@@ -223,9 +279,9 @@ export function formatEnableReport(r, input) {
|
|
|
223
279
|
if (r.loadResult) {
|
|
224
280
|
lines.push(` Load: ${r.loadCommand.join(" ")} → ok`);
|
|
225
281
|
}
|
|
282
|
+
lines.push(` First run: completed through the service manager, exit 0`);
|
|
226
283
|
lines.push("");
|
|
227
|
-
lines.push(`
|
|
228
|
-
lines.push(` before the first scheduled fire. Disable with \`flair rem nightly disable\`.`);
|
|
284
|
+
lines.push(`Disable with \`flair rem nightly disable\`.`);
|
|
229
285
|
return { lines, ok: true };
|
|
230
286
|
}
|
|
231
287
|
/**
|
|
@@ -268,9 +324,29 @@ export function formatStatusReport(s) {
|
|
|
268
324
|
export function enableScheduler(opts) {
|
|
269
325
|
const plat = detectPlatform(opts.platformOverride);
|
|
270
326
|
const flairBin = opts.flairBin ?? process.argv[1] ?? "flair";
|
|
327
|
+
const nodeBin = resolveNodeBin(opts.nodeBin);
|
|
271
328
|
const shimPath = opts.shimPathOverride ?? SHIM_PATH_DEFAULT;
|
|
272
329
|
const templateRoot = opts.templateRootOverride ?? defaultTemplateRoot();
|
|
273
|
-
const subs = buildSubstitutions(opts, shimPath, flairBin);
|
|
330
|
+
const subs = buildSubstitutions(opts, shimPath, flairBin, nodeBin);
|
|
331
|
+
// 0. Create the log directory the unit files point stdout/stderr at.
|
|
332
|
+
// Nothing else ever creates it — launchd kills a job whose StandardOutPath
|
|
333
|
+
// directory is missing (spawn error 209) and systemd fails the unit (#1231).
|
|
334
|
+
//
|
|
335
|
+
// Mode 0700 is load-bearing, NOT cosmetic: REM's nightly log carries
|
|
336
|
+
// distillation CANDIDATE CONTENT — actual memory text, not just counts.
|
|
337
|
+
// Relaxing it to 0755 (e.g. "for shared debugging") would expose memory
|
|
338
|
+
// content to every local user.
|
|
339
|
+
const logsDir = resolve(subs.HOME, ".flair", "logs");
|
|
340
|
+
try {
|
|
341
|
+
mkdirSync(logsDir, { recursive: true, mode: 0o700 });
|
|
342
|
+
}
|
|
343
|
+
catch (err) {
|
|
344
|
+
throw new Error(`could not create the scheduler log directory ${logsDir}: ${err?.message ?? err}. ` +
|
|
345
|
+
`The service manager writes the job's stdout/stderr there; without it the first run dies ` +
|
|
346
|
+
`before producing any output. Fix whatever blocks creating that directory, then re-run ` +
|
|
347
|
+
`\`flair rem nightly enable\`.`);
|
|
348
|
+
}
|
|
349
|
+
const stderrLogPath = resolve(logsDir, "rem-nightly.stderr.log");
|
|
274
350
|
// 1. Deploy the shim (always — both platforms invoke it).
|
|
275
351
|
const shimContents = renderTemplate(readTemplate(templateRoot, "bin/flair-rem-nightly.sh.tmpl"), subs);
|
|
276
352
|
writeFileWithDir(shimPath, shimContents, 0o700);
|
|
@@ -282,12 +358,26 @@ export function enableScheduler(opts) {
|
|
|
282
358
|
writeFileWithDir(plistPath, plistContents, 0o600);
|
|
283
359
|
const loadCommand = ["launchctl", "bootstrap", `gui/${process.getuid?.() ?? ""}`, plistPath];
|
|
284
360
|
let loadResult;
|
|
361
|
+
let firstRun;
|
|
285
362
|
if (!opts.skipLoad) {
|
|
286
363
|
// Bootout first in case a prior install left the job loaded.
|
|
287
364
|
spawnReport(["launchctl", "bootout", `gui/${process.getuid?.() ?? ""}`, plistPath]);
|
|
288
365
|
loadResult = spawnReport(loadCommand);
|
|
366
|
+
if (loadResult.code === 0) {
|
|
367
|
+
// Ordering gate (#1231): verify the first run ONLY after the load
|
|
368
|
+
// exited 0. A load failure is its own failure mode with its own
|
|
369
|
+
// remedy — kickstarting on top of it would blur which actor failed.
|
|
370
|
+
firstRun = verifyFirstRun({
|
|
371
|
+
plat,
|
|
372
|
+
darwinTarget: `gui/${process.getuid?.() ?? ""}/dev.flair.rem.nightly`,
|
|
373
|
+
stderrLogPath,
|
|
374
|
+
});
|
|
375
|
+
}
|
|
289
376
|
}
|
|
290
|
-
return {
|
|
377
|
+
return {
|
|
378
|
+
platform: plat, shimPath, schedulerPath: plistPath, loadCommand, loadResult,
|
|
379
|
+
firstRunVerified: firstRun?.verified === true, firstRun,
|
|
380
|
+
};
|
|
291
381
|
}
|
|
292
382
|
// Linux: systemd user units.
|
|
293
383
|
const timerPath = opts.systemdTimerOverride ?? SYSTEMD_TIMER_PATH;
|
|
@@ -298,11 +388,21 @@ export function enableScheduler(opts) {
|
|
|
298
388
|
writeFileWithDir(timerPath, timerContents, 0o600);
|
|
299
389
|
const loadCommand = ["systemctl", "--user", "enable", "--now", "flair-rem-nightly.timer"];
|
|
300
390
|
let loadResult;
|
|
391
|
+
let firstRun;
|
|
301
392
|
if (!opts.skipLoad) {
|
|
302
393
|
spawnReport(["systemctl", "--user", "daemon-reload"]);
|
|
303
394
|
loadResult = spawnReport(loadCommand);
|
|
395
|
+
if (loadResult.code === 0) {
|
|
396
|
+
// Ordering gate (#1231): only after the load exited 0. Starts the
|
|
397
|
+
// SERVICE unit directly (oneshot ⇒ blocks until the run exits) rather
|
|
398
|
+
// than waiting for the nightly timer to fire.
|
|
399
|
+
firstRun = verifyFirstRun({ plat, linuxServiceUnit: "flair-rem-nightly.service", stderrLogPath });
|
|
400
|
+
}
|
|
304
401
|
}
|
|
305
|
-
return {
|
|
402
|
+
return {
|
|
403
|
+
platform: plat, shimPath, schedulerPath: timerPath, loadCommand, loadResult,
|
|
404
|
+
firstRunVerified: firstRun?.verified === true, firstRun,
|
|
405
|
+
};
|
|
306
406
|
}
|
|
307
407
|
/**
|
|
308
408
|
* Removes the scheduler entry. Audit log + snapshots are preserved.
|