@tpsdev-ai/flair 0.30.0 → 0.31.0

package/dist/src/rem/scheduler.js

@@ -13,25 +13,25 @@
  * No daemon code lives here — the scheduler invokes the shim, the shim
  * invokes `flair rem nightly run-once`, the runner module does the work.
  */
-import { existsSync, mkdirSync, writeFileSync, readFileSync, chmodSync, rmSync } from "node:fs";
+import { existsSync, chmodSync, rmSync } from "node:fs";
 import { resolve, dirname } from "node:path";
-import { homedir, platform } from "node:os";
-import { spawn, spawnSync } from "node:child_process";
+import { homedir } from "node:os";
+import { spawn } from "node:child_process";
 import { fileURLToPath } from "node:url";
+import { escapeXml } from "../lib/xml-escape.js";
+import { detectPlatform as detectPlatformFor, spawnReport, readTemplate as readTemplateFrom, renderTemplateWith, writeFileWithDir, interpretActiveResult, describeLoadFailure as describeLoadFailureFor, STATUS_CHECK_TIMEOUT_MS, } from "../lib/scheduler-platform.js";
+// Re-exported so this module's public surface is unchanged by the extraction
+// into src/lib/scheduler-platform.ts (a second scheduler — `flair federation
+// sync enable` — needs the identical launchctl/systemctl interpretation, and
+// flair#850's lesson must have exactly one implementation).
+export { interpretActiveResult };
 export const SHIM_PATH_DEFAULT = resolve(homedir(), ".flair", "bin", "flair-rem-nightly");
 export const LAUNCHD_PLIST_PATH = resolve(homedir(), "Library", "LaunchAgents", "dev.flair.rem.nightly.plist");
 export const SYSTEMD_USER_DIR = resolve(homedir(), ".config", "systemd", "user");
 export const SYSTEMD_TIMER_PATH = resolve(SYSTEMD_USER_DIR, "flair-rem-nightly.timer");
 export const SYSTEMD_SERVICE_PATH = resolve(SYSTEMD_USER_DIR, "flair-rem-nightly.service");
 function detectPlatform(override) {
-    if (override)
-        return override;
-    const p = platform();
-    if (p === "darwin")
-        return "darwin";
-    if (p === "linux")
-        return "linux";
-    throw new Error(`unsupported platform for REM nightly scheduler: ${p} (only darwin and linux)`);
+    return detectPlatformFor("REM nightly scheduler", override);
 }
 function defaultTemplateRoot() {
     // Templates live alongside dist/ in the published package and alongside
@@ -50,19 +50,30 @@ function defaultTemplateRoot() {
     throw new Error(`unable to locate templates directory (looked in: ${candidates.join(", ")})`);
 }
 export function renderTemplate(text, subs) {
-    return text.replace(/\{\{([A-Z_]+)\}\}/g, (_, key) => {
-        const value = subs[key];
-        if (value === undefined)
-            throw new Error(`unknown template placeholder: ${key}`);
-        return String(value);
-    });
+    return renderTemplateWith(text, { ...subs }, (v) => v);
+}
+/**
+ * renderTemplate() for the launchd plist template specifically: substituted
+ * values are XML-escaped.
+ *
+ * A launchd plist is XML, so a substitution carrying `&`, `<`, `>`, `"` or
+ * `'` makes it malformed and `launchctl bootstrap` rejects it — the timer
+ * silently never registers. FLAIR_URL is the realistic carrier (a URL with
+ * more than one query parameter contains `&`), but HOME and SHIM_PATH are
+ * arbitrary paths and equally capable of it. HOUR/MINUTE and AGENT_ID are
+ * validated upstream in buildSubstitutions(), but they are escaped too
+ * rather than exempted — the point of a chokepoint is that no value gets to
+ * argue it is special.
+ *
+ * Deliberately NOT folded into renderTemplate(): the same substitutions are
+ * rendered into the systemd unit files and the shell shim, where XML
+ * escaping would be corruption rather than a fix.
+ */
+export function renderPlistTemplate(text, subs) {
+    return renderTemplateWith(text, { ...subs }, escapeXml);
 }
 export function readTemplate(rootDir, relativePath) {
-    const full = resolve(rootDir, relativePath);
-    if (!existsSync(full)) {
-        throw new Error(`template not found: ${full}`);
-    }
-    return readFileSync(full, "utf-8");
+    return readTemplateFrom(rootDir, relativePath);
 }
 /**
  * Validates the hour:minute schedule. Throws on invalid input rather than
@@ -93,32 +104,6 @@ function buildSubstitutions(opts, shimPath, flairBin) {
         MINUTE_PAD: String(opts.minute).padStart(2, "0"),
     };
 }
-function writeFileWithDir(path, contents, mode = 0o600) {
-    const dir = dirname(path);
-    if (!existsSync(dir))
-        mkdirSync(dir, { recursive: true, mode: 0o700 });
-    writeFileSync(path, contents, { mode });
-}
-// 30s ceiling on launchctl/systemctl invocations so a hung service manager
-// can't block the CLI indefinitely. Sherlock #415 follow-up.
-const SPAWN_TIMEOUT_MS = 30_000;
-// Status-check spawns (launchctl print / systemctl is-active) return
-// near-instantly when the service manager is reachable, and fail fast (no
-// hang) when it isn't (e.g. "Failed to connect to bus"). A short ceiling
-// keeps `flair rem nightly status` and the Health endpoint responsive even
-// when the query is inconclusive.
-const STATUS_CHECK_TIMEOUT_MS = 5_000;
-function spawnReport(cmd, timeoutMs = SPAWN_TIMEOUT_MS) {
-    const r = spawnSync(cmd[0], cmd.slice(1), {
-        encoding: "buffer",
-        timeout: timeoutMs,
-    });
-    return {
-        code: r.status,
-        stdout: r.stdout?.toString("utf-8") ?? "",
-        stderr: r.stderr?.toString("utf-8") ?? "",
-    };
-}
 /**
  * Command to genuinely query the platform scheduler's active/loaded state
  * for a given install (as opposed to the shape of `loadCommand`, which
@@ -131,37 +116,6 @@ function activeCheckCommand(plat) {
     }
     return ["systemctl", "--user", "is-active", "flair-rem-nightly.timer"];
 }
-/**
- * Interprets the result of `activeCheckCommand()`. Shared by the sync
- * (CLI) and async (server) callers.
- *
- * - true  — the service manager confirms the job is loaded/active.
- * - false — confirmed NOT active. This includes the "no session bus" case
- *   (flair#850): when `systemctl --user` can't reach a bus, it fails
- *   before printing a status word ("Failed to connect to bus: No medium
- *   found") — but nothing CAN be running without a bus, so `false` is the
- *   honest answer, not "unknown".
- * - null  — genuinely inconclusive (the command itself couldn't run at
- *   all — e.g. the binary is missing — with no output to interpret).
- */
-export function interpretActiveResult(plat, code, stdout, stderr) {
-    const noOutput = !stdout.trim() && !stderr.trim();
-    if (plat === "darwin") {
-        if (code === 0)
-            return true;
-        if (code === null && noOutput)
-            return null; // spawn itself failed — inconclusive
-        return false; // launchctl ran and reported not-loaded
-    }
-    const out = stdout.trim();
-    if (out === "active" || out === "activating")
-        return true;
-    if (out === "inactive" || out === "failed" || out === "unknown")
-        return false;
-    if (code === null && noOutput)
-        return null; // spawn itself failed — inconclusive
-    return false; // covers the no-bus case: empty stdout, nonzero/failed exit
-}
 /**
  * Synchronous active-state check for CLI use (`flair rem nightly status`).
  * Blocking is fine here — this is a one-shot process and the caller wants
@@ -220,13 +174,7 @@ export async function queryActiveStateAsync(plat, timeoutMs = STATUS_CHECK_TIMEO
  * still has something to go on.
  */
 export function describeLoadFailure(plat, loadResult) {
-    const stderr = loadResult.stderr || "";
-    if (plat === "linux" && /failed to connect to bus/i.test(stderr)) {
-        return ("No systemd user session bus is available in this session (common over ssh without lingering, " +
-            "in containers, or under CI). Fix: enable lingering for this user — `loginctl enable-linger <user>` " +
-            "— then re-run `flair rem nightly enable`.");
-    }
-    return null;
+    return describeLoadFailureFor(plat, loadResult, "flair rem nightly enable");
 }
 /**
  * Formats the `flair rem nightly enable` report from an `EnableResult`.
@@ -330,7 +278,7 @@ export function enableScheduler(opts) {
     // 2. Write the scheduler entry.
     if (plat === "darwin") {
         const plistPath = opts.launchdPlistOverride ?? LAUNCHD_PLIST_PATH;
-        const plistContents = renderTemplate(readTemplate(templateRoot, "launchd/dev.flair.rem.nightly.plist.tmpl"), subs);
+        const plistContents = renderPlistTemplate(readTemplate(templateRoot, "launchd/dev.flair.rem.nightly.plist.tmpl"), subs);
         writeFileWithDir(plistPath, plistContents, 0o600);
         const loadCommand = ["launchctl", "bootstrap", `gui/${process.getuid?.() ?? ""}`, plistPath];
         let loadResult;

package/docs/deploying-on-fabric.md

@@ -0,0 +1,267 @@
+# Deploying on Harper Fabric
+
+Run Flair as a component on [Harper Fabric](https://www.harperdb.io/) — managed, hosted,
+multi-region — instead of a local process you own. The hosted counterpart to
+[deployment.md](deployment.md); two things define it: you *deploy* rather than install,
+and **you get no shell on the node.**
+
+## When to use it
+
+| | Standalone local | Harper Fabric |
+|---|---|---|
+| You run | `flair init` | `flair deploy` |
+| Shell on the node | yes | **no** |
+| Multi-region / failover | no | yes |
+| Upgrade | `flair upgrade` | `flair upgrade --target <url>` |
+
+Choose Fabric for an always-on hub in more than one region. For a single instance on a VPS
+you already have, [Remote Server](../README.md#remote-server) is simpler and keeps your shell.
+
+---
+
+## Quickstart
+
+### 1. Deploy the component
+
+```bash
+export FABRIC_USER=<admin> FABRIC_PASSWORD=<pass>
+
+# Validate args and package layout without deploying
+flair deploy --fabric-org <org> --fabric-cluster <cluster> --dry-run
+
+flair deploy --fabric-org <org> --fabric-cluster <cluster>
+```
+
+Target defaults to `https://<cluster>.<org>.harperfabric.com`; override with `--target`.
+
+Credentials go via the environment, not argv, so they stay out of `ps`. Use
+`--fabric-password-file <path>` (mode `0600`) when scripting; inline flags leak to history.
+
+Deploy verifies the served API (Harper reports success for a component serving
+nothing; `--no-verify` opts out), waits 600s for replication (Harper's 120s default
+aborts mid-replicate on Fabric), and polls for convergence before reporting one.
+
+> `--fabric-token` is accepted but **fails** — `deploy_component` is Basic-auth only.
+
+### 2. Provision
+
+```bash
+flair init --target https://<cluster>.<org>.harperfabric.com \
+  --ops-target <ops-url> \
+  --cluster-admin-user <user> --cluster-admin-pass <pass> \
+  --remote --force
+```
+
+- `--force` is required — this writes to a live instance.
+- `--remote` marks it a federation **hub** and creates the `flair_pair_initiator` role;
+  without it, pairing later fails role-not-found.
+- Generated admin password lands in `~/.tps/secrets/flair-fabric-hdb` (mode `0600`);
+  `--flair-admin-pass` to choose your own.
+
+Run **once**, before serving multi-region traffic — see
+[Multi-region](#multi-region-replication-vs-federation).
+
+### 3. Verify
+
+```bash
+curl -sf https://<cluster>.<org>.harperfabric.com/Health
+flair fleet verify --target https://<cluster>.<org>.harperfabric.com
+flair status --target https://<cluster>.<org>.harperfabric.com
+```
+
+### 4. Connect a client
+
+```bash
+export FLAIR_URL=https://<cluster>.<org>.harperfabric.com
+
+# Register an agent — --ops-target is required, see Ports below
+flair agent add mybot --target "$FLAIR_URL" --ops-target <ops-url>
+```
+
+Also set `FLAIR_PUBLIC_URL` to that URL in the component's environment — OAuth metadata
+and A2A discovery advertise it, or clients see a loopback address.
+
+---
+
+## Ports: the derivation trap
+
+Locally, Flair serves data on `19926` and the ops API on `19925`. The CLI derives
+**ops = data − 1** everywhere.
+
+A managed endpoint is HTTPS on 443 with no port, so derivation produces **`:442`** —
+where nothing answers:
+
+```bash
+# --target https://<cluster>.<org>.harperfabric.com   → ops derived as :442  ✗
+# --target https://<fabric-node>:19926/<instance>     → ops derived as :19925 ✓
+```
+
+**Pass `--ops-target <url>` explicitly** (or set `FLAIR_OPS_TARGET`) on any command that
+touches the ops API: `init --target`, `agent add --target`, `federation token --target`.
+
+> **Gap — needs a Fabric account to verify.** This guide does not state the correct
+> ops-API URL for a managed `*.harperfabric.com` instance, or whether the ops API is
+> reachable remotely there at all. The derivation above is certain (read from source);
+> the right value to pass is not.
+
+Precedence: `--target` > `--url` > `FLAIR_TARGET` > `FLAIR_URL` > localhost. For ops:
+`--ops-target` > `FLAIR_OPS_TARGET` > derived > localhost.
+
+## Auth
+
+Flair ships `authorizeLocal: false` — it only governs loopback, so it changes nothing
+here; remote callers always need real credentials.
+
+- Admin ops use Basic auth; agents use Ed25519 ([auth.md](auth.md)).
+- `/Health` is public — your liveness check.
+- Don't debug with raw `curl` — on Fabric the auth gate fires before the resource
+  handler, so a hand-rolled request 401s in a way that looks like a Flair bug and isn't.
+
+---
+
+## Pairing a spoke to a hosted hub
+
+No shell on the hub, so mint the token remotely rather than over `ssh`:
+
+```bash
+# On any machine — note --ops-target, this hits the ops API
+FLAIR_ADMIN_PASS=<hub-admin-password> flair federation token \
+  --target https://<cluster>.<org>.harperfabric.com \
+  --ops-target <ops-url> > ./pair-triple.json
+
+# On the spoke
+flair federation pair https://<cluster>.<org>.harperfabric.com \
+  --token-from ./pair-triple.json
+```
+
+`federation token` takes `--admin-pass`, **not** `--admin-pass-file`. The triple holds a
+one-time credential — delete it after pairing (60-min TTL, `--ttl`).
+
+### Federation is push-only
+
+**A spoke pushes up. It cannot pull down.** `POST /FederationSync` is one-directional per
+call and there is no pull endpoint anywhere.
+
+For both directions, each side pairs **as a spoke of the other**: two pairings, two
+tokens, two syncs. No setting makes sync bidirectional.
+
+Syncs `Memory`, `Soul`, `Agent`, `Relationship`. **`Presence` is not federated** — no
+cross-spoke roster.
+
+### Keeping a spoke synced
+
+`flair federation sync` is one-shot; `watch` dies with its terminal. Use the scheduled
+driver ([federation.md](federation.md#keeping-it-synced)):
+
+```bash
+flair federation sync enable --interval 300
+flair federation sync status
+```
+
+> **The driver is local-only.** It writes a launchd job or systemd timer **on the machine
+> running the CLI**; `--target` just points that local job at a remote instance. You
+> cannot install a driver on a Fabric node.
+
+A periodic one-shot, not a supervised watcher; `--interval` is the latency knob.
+`flair federation status` omits its driver verdict for a remote `--target`.
+
+---
+
+## Multi-region: replication vs federation
+
+Fabric gives you N regional nodes running one component — **not** N Flair instances.
+Instance identity lives in the `Instance` table (`schemas/federation.graphql`); it is a
+Harper table, so **it replicates** — every node shares one Flair identity.
+
+| Layer | Handles | Scope |
+|---|---|---|
+| Harper replication | your app across its own regions | intra-app, automatic |
+| Flair federation | this app ↔ another Flair instance | instance-to-instance, you pair it |
+
+**You do not federate your own regions to each other** — Harper already did. Use
+`flair federation pair` only to reach a *separate* Flair instance.
+
+One asymmetry: identity replicates, but `flair-instance.yaml` (port config, per data
+directory) does not. A node reporting a different **port** is expected; a different
+**instance id** is not.
+
+Provision once, from one place, before serving multi-region traffic: step 2's
+`flair init … --remote` seeds the identity row so it exists before any node needs it.
+
+---
+
+## Operating it
+
+### What works remotely
+
+| Command | Gives you |
+|---|---|
+| `GET /Health` | liveness, no auth |
+| `flair status --target <url>` | subsystem rollups; the only byte counts available remotely |
+| `flair quality --target <url>` | recall/coverage; halted-migration reasons |
+| `flair fleet verify --target <url>` | health, auth, version across origin + Flair peers |
+| `flair federation status\|verify\|reachability --target <url>` | peer table, sync recency, probes |
+
+`fleet verify` exit codes: 1 origin failed, 2 peer version skew, 3 peer unverifiable.
+
+> **A credential mismatch renders as an empty section, not an error.** `flair status`
+> reads `/HealthDetail` with `FLAIR_ADMIN_PASS` / `HDB_ADMIN_PASSWORD` / a pinned agent
+> key — **not** the `FABRIC_*` credentials `deploy` and `fleet verify` use. On failure it
+> renders blank — a blank Disk section means "couldn't authenticate" as often as
+> "nothing to report".
+
+### What doesn't
+
+**`flair doctor` takes no `--target`** — it hardcodes localhost, reads a local PID file
+and shells out to `lsof`. The command you'd reach for when something breaks is unavailable
+here. Unavailable too: `start`, `stop`, `restart`, `snapshot`, `reembed`, `rem`, `bridge`.
+
+**Fabric's own cluster topology is invisible.** `fleet verify` sweeps *Flair's* federation
+peer table, not Harper's cluster nodes — `cluster_status` is harper-pro-only and absent
+from the OSS `harper` build. **`0 peers known` means "0 on file", never "0 exist."**
+
+**There is no disk or quota telemetry.** `flair status` reports usage for two directories:
+no free space, no total, no quota, no warning threshold, walk capped at six levels, no
+per-component size. `system_information` is never called — an instance can hit its quota
+with nothing saying so. The one indirect signal is a migration halting for space.
+
+> **On `get_components`.** A source comment claims Harper excludes `node_modules`
+> server-side. That is uncited and unverified, so this guide doesn't rely on it — and it
+> wouldn't matter: Flair calls it only as a post-failure convergence oracle and discards
+> the `size` field. Component disk usage is invisible structurally.
+
+### Known hazard: unbounded npm cache
+
+**Open — [flair#886](https://github.com/tpsdev-ai/flair/issues/886).** Every deploy runs a
+server-side `npm install` using the node's default cache. npm never evicts it, so it
+grows until it fills the quota.
+
+No in-product mitigation: no cache flag, no alternate location, no cleanup. One install
+per deploy bounds the *rate*, not the total. Harper consults `install_command` only when
+`node_modules` is absent and `deploy_component` has no force-reinstall option, so the
+obvious fix isn't available — and clearing the cache needs node access this shape
+doesn't give you.
+
+### Backup and rollback
+
+`flair snapshot` is local-only. Back up before every upgrade — `flair backup` takes
+**`--url`**, not `--target`:
+
+```bash
+flair backup --url https://<cluster>.<org>.harperfabric.com \
+  --admin-pass-file <path> --output ./flair-backup.json
+
+# --check shows the version diff and plan without deploying
+flair upgrade --target https://<cluster>.<org>.harperfabric.com --check
+```
+
+Full path: [upgrade.md](upgrade.md#upgrading-a-fabric-deployed-instance).
+
+---
+
+## See also
+
+- [deployment.md](deployment.md) — the standalone local shape
+- [upgrade.md](upgrade.md#upgrading-a-fabric-deployed-instance) — Fabric upgrades, fleet verify
+- [federation.md](federation.md) — pairing, sync driver, conflict resolution
+- [spoke-bringup.md](spoke-bringup.md) · [auth.md](auth.md) · [secrets-and-keys.md](secrets-and-keys.md)

package/docs/deployment.md

@@ -116,6 +116,11 @@ Note: embeddings run on CPU in Docker (no Metal acceleration). Performance is ac
 
 Deploying to a Harper Fabric cluster is a different mechanism from the installs above — `flair deploy` pushes Flair as a cluster component instead of `npm install -g`. To upgrade an already-deployed Fabric instance in place, use `FABRIC_USER=<admin> FABRIC_PASSWORD=<pass> flair upgrade --target <fabric-url>` (or `--fabric-password-file <path>` in place of the env var), not the local upgrade path. Inline `--fabric-user`/`--fabric-password` flags also work but are discouraged — both leak to shell history and `ps`. See [`docs/upgrade.md` — Upgrading a Fabric-deployed instance](upgrade.md#upgrading-a-fabric-deployed-instance) for the full walkthrough, including the automatic post-deploy fleet-convergence sweep.
 
+For the hosted shape end to end — when to choose it, ports and auth against a managed
+Fabric endpoint, pairing local spokes to a hosted hub, and what you can and cannot
+observe without a shell on the node — see
+[`docs/deploying-on-fabric.md`](deploying-on-fabric.md).
+
 ---
 
 ## Remote Access