@tpsdev-ai/flair 0.52.0 → 0.54.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/build-info.json +3 -3
- package/dist/cli.js +1914 -15328
- package/dist/commands/agent.js +453 -0
- package/dist/commands/attention.js +121 -0
- package/dist/commands/backup.js +115 -0
- package/dist/commands/bootstrap.js +91 -0
- package/dist/commands/bridge.js +608 -0
- package/dist/commands/deploy.js +180 -0
- package/dist/commands/doctor.js +1654 -0
- package/dist/commands/export.js +110 -0
- package/dist/commands/federation.js +1575 -0
- package/dist/commands/fleet.js +73 -0
- package/dist/commands/grant.js +109 -0
- package/dist/commands/hook.js +193 -0
- package/dist/commands/idp.js +193 -0
- package/dist/commands/import.js +134 -0
- package/dist/commands/init.js +1203 -0
- package/dist/commands/inspect.js +45 -0
- package/dist/commands/keys.js +187 -0
- package/dist/commands/mcp.js +707 -0
- package/dist/commands/memory.js +501 -0
- package/dist/commands/migrate-harness-memory.js +270 -0
- package/dist/commands/orgevent.js +138 -0
- package/dist/commands/presence.js +76 -0
- package/dist/commands/principal.js +338 -0
- package/dist/commands/quality.js +1164 -0
- package/dist/commands/reembed.js +296 -0
- package/dist/commands/relationship.js +76 -0
- package/dist/commands/rem.js +1048 -0
- package/dist/commands/restore.js +130 -0
- package/dist/commands/search.js +244 -0
- package/dist/commands/service.js +315 -0
- package/dist/commands/session.js +184 -0
- package/dist/commands/soul.js +155 -0
- package/dist/commands/status.js +914 -0
- package/dist/commands/test.js +93 -0
- package/dist/commands/uninstall.js +143 -0
- package/dist/commands/upgrade.js +1592 -0
- package/dist/commands/workspace.js +114 -0
- package/dist/deploy.js +24 -0
- package/dist/fabric-npm-install.js +87 -0
- package/dist/federation-verify.js +498 -0
- package/dist/fleet-verify.js +144 -21
- package/dist/install/clients.js +167 -0
- package/dist/lib/auth-resolve.js +76 -1
- package/dist/lib/daemon-liveness.js +131 -2
- package/dist/lib/doctor-config-path.js +61 -0
- package/dist/lib/doctor-federation-driver.js +189 -0
- package/dist/lib/doctor-run.js +40 -0
- package/dist/lib/entity-vocab-cli.js +3 -3
- package/dist/lib/federation-pair-identity.js +47 -0
- package/dist/lib/launchd-repair.js +199 -0
- package/dist/lib/ops-api-bind.js +115 -0
- package/dist/lib/owned-pins.js +219 -0
- package/dist/lib/stabilize-mqtt-network.js +123 -0
- package/dist/lib/uninstall-purge.js +218 -0
- package/dist/rem/restore.js +8 -10
- package/dist/resources/AgentReadPosition.js +74 -0
- package/dist/resources/Federation.js +8 -2
- package/dist/resources/Memory.js +4 -3
- package/dist/resources/MemoryBootstrap.js +41 -25
- package/dist/resources/MemoryCandidate.js +5 -6
- package/dist/resources/OrgEventCatchup.js +126 -47
- package/dist/resources/agent-read-position-lib.js +83 -0
- package/dist/resources/agent-read-position.js +120 -0
- package/dist/resources/embeddings-boot.js +32 -0
- package/dist/resources/federation-peer-liveness.js +73 -0
- package/dist/resources/health.js +68 -19
- package/dist/resources/mcp-tools.js +43 -279
- package/dist/resources/memory-visibility.js +3 -3
- package/dist/resources/migration-boot.js +59 -18
- package/dist/resources/migrations/embedding-stamp.js +20 -1
- package/dist/resources/migrations/recheck.js +43 -0
- package/dist/resources/migrations/runner.js +6 -1
- package/dist/resources/migrations/stamp-outstanding.js +171 -0
- package/dist/resources/migrations/visibility-backfill.js +2 -2
- package/dist/resources/org-event-catchup-lib.js +47 -0
- package/dist/resources/record-owner-guard.js +1 -0
- package/dist/stamp-migration-verify.js +163 -0
- package/dist/stamp-outstanding.js +144 -0
- package/docs/api-reference.md +4 -2
- package/docs/deploying-on-fabric.md +11 -10
- package/docs/deployment.md +3 -1
- package/docs/federation.md +19 -0
- package/docs/hosted-on-fabric.md +3 -3
- package/docs/quickstart.md +2 -1
- package/docs/releasing.md +15 -7
- package/docs/spoke-bringup.md +10 -5
- package/docs/standalone-local.md +3 -1
- package/docs/upgrade.md +25 -6
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/LICENSE +19 -0
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/README.md +22 -0
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/dist/index.d.ts +70 -0
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/dist/index.js +665 -0
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/package.json +46 -0
- package/package.json +9 -4
- package/schemas/agent.graphql +15 -0
- package/templates/launchd/start-flair-with-admin-pass.sh +73 -0
|
@@ -0,0 +1,1575 @@
|
|
|
1
|
+
import nacl from "tweetnacl";
|
|
2
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { randomBytes } from "node:crypto";
|
|
4
|
+
import { keystore, keyPath as keystoreKeyPath } from "../keystore.js";
|
|
5
|
+
import * as render from "../render.js";
|
|
6
|
+
import { runFederationVerify } from "../federation-verify.js";
|
|
7
|
+
import { resolveHubPeerIdentity } from "../lib/federation-pair-identity.js";
|
|
8
|
+
import { defaultAdminPassPath, isLocalBase, resolveAdminUser, } from "../lib/auth-resolve.js";
|
|
9
|
+
import { DEFAULT_INTERVAL_SECONDS as FEDERATION_SYNC_DEFAULT_INTERVAL } from "../federation/scheduler.js";
|
|
10
|
+
let cli;
|
|
11
|
+
/** Bind shared CLI helpers. cli.ts calls this immediately before register(program). */
|
|
12
|
+
export function bindCli(fns) {
|
|
13
|
+
cli = fns;
|
|
14
|
+
}
|
|
15
|
+
function api(...args) {
|
|
16
|
+
return cli.api(...args);
|
|
17
|
+
}
|
|
18
|
+
function resolveTarget(opts) {
|
|
19
|
+
return cli.resolveTarget(opts);
|
|
20
|
+
}
|
|
21
|
+
function resolveBaseUrl(opts) {
|
|
22
|
+
return cli.resolveBaseUrl(opts);
|
|
23
|
+
}
|
|
24
|
+
function resolveEffectiveOpsUrl(opts) {
|
|
25
|
+
return cli.resolveEffectiveOpsUrl(opts);
|
|
26
|
+
}
|
|
27
|
+
function resolveOpsPort(opts) {
|
|
28
|
+
return cli.resolveOpsPort(opts);
|
|
29
|
+
}
|
|
30
|
+
function applyAdminPassFile(opts) {
|
|
31
|
+
cli.applyAdminPassFile(opts);
|
|
32
|
+
}
|
|
33
|
+
function addSharedCredentialOptions(cmd) {
|
|
34
|
+
return cli.addSharedCredentialOptions(cmd);
|
|
35
|
+
}
|
|
36
|
+
function addSharedIdentityOption(cmd) {
|
|
37
|
+
return cli.addSharedIdentityOption(cmd);
|
|
38
|
+
}
|
|
39
|
+
function shouldShowInlineSecretWarning(optValue, fromEnv, secretFlagNames, flagName) {
|
|
40
|
+
return cli.shouldShowInlineSecretWarning(optValue, fromEnv, secretFlagNames, flagName);
|
|
41
|
+
}
|
|
42
|
+
// Federation crypto helpers — inlined to avoid cross-boundary imports from
|
|
43
|
+
// src/ into resources/, which don't survive npm packaging (see also
|
|
44
|
+
// resources/federation-crypto.ts; the two must stay in sync).
|
|
45
|
+
function sortKeys(val) {
|
|
46
|
+
if (val === null || val === undefined || typeof val !== "object")
|
|
47
|
+
return val;
|
|
48
|
+
if (Array.isArray(val))
|
|
49
|
+
return val.map(sortKeys);
|
|
50
|
+
const sorted = {};
|
|
51
|
+
for (const key of Object.keys(val).sort()) {
|
|
52
|
+
sorted[key] = sortKeys(val[key]);
|
|
53
|
+
}
|
|
54
|
+
return sorted;
|
|
55
|
+
}
|
|
56
|
+
function canonicalize(obj) {
|
|
57
|
+
return JSON.stringify(sortKeys(obj));
|
|
58
|
+
}
|
|
59
|
+
function signBody(body, secretKey) {
|
|
60
|
+
const message = new TextEncoder().encode(canonicalize(body));
|
|
61
|
+
const sig = nacl.sign.detached(message, secretKey);
|
|
62
|
+
return Buffer.from(sig).toString("base64url");
|
|
63
|
+
}
|
|
64
|
+
// Per-record principalId (federation-edge-hardening slice 3a / flair#1416).
|
|
65
|
+
// Sourced from the write-time provenance stamp (memory-provenance slice 1,
|
|
66
|
+
// Memory.ts's buildProvenance) when present. `provenance` is persisted as
|
|
67
|
+
// a JSON STRING (not an object), so it must be parsed — a raw
|
|
68
|
+
// `row.provenance?.verified?.agentId` would silently always be undefined.
|
|
69
|
+
// Soul/Agent/Relationship rows never carry a provenance stamp today, so
|
|
70
|
+
// this is a no-op for them (those tables are not principal-owning).
|
|
71
|
+
//
|
|
72
|
+
// As of v:2 this value is IN the signed payload. The receiver validates
|
|
73
|
+
// it against data.agentId for Memory (PRINCIPAL_OWNING_TABLES); it is
|
|
74
|
+
// no longer informational-only. Credential.principalId is an unrelated
|
|
75
|
+
// owner field — do not grep that path when changing this one.
|
|
76
|
+
//
|
|
77
|
+
// S2 COMMENT-PIN (Kern P2-5, flair#1521): Message is principal-owning by `from`
|
|
78
|
+
// (PRINCIPAL_OWNER_FIELD in federation-classify.ts) but its rows carry NO
|
|
79
|
+
// `provenance` stamp, so principalIdFromRow returns undefined for them → a v:2
|
|
80
|
+
// Message push would omit `principalId` → the receiver's v≥2
|
|
81
|
+
// checkPrincipalEntitlement skips EVERY Message as `principal_mismatch` (a
|
|
82
|
+
// 100%-skip sync, not a migration). S1 does not push Message (receive-only —
|
|
83
|
+
// the spoke push list below is a separate hardcoded set), so this is inert
|
|
84
|
+
// today; the S2 pusher MUST stamp `principalId = row.from` for Message here
|
|
85
|
+
// (a per-table owner-aware derivation, not `provenance.verified.agentId`).
|
|
86
|
+
function principalIdFromRow(row) {
|
|
87
|
+
if (typeof row?.provenance !== "string" || row.provenance.length === 0)
|
|
88
|
+
return undefined;
|
|
89
|
+
try {
|
|
90
|
+
return JSON.parse(row.provenance)?.verified?.agentId ?? undefined;
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// Federation push private-visibility filter — inlined for the SAME reason as
|
|
97
|
+
// the crypto helpers above (see comment there; also resources/memory-
|
|
98
|
+
// visibility.ts, the canonical definition; the two must stay in sync).
|
|
99
|
+
//
|
|
100
|
+
// federation-edge-hardening slice 2 (the office-visibility read leak: one rule, one place): the
|
|
101
|
+
// push side of federation sync (runFederationSyncOnce below) must exclude
|
|
102
|
+
// `private` Memory rows from what gets sent to peers, using the EXACT same
|
|
103
|
+
// "not private" semantics as resources/memory-read-scope.ts's resolveReadScope()
|
|
104
|
+
// — a record with NO visibility field (legacy, pre-dates the field) is NOT
|
|
105
|
+
// private and must keep syncing exactly as before. Only `visibility ===
|
|
106
|
+
// "private"` is excluded; null/undefined/"shared"/anything else is included.
|
|
107
|
+
const FEDERATION_PRIVATE_VISIBILITY = "private";
|
|
108
|
+
function isFederationPrivateVisibility(visibility) {
|
|
109
|
+
return visibility === FEDERATION_PRIVATE_VISIBILITY;
|
|
110
|
+
}
|
|
111
|
+
async function loadInstanceSecretKey(instanceId, opts) {
|
|
112
|
+
// Try keystore first
|
|
113
|
+
const seed = keystore.getPrivateKeySeed(instanceId);
|
|
114
|
+
if (seed) {
|
|
115
|
+
return nacl.sign.keyPair.fromSeed(seed).secretKey;
|
|
116
|
+
}
|
|
117
|
+
// Fallback: check DB for legacy _keySeed
|
|
118
|
+
const opsPort = resolveOpsPort(opts);
|
|
119
|
+
const adminPass = opts.adminPass ?? process.env.FLAIR_ADMIN_PASS ?? "";
|
|
120
|
+
const auth = `Basic ${Buffer.from(`${resolveAdminUser(opts.adminUser)}:${adminPass}`).toString("base64")}`;
|
|
121
|
+
const res = await fetch(`http://127.0.0.1:${opsPort}/`, {
|
|
122
|
+
method: "POST",
|
|
123
|
+
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
124
|
+
body: JSON.stringify({ operation: "search_by_value", schema: "flair", table: "Instance", search_attribute: "id", search_type: "equals", search_value: instanceId, get_attributes: ["*"] }),
|
|
125
|
+
});
|
|
126
|
+
if (res.ok) {
|
|
127
|
+
const rows = await res.json();
|
|
128
|
+
if (rows[0]?._keySeed) {
|
|
129
|
+
const seedFromDb = Buffer.from(rows[0]._keySeed, "base64url");
|
|
130
|
+
// Migrate to keystore
|
|
131
|
+
keystore.setPrivateKeySeed(instanceId, new Uint8Array(seedFromDb));
|
|
132
|
+
return nacl.sign.keyPair.fromSeed(new Uint8Array(seedFromDb)).secretKey;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
// flair#1233: the old advice here — "Re-run 'flair federation status' to
|
|
136
|
+
// regenerate" — was impossible: the server's create branch only fires when
|
|
137
|
+
// NO Instance row exists, so a re-run can never regenerate a key for an
|
|
138
|
+
// existing identity. Name the real remedy instead.
|
|
139
|
+
throw new Error(`No usable private key for instance ${instanceId}. Expected keystore file: ${keystoreKeyPath(instanceId)} ` +
|
|
140
|
+
`(no legacy _keySeed in the Instance table either). Restore that key file from a backup of ~/.flair/keys ` +
|
|
141
|
+
`(and FLAIR_KEY_PASSPHRASE, if one was set when it was written), or re-key this instance: delete its ` +
|
|
142
|
+
`Instance row and re-pair to mint a fresh identity.`);
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Sign a request body and return a new body with the signature field added.
|
|
146
|
+
*/
|
|
147
|
+
export function signRequestBody(body, secretKey) {
|
|
148
|
+
// Fresh signing with anti-replay: embeds _ts and _nonce before signing.
|
|
149
|
+
// Equivalent to federation-crypto.ts signBodyFresh — duplicated here because
|
|
150
|
+
// the CLI module has its own local signBody for dependency isolation.
|
|
151
|
+
const freshBody = {
|
|
152
|
+
...body,
|
|
153
|
+
_ts: Date.now(),
|
|
154
|
+
_nonce: Buffer.from(nacl.randomBytes(16)).toString("base64url"),
|
|
155
|
+
};
|
|
156
|
+
const sig = signBody(freshBody, secretKey);
|
|
157
|
+
return { ...freshBody, signature: sig };
|
|
158
|
+
}
|
|
159
|
+
// Alias: signBodyFresh for clarity at call sites
|
|
160
|
+
const signBodyFresh = signRequestBody;
|
|
161
|
+
/**
|
|
162
|
+
* The most recent CONTACT with any peer (max of peer.lastSyncAt), or null.
|
|
163
|
+
*
|
|
164
|
+
* Contact, not merge: a sync that reaches the peer and legitimately has
|
|
165
|
+
* nothing to send still proves the driver ran. This is half of what lets
|
|
166
|
+
* `federation status` tell "nothing is driving sync" apart from "sync is
|
|
167
|
+
* running, the peer is unreachable" (flair#922) — the other half is whether
|
|
168
|
+
* the service manager has a driver loaded.
|
|
169
|
+
*
|
|
170
|
+
* Returns null on ANY failure. A driver verdict is a diagnostic aid; it must
|
|
171
|
+
* never be the reason `federation status` fails.
|
|
172
|
+
*/
|
|
173
|
+
async function latestPeerContact(opts) {
|
|
174
|
+
try {
|
|
175
|
+
const target = resolveTarget(opts);
|
|
176
|
+
const baseUrl = target ? target.replace(/\/$/, "") : undefined;
|
|
177
|
+
const { peers } = await api("GET", "/FederationPeers", undefined, baseUrl ? { baseUrl } : undefined);
|
|
178
|
+
let best = null;
|
|
179
|
+
for (const p of peers ?? []) {
|
|
180
|
+
if (!p?.lastSyncAt)
|
|
181
|
+
continue;
|
|
182
|
+
const t = Date.parse(p.lastSyncAt);
|
|
183
|
+
if (Number.isFinite(t) && (best === null || t > best))
|
|
184
|
+
best = t;
|
|
185
|
+
}
|
|
186
|
+
return best === null ? null : new Date(best).toISOString();
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* True when the CLI is pointed at the instance running on THIS machine.
|
|
194
|
+
*
|
|
195
|
+
* The scheduler check is inherently local — launchctl/systemctl only know
|
|
196
|
+
* about jobs on the host the CLI is running on. Reporting "no driver" while
|
|
197
|
+
* `--target` points at someone else's hub would be a confident claim about a
|
|
198
|
+
* machine we cannot see, so the driver block is suppressed for remote targets.
|
|
199
|
+
*/
|
|
200
|
+
function driverCheckAppliesTo(opts) {
|
|
201
|
+
const target = resolveTarget(opts);
|
|
202
|
+
return !target || isLocalBase(target.replace(/\/$/, ""));
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* flair#1108: a bare undici/Node "fetch failed" names neither the URL
|
|
206
|
+
* that was probed nor the knob that would change it. These helpers are
|
|
207
|
+
* the operator-facing sentence and the setting that produced (or would
|
|
208
|
+
* change) that URL. Pure so the contract can be unit-tested without
|
|
209
|
+
* driving process.exit.
|
|
210
|
+
*/
|
|
211
|
+
export function federationStatusUrlSetting(opts) {
|
|
212
|
+
if (opts.target)
|
|
213
|
+
return "--target";
|
|
214
|
+
if (process.env.FLAIR_TARGET)
|
|
215
|
+
return "FLAIR_TARGET";
|
|
216
|
+
if (process.env.FLAIR_URL)
|
|
217
|
+
return "FLAIR_URL";
|
|
218
|
+
if (opts.port !== undefined && opts.port !== null && String(opts.port) !== "")
|
|
219
|
+
return "--port";
|
|
220
|
+
return "FLAIR_URL or --port";
|
|
221
|
+
}
|
|
222
|
+
export function describeFederationStatusFetchFailed(url, setting) {
|
|
223
|
+
return `fetch failed against ${url} (set ${setting})`;
|
|
224
|
+
}
|
|
225
|
+
/** True for a connect-level failure (no HTTP status): Node's undici
|
|
226
|
+
* `TypeError: fetch failed`, Bun's `Unable to connect…`, or a cause
|
|
227
|
+
* carrying a connect/DNS errno. Auth and HTTP errors stay out. */
|
|
228
|
+
export function isFederationStatusConnectFailure(err) {
|
|
229
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
230
|
+
if (/\bfetch failed\b/i.test(msg))
|
|
231
|
+
return true;
|
|
232
|
+
if (/unable to connect/i.test(msg))
|
|
233
|
+
return true;
|
|
234
|
+
const cause = err instanceof Error ? err.cause : undefined;
|
|
235
|
+
const code = cause && typeof cause === "object" && cause && "code" in cause
|
|
236
|
+
? String(cause.code)
|
|
237
|
+
: "";
|
|
238
|
+
return /^(ECONNREFUSED|ENOTFOUND|ECONNRESET|ETIMEDOUT|EAI_AGAIN|EHOSTUNREACH)$/.test(code);
|
|
239
|
+
}
|
|
240
|
+
export function rewriteFederationStatusFetchFailed(err, url, setting) {
|
|
241
|
+
if (!isFederationStatusConnectFailure(err))
|
|
242
|
+
return err;
|
|
243
|
+
const next = new Error(describeFederationStatusFetchFailed(url, setting));
|
|
244
|
+
if (err && typeof err === "object" && "status" in err) {
|
|
245
|
+
next.status = err.status;
|
|
246
|
+
}
|
|
247
|
+
return next;
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Auth-shaped vs connect-level for `federation status`. A rewritten
|
|
251
|
+
* fetch-failed sentence embeds the probed URL; that URL can contain a
|
|
252
|
+
* whole-token `401` (e.g. `--port 401`). The old `message.includes("401")`
|
|
253
|
+
* check then printed the credential remedy and hid the URL+setting this
|
|
254
|
+
* change exists to surface (Bugbot on flair#1108).
|
|
255
|
+
*/
|
|
256
|
+
export function isFederationStatusAuthFailure(err) {
|
|
257
|
+
if (!err)
|
|
258
|
+
return false;
|
|
259
|
+
if (isFederationStatusConnectFailure(err))
|
|
260
|
+
return false;
|
|
261
|
+
if (typeof err === "object" && "status" in err) {
|
|
262
|
+
const status = err.status;
|
|
263
|
+
if (status === 401 || status === 403)
|
|
264
|
+
return true;
|
|
265
|
+
}
|
|
266
|
+
const m = err instanceof Error
|
|
267
|
+
? err.message
|
|
268
|
+
: String(typeof err === "object" && err && "message" in err
|
|
269
|
+
? err.message ?? err
|
|
270
|
+
: err);
|
|
271
|
+
return m.includes("missing_or_invalid_authorization") || /(?:^|\D)401(?:\D|$)/.test(m);
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Whether to print the "set one of: FLAIR_AGENT_ID / FLAIR_ADMIN_PASS /
|
|
275
|
+
* FLAIR_TOKEN" block. Narrower than `isFederationStatusAuthFailure`: a
|
|
276
|
+
* 403 with credentials already sent (wrong password) is fatal, but the
|
|
277
|
+
* server's own body is the honest message — the credential-list remedy
|
|
278
|
+
* is for missing/invalid auth (401), not a rejected password (flair#634).
|
|
279
|
+
*/
|
|
280
|
+
export function isFederationStatusAuthRemedy(err) {
|
|
281
|
+
if (!err || isFederationStatusConnectFailure(err))
|
|
282
|
+
return false;
|
|
283
|
+
if (typeof err === "object" && "status" in err && err.status === 401) {
|
|
284
|
+
return true;
|
|
285
|
+
}
|
|
286
|
+
const m = err instanceof Error
|
|
287
|
+
? err.message
|
|
288
|
+
: String(typeof err === "object" && err && "message" in err
|
|
289
|
+
? err.message ?? err
|
|
290
|
+
: err);
|
|
291
|
+
return m.includes("missing_or_invalid_authorization") || /(?:^|\D)401(?:\D|$)/.test(m);
|
|
292
|
+
}
|
|
293
|
+
/** Parse a JSON triple file for --token-from.
|
|
294
|
+
* Expected shape: { "token": "...", "user": "pair-bootstrap-<id>", "password": "...", "expiresAt": "<ISO>" }
|
|
295
|
+
* Returns the triple on success. Validation failures exit(1).
|
|
296
|
+
*/
|
|
297
|
+
export function parseTokenFromFile(filePath) {
|
|
298
|
+
let raw;
|
|
299
|
+
if (filePath === "-") {
|
|
300
|
+
raw = readFileSync("/dev/stdin", "utf-8");
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
if (!existsSync(filePath)) {
|
|
304
|
+
console.error(`Error: --token-from file not found: ${filePath}`);
|
|
305
|
+
process.exit(1);
|
|
306
|
+
}
|
|
307
|
+
raw = readFileSync(filePath, "utf-8");
|
|
308
|
+
}
|
|
309
|
+
let parsed;
|
|
310
|
+
try {
|
|
311
|
+
parsed = JSON.parse(raw);
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
console.error(`Error: --token-from file is not valid JSON: ${filePath}`);
|
|
315
|
+
process.exit(1);
|
|
316
|
+
}
|
|
317
|
+
// Validate all four fields present and non-empty
|
|
318
|
+
const required = ["token", "user", "password", "expiresAt"];
|
|
319
|
+
for (const field of required) {
|
|
320
|
+
if (!parsed[field] || typeof parsed[field] !== "string" || parsed[field].trim() === "") {
|
|
321
|
+
console.error(`Error: --token-from JSON is missing or has empty required field "${field}"`);
|
|
322
|
+
process.exit(1);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
// Validate expiresAt is a parseable date and is in the future
|
|
326
|
+
const expiry = new Date(parsed.expiresAt);
|
|
327
|
+
if (isNaN(expiry.getTime())) {
|
|
328
|
+
console.error(`Error: --token-from JSON has invalid expiresAt date: "${parsed.expiresAt}"`);
|
|
329
|
+
process.exit(1);
|
|
330
|
+
}
|
|
331
|
+
const now = new Date();
|
|
332
|
+
if (expiry <= now) {
|
|
333
|
+
console.error(`Error: --token-from JSON has expired token (expiresAt: ${parsed.expiresAt})`);
|
|
334
|
+
process.exit(1);
|
|
335
|
+
}
|
|
336
|
+
const fiveMin = 5 * 60 * 1000;
|
|
337
|
+
if (expiry.getTime() - now.getTime() < fiveMin) {
|
|
338
|
+
console.error(`warning: pairing token expires in less than 5 minutes (expiresAt: ${parsed.expiresAt})`);
|
|
339
|
+
}
|
|
340
|
+
return {
|
|
341
|
+
token: parsed.token.trim(),
|
|
342
|
+
user: parsed.user.trim(),
|
|
343
|
+
password: parsed.password.trim(),
|
|
344
|
+
expiresAt: parsed.expiresAt.trim(),
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
export async function runFederationSyncOnce(opts) {
|
|
348
|
+
const target = resolveTarget(opts);
|
|
349
|
+
const baseUrl = target ? target.replace(/\/$/, "") : undefined;
|
|
350
|
+
// Same allowAdmin listing as federation verify — flag/file admin pass
|
|
351
|
+
// must reach GET /FederationPeers and /FederationInstance, not just the
|
|
352
|
+
// ops-API Basic header used later in this function.
|
|
353
|
+
const apiOpts = (baseUrl || opts.adminPass || opts.adminUser)
|
|
354
|
+
? {
|
|
355
|
+
...(baseUrl ? { baseUrl } : {}),
|
|
356
|
+
...(opts.adminPass ? { explicitAdminPass: opts.adminPass } : {}),
|
|
357
|
+
...(opts.adminUser ? { adminUser: opts.adminUser } : {}),
|
|
358
|
+
}
|
|
359
|
+
: undefined;
|
|
360
|
+
let totalMerged = 0;
|
|
361
|
+
let totalSkipped = 0;
|
|
362
|
+
try {
|
|
363
|
+
const { peers } = await api("GET", "/FederationPeers", undefined, apiOpts);
|
|
364
|
+
const hub = peers.find((p) => p.role === "hub" && p.status !== "revoked");
|
|
365
|
+
if (!hub) {
|
|
366
|
+
return { pushed: 0, skipped: 0, error: new Error("No hub peer configured. Use 'flair federation pair' first.") };
|
|
367
|
+
}
|
|
368
|
+
console.log(`Syncing to hub: ${hub.id}...`);
|
|
369
|
+
const since = hub.lastSyncAt ?? new Date(0).toISOString();
|
|
370
|
+
// Capture sync start time BEFORE we query records. We advance the local
|
|
371
|
+
// hub peer's lastSyncAt to this value after success so the next poll's
|
|
372
|
+
// `since` cursor moves forward — fixes task #146 (federation peer
|
|
373
|
+
// .lastSyncAt update bug). Records updated DURING this sync will have
|
|
374
|
+
// updatedAt > syncStartedAt and be picked up next cycle, not missed.
|
|
375
|
+
const syncStartedAt = new Date().toISOString();
|
|
376
|
+
const opsEndpoint = resolveEffectiveOpsUrl(opts) ?? `http://127.0.0.1:${resolveOpsPort(opts)}`;
|
|
377
|
+
const adminPass = opts.adminPass ?? process.env.FLAIR_ADMIN_PASS ?? "";
|
|
378
|
+
const auth = `Basic ${Buffer.from(`${resolveAdminUser(opts.adminUser)}:${adminPass}`).toString("base64")}`;
|
|
379
|
+
const tables = ["Memory", "Soul", "Agent", "Relationship"];
|
|
380
|
+
const instance = await api("GET", "/FederationInstance", undefined, apiOpts);
|
|
381
|
+
const hubUrl = hub.endpoint ?? hub.id;
|
|
382
|
+
// ── Batching constants ──────────────────────────────────────────────
|
|
383
|
+
// 2MB JSON budget (server cap is 10MB; 2MB leaves headroom for headers
|
|
384
|
+
// and signature metadata) + 50 records max per batch. The hub merge itself
|
|
385
|
+
// is fast (~1.7s/50 records, per its SyncLog), but the Fabric ingress was
|
|
386
|
+
// observed to intermittently stall on larger POSTs — a 50-record batch hung
|
|
387
|
+
// ~2 min while the same records split into 2×25 went through immediately.
|
|
388
|
+
// 50 keeps batches in the reliable range, and sendBatch's adaptive split
|
|
389
|
+
// recovers if a stretch still stalls.
|
|
390
|
+
const BUDGET_BYTES = 2_000_000;
|
|
391
|
+
const BUDGET_RECORDS = 50;
|
|
392
|
+
// ── sendBatch helper ────────────────────────────────────────────────
|
|
393
|
+
// Secret key is lazy-loaded: only needed when there are records to send.
|
|
394
|
+
// Loading earlier would cause a spurious error when SQL queries fail
|
|
395
|
+
// (e.g. 401) before we know we have records.
|
|
396
|
+
let secretKey;
|
|
397
|
+
// Statuses the Fabric ingress returns when a batch POST didn't complete in
|
|
398
|
+
// time (408) or was too large (413), plus the transient gateway 5xx family.
|
|
399
|
+
// Splitting the batch and retrying smaller chunks lets the sync converge
|
|
400
|
+
// instead of aborting the whole run.
|
|
401
|
+
const TIMEOUT_STATUSES = new Set([408, 413, 502, 503, 504]);
|
|
402
|
+
// Per-batch wall-clock cap. Without it a stalled connection to the Fabric
|
|
403
|
+
// ingress hangs the whole sync until the *gateway's* timeout fires (~2 min
|
|
404
|
+
// observed), which is what stranded the re-pair. A 45s cap is generous —
|
|
405
|
+
// a healthy 50-record batch merges in <2s — so a trip means a real stall,
|
|
406
|
+
// and we split-and-retry rather than wait it out.
|
|
407
|
+
const BATCH_TIMEOUT_MS = 45_000;
|
|
408
|
+
async function sendBatch(batch) {
|
|
409
|
+
if (!secretKey)
|
|
410
|
+
secretKey = await loadInstanceSecretKey(instance.id, opts);
|
|
411
|
+
const syncBody = { instanceId: instance.id, records: batch, lamportClock: Date.now() };
|
|
412
|
+
const signedSyncBody = signBodyFresh(syncBody, secretKey);
|
|
413
|
+
// Halve and retry down to a single record. Covers both an explicit
|
|
414
|
+
// timeout status AND a client-side abort (stalled socket). The hub
|
|
415
|
+
// merges idempotently (put-by-id), so retried records are safe.
|
|
416
|
+
const splittable = (status) => batch.length > 1 && (status === null || TIMEOUT_STATUSES.has(status));
|
|
417
|
+
const split = async () => {
|
|
418
|
+
const mid = Math.floor(batch.length / 2);
|
|
419
|
+
const left = await sendBatch(batch.slice(0, mid));
|
|
420
|
+
const right = await sendBatch(batch.slice(mid));
|
|
421
|
+
return { merged: left.merged + right.merged, skipped: left.skipped + right.skipped };
|
|
422
|
+
};
|
|
423
|
+
let syncRes;
|
|
424
|
+
try {
|
|
425
|
+
syncRes = await fetch(`${hubUrl}/FederationSync`, {
|
|
426
|
+
method: "POST",
|
|
427
|
+
headers: { "Content-Type": "application/json" },
|
|
428
|
+
body: JSON.stringify(signedSyncBody),
|
|
429
|
+
signal: AbortSignal.timeout(BATCH_TIMEOUT_MS),
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
catch (err) {
|
|
433
|
+
// Timeout/abort or network drop — no status. Split if we can.
|
|
434
|
+
if (splittable(null))
|
|
435
|
+
return await split();
|
|
436
|
+
throw new Error(`Sync batch (${batch.length} record${batch.length === 1 ? "" : "s"}) failed: ${err?.message ?? err}`);
|
|
437
|
+
}
|
|
438
|
+
if (!syncRes.ok) {
|
|
439
|
+
if (splittable(syncRes.status))
|
|
440
|
+
return await split();
|
|
441
|
+
const text = await syncRes.text().catch(() => "");
|
|
442
|
+
throw new Error(`Sync batch failed: ${syncRes.status} ${text}`);
|
|
443
|
+
}
|
|
444
|
+
return await syncRes.json();
|
|
445
|
+
}
|
|
446
|
+
let totalBatches = 0;
|
|
447
|
+
// Memory rows that passed the since-cursor filter but were excluded as
|
|
448
|
+
// private. Used only so the quiet path can distinguish "nothing since
|
|
449
|
+
// the cursor" from "found rows, all withheld" (flair#1232). Does not
|
|
450
|
+
// change what gets pushed — private still never leaves the instance.
|
|
451
|
+
let privateHeldBack = 0;
|
|
452
|
+
for (const table of tables) {
|
|
453
|
+
let rows = [];
|
|
454
|
+
for (const query of [
|
|
455
|
+
{ search_attribute: "updatedAt", search_type: "greater_than", search_value: since },
|
|
456
|
+
// Rows with null updatedAt (legacy direct-insert rows) use createdAt.
|
|
457
|
+
// COALESCE(updatedAt, createdAt) > since → pick up null-updatedAt rows
|
|
458
|
+
// whose createdAt > since. Filtered in JS below.
|
|
459
|
+
{ search_attribute: "updatedAt", search_type: "equals", search_value: null },
|
|
460
|
+
]) {
|
|
461
|
+
let res;
|
|
462
|
+
try {
|
|
463
|
+
res = await fetch(`${opsEndpoint}/`, {
|
|
464
|
+
method: "POST",
|
|
465
|
+
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
466
|
+
body: JSON.stringify({ operation: "search_by_conditions", schema: "flair", table, operator: "and", conditions: [query], get_attributes: ["*"] }),
|
|
467
|
+
signal: AbortSignal.timeout(15_000),
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
catch (err) {
|
|
471
|
+
return { pushed: totalMerged, skipped: totalSkipped, error: err instanceof Error ? err : new Error(String(err)) };
|
|
472
|
+
}
|
|
473
|
+
if (!res.ok) {
|
|
474
|
+
const text = await res.text().catch(() => "");
|
|
475
|
+
return { pushed: totalMerged, skipped: totalSkipped, error: new Error(`SQL query failed (${res.status}): ${text}`) };
|
|
476
|
+
}
|
|
477
|
+
const batch = await res.json();
|
|
478
|
+
// For null-updatedAt rows, use createdAt as the effective timestamp.
|
|
479
|
+
// Skip rows created before the last sync cursor. Only Memory carries
|
|
480
|
+
// a `visibility` field (Soul/Agent/Relationship don't — see
|
|
481
|
+
// schemas/memory.graphql vs agent.graphql), so the private-exclusion
|
|
482
|
+
// filter only applies there; on the other 3 tables `row.visibility`
|
|
483
|
+
// is always undefined, which isFederationPrivateVisibility() treats
|
|
484
|
+
// as non-private (included) — a no-op for them.
|
|
485
|
+
const sinceCursor = batch.filter((r) => r.updatedAt !== null || r.createdAt > since);
|
|
486
|
+
const federable = sinceCursor.filter((r) => table !== "Memory" || !isFederationPrivateVisibility(r.visibility));
|
|
487
|
+
if (table === "Memory")
|
|
488
|
+
privateHeldBack += sinceCursor.length - federable.length;
|
|
489
|
+
rows = rows.concat(federable);
|
|
490
|
+
}
|
|
491
|
+
if (rows.length === 0)
|
|
492
|
+
continue;
|
|
493
|
+
// Records are signed (below) before they're batched, so the secret key
|
|
494
|
+
// is needed here rather than only inside sendBatch. Still deferred
|
|
495
|
+
// until we know THIS table has rows to send — preserves the "don't
|
|
496
|
+
// load the key on a no-op run" property the original lazy load had.
|
|
497
|
+
if (!secretKey)
|
|
498
|
+
secretKey = await loadInstanceSecretKey(instance.id, opts);
|
|
499
|
+
let batch = [];
|
|
500
|
+
let batchBytes = 0;
|
|
501
|
+
for (const row of rows) {
|
|
502
|
+
const updatedAt = row.updatedAt ?? row.createdAt;
|
|
503
|
+
const originatorInstanceId = instance.id;
|
|
504
|
+
// Per-record signature (federation-edge-hardening slice 3a): signed by
|
|
505
|
+
// THIS instance — the originator — over a versioned canonical form, so
|
|
506
|
+
// a receiver (including a hub relaying this record onward to other
|
|
507
|
+
// spokes) can verify authorship independent of who forwarded the
|
|
508
|
+
// batch. Closes the hub-relay forgery hole — see
|
|
509
|
+
// resources/Federation.ts FederationSync.post's verification gate.
|
|
510
|
+
//
|
|
511
|
+
// CONTRACT — must match reconstructRecordVerifyBody
|
|
512
|
+
// (resources/federation-classify.ts) byte-for-byte. canonicalize()
|
|
513
|
+
// sorts keys, so field ORDER doesn't matter, but the field SET and
|
|
514
|
+
// values do. `v` versions the canonical form itself: a v:1
|
|
515
|
+
// signature cannot verify as v:2 (principalId in the field set).
|
|
516
|
+
//
|
|
517
|
+
// v: 2 puts principalId in the signed payload when the row carries
|
|
518
|
+
// a provenance stamp, and puts `v` on the wire so Phase 1
|
|
519
|
+
// receivers (`const v = record.v ?? 1`) don't default these
|
|
520
|
+
// records back to 1. Soul/Agent/Relationship have no stamp and
|
|
521
|
+
// omit principalId; Memory without a stamp also omits it (the
|
|
522
|
+
// receiver then skips Memory as principal_mismatch — absent is
|
|
523
|
+
// not an accept).
|
|
524
|
+
const principalId = principalIdFromRow(row);
|
|
525
|
+
const signedPayload = {
|
|
526
|
+
v: 2,
|
|
527
|
+
table,
|
|
528
|
+
id: row.id,
|
|
529
|
+
data: row,
|
|
530
|
+
updatedAt,
|
|
531
|
+
originatorInstanceId,
|
|
532
|
+
};
|
|
533
|
+
if (principalId)
|
|
534
|
+
signedPayload.principalId = principalId;
|
|
535
|
+
const signature = signBody(signedPayload, secretKey);
|
|
536
|
+
const sr = {
|
|
537
|
+
v: 2,
|
|
538
|
+
table,
|
|
539
|
+
id: row.id,
|
|
540
|
+
data: row,
|
|
541
|
+
updatedAt,
|
|
542
|
+
originatorInstanceId,
|
|
543
|
+
signature,
|
|
544
|
+
};
|
|
545
|
+
if (principalId)
|
|
546
|
+
sr.principalId = principalId;
|
|
547
|
+
const srBytes = JSON.stringify(sr).length;
|
|
548
|
+
if (batch.length >= BUDGET_RECORDS || (batch.length > 0 && batchBytes + srBytes > BUDGET_BYTES)) {
|
|
549
|
+
const result = await sendBatch(batch);
|
|
550
|
+
totalMerged += result.merged;
|
|
551
|
+
totalSkipped += result.skipped;
|
|
552
|
+
totalBatches++;
|
|
553
|
+
batch = [];
|
|
554
|
+
batchBytes = 0;
|
|
555
|
+
}
|
|
556
|
+
batch.push(sr);
|
|
557
|
+
batchBytes += srBytes;
|
|
558
|
+
}
|
|
559
|
+
// Send final partial batch for this table
|
|
560
|
+
if (batch.length > 0) {
|
|
561
|
+
const result = await sendBatch(batch);
|
|
562
|
+
totalMerged += result.merged;
|
|
563
|
+
totalSkipped += result.skipped;
|
|
564
|
+
totalBatches++;
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
// Advance the local hub peer's lastSyncAt cursor. The hub-side
|
|
568
|
+
// FederationSync handler updates ITS view of the spoke peer, but the
|
|
569
|
+
// spoke never updated its own view of the hub — so `since` stayed at
|
|
570
|
+
// whatever value was on the peer record at pair time (often near-epoch),
|
|
571
|
+
// and every poll re-queried `updatedAt > since` and re-sent every
|
|
572
|
+
// memory ever written. The receiver-side contentHash gate in
|
|
573
|
+
// Federation.ts prevents the actual blob re-write, but advancing the
|
|
574
|
+
// cursor here stops the redundant network traffic + Lambda compute
|
|
575
|
+
// entirely. Task #146. Even no-change runs should advance.
|
|
576
|
+
try {
|
|
577
|
+
const advanceRes = await fetch(`${opsEndpoint}/`, {
|
|
578
|
+
method: "POST",
|
|
579
|
+
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
580
|
+
body: JSON.stringify({
|
|
581
|
+
operation: "update",
|
|
582
|
+
database: "flair",
|
|
583
|
+
table: "Peer",
|
|
584
|
+
records: [{ id: hub.id, lastSyncAt: syncStartedAt }],
|
|
585
|
+
}),
|
|
586
|
+
signal: AbortSignal.timeout(10_000),
|
|
587
|
+
});
|
|
588
|
+
if (!advanceRes.ok) {
|
|
589
|
+
const txt = await advanceRes.text().catch(() => "");
|
|
590
|
+
console.warn(`⚠️ Local hub.lastSyncAt advance failed (${advanceRes.status}): ${txt.slice(0, 200)}. Next poll will re-send memories.`);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
catch (advErr) {
|
|
594
|
+
console.warn(`⚠️ Local hub.lastSyncAt advance error: ${advErr?.message ?? advErr}. Next poll will re-send memories.`);
|
|
595
|
+
}
|
|
596
|
+
if (totalBatches === 0) {
|
|
597
|
+
// No-change syncs must still ping the hub so it updates the
|
|
598
|
+
// spoke's lastSyncAt (liveness). Without this, idle-but-alive spokes
|
|
599
|
+
// look indistinguishable from dead ones on the hub dashboard.
|
|
600
|
+
try {
|
|
601
|
+
if (!secretKey)
|
|
602
|
+
secretKey = await loadInstanceSecretKey(instance.id, opts);
|
|
603
|
+
const pingBody = signBodyFresh({
|
|
604
|
+
instanceId: instance.id,
|
|
605
|
+
records: [],
|
|
606
|
+
lamportClock: Date.now(),
|
|
607
|
+
}, secretKey);
|
|
608
|
+
const pingRes = await fetch(`${hubUrl}/FederationSync`, {
|
|
609
|
+
method: "POST",
|
|
610
|
+
headers: { "Content-Type": "application/json" },
|
|
611
|
+
body: JSON.stringify(pingBody),
|
|
612
|
+
signal: AbortSignal.timeout(10_000),
|
|
613
|
+
});
|
|
614
|
+
if (!pingRes.ok) {
|
|
615
|
+
const txt = await pingRes.text().catch(() => "");
|
|
616
|
+
console.warn(`⚠️ Liveness ping to hub failed (${pingRes.status}): ${txt.slice(0, 200)}. Hub won't update spoke liveness.`);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
catch (pingErr) {
|
|
620
|
+
console.warn(`⚠️ Liveness ping error: ${pingErr?.message ?? pingErr}. Hub won't update spoke liveness.`);
|
|
621
|
+
}
|
|
622
|
+
// flair#1232: "No changes" is true only when nothing was found since
|
|
623
|
+
// the cursor. If rows were found and every one was withheld as private,
|
|
624
|
+
// say so — count and reason only, never content. A zero withheld count
|
|
625
|
+
// must not invent a private-withheld story.
|
|
626
|
+
console.log(privateHeldBack > 0
|
|
627
|
+
? `No federable changes since last sync (${privateHeldBack} row${privateHeldBack === 1 ? "" : "s"} held back: private visibility).`
|
|
628
|
+
: "No changes since last sync.");
|
|
629
|
+
return { pushed: 0, skipped: 0 };
|
|
630
|
+
}
|
|
631
|
+
console.log(`✅ Synced ${totalMerged} records (${totalSkipped} skipped) across ${totalBatches} batches`);
|
|
632
|
+
return { pushed: totalMerged, skipped: totalSkipped };
|
|
633
|
+
}
|
|
634
|
+
catch (err) {
|
|
635
|
+
return { pushed: totalMerged, skipped: totalSkipped, error: err instanceof Error ? err : new Error(String(err)) };
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
export async function runFederationWatch(opts) {
|
|
639
|
+
const intervalMs = Math.max(5, parseFloat(opts.interval) || 30) * 1000;
|
|
640
|
+
let stopped = false;
|
|
641
|
+
const stop = () => { stopped = true; };
|
|
642
|
+
process.on("SIGINT", stop);
|
|
643
|
+
process.on("SIGTERM", stop);
|
|
644
|
+
console.log(`flair federation watch — interval ${intervalMs / 1000}s. Ctrl-C to stop.`);
|
|
645
|
+
try {
|
|
646
|
+
while (!stopped) {
|
|
647
|
+
try {
|
|
648
|
+
const r = await runFederationSyncOnce(opts);
|
|
649
|
+
const ts = new Date().toISOString();
|
|
650
|
+
if (r.error)
|
|
651
|
+
console.error(`[${ts}] sync error: ${r.error.message}`);
|
|
652
|
+
else
|
|
653
|
+
console.log(`[${ts}] sync ok — pushed ${r.pushed}, skipped ${r.skipped}`);
|
|
654
|
+
}
|
|
655
|
+
catch (err) {
|
|
656
|
+
console.error(`[${new Date().toISOString()}] watch loop error: ${err.message}`);
|
|
657
|
+
}
|
|
658
|
+
// Sleep but exit early on signal
|
|
659
|
+
const t = Date.now();
|
|
660
|
+
while (!stopped && Date.now() - t < intervalMs) {
|
|
661
|
+
const remaining = intervalMs - (Date.now() - t);
|
|
662
|
+
await new Promise((r) => setTimeout(r, Math.min(250, remaining)));
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
finally {
|
|
667
|
+
process.removeListener("SIGINT", stop);
|
|
668
|
+
process.removeListener("SIGTERM", stop);
|
|
669
|
+
}
|
|
670
|
+
console.log("flair federation watch — stopped.");
|
|
671
|
+
}
|
|
672
|
+
// `flair federation prune` — remove stale spoke peers (never the hub).
|
|
673
|
+
// Productizes flair#695 into a real CLI
|
|
674
|
+
// subcommand with safety: dry-run is the default, --apply required to delete.
|
|
675
|
+
function parseDuration(spec) {
|
|
676
|
+
// Accept forms like "30d", "12h", "90m". Returns milliseconds.
|
|
677
|
+
// Rejects zero and sub-1-minute durations: a 0-ms cutoff would equal Date.now()
|
|
678
|
+
// and prune every non-hub peer (Sherlock review on #314).
|
|
679
|
+
const m = spec.match(/^(\d+)\s*([smhd])$/i);
|
|
680
|
+
if (!m)
|
|
681
|
+
return null;
|
|
682
|
+
const n = Number(m[1]);
|
|
683
|
+
const unit = m[2].toLowerCase();
|
|
684
|
+
const mul = { s: 1000, m: 60 * 1000, h: 60 * 60 * 1000, d: 24 * 60 * 60 * 1000 }[unit] ?? null;
|
|
685
|
+
if (mul == null)
|
|
686
|
+
return null;
|
|
687
|
+
const ms = n * mul;
|
|
688
|
+
const ONE_MINUTE = 60 * 1000;
|
|
689
|
+
if (ms < ONE_MINUTE)
|
|
690
|
+
return null;
|
|
691
|
+
return ms;
|
|
692
|
+
}
|
|
693
|
+
export function register(program) {
|
|
694
|
+
// ─── flair federation ────────────────────────────────────────────────────────
|
|
695
|
+
const federation = program.command("federation").description("Manage federation (hub-and-spoke sync)");
|
|
696
|
+
federation
|
|
697
|
+
.command("status")
|
|
698
|
+
.description("Show federation status and peer connections")
|
|
699
|
+
.option("--port <port>", "Harper HTTP port")
|
|
700
|
+
.option("--target <url>", "Remote Flair URL (env: FLAIR_TARGET)")
|
|
701
|
+
.option("--ops-target <url>", "Explicit ops API URL (env: FLAIR_OPS_TARGET; bypasses port derivation)")
|
|
702
|
+
.option("--json", "Emit JSON {instance, peers, driver} (also: pipe + FLAIR_OUTPUT=json)")
|
|
703
|
+
.action(async (opts) => {
|
|
704
|
+
// Same URL api() would have derived, including --port (the command
|
|
705
|
+
// advertised --port but previously dropped it on the floor). Naming
|
|
706
|
+
// that URL on fetch failure is only honest if it is the URL we probe.
|
|
707
|
+
const baseUrl = resolveBaseUrl(opts).replace(/\/$/, "");
|
|
708
|
+
const urlSetting = federationStatusUrlSetting(opts);
|
|
709
|
+
const mode = render.resolveOutputMode(opts);
|
|
710
|
+
// flair#1233: fetch instance and peers INDEPENDENTLY. One read failing
|
|
711
|
+
// must never take down the whole render — the principle latestPeerContact
|
|
712
|
+
// already documents for the driver verdict ("must never be the reason
|
|
713
|
+
// `federation status` fails"), extended to the two primary reads. Print
|
|
714
|
+
// what's available; mark the rest unverifiable.
|
|
715
|
+
let instance = null;
|
|
716
|
+
let instanceErr = null;
|
|
717
|
+
try {
|
|
718
|
+
instance = await api("GET", "/FederationInstance", undefined, { baseUrl });
|
|
719
|
+
}
|
|
720
|
+
catch (err) {
|
|
721
|
+
instanceErr = rewriteFederationStatusFetchFailed(err, baseUrl, urlSetting);
|
|
722
|
+
}
|
|
723
|
+
// peers: null = unverifiable (the read failed), [] = verified empty.
|
|
724
|
+
let peers = null;
|
|
725
|
+
let peersErr = null;
|
|
726
|
+
try {
|
|
727
|
+
const r = await api("GET", "/FederationPeers", undefined, { baseUrl });
|
|
728
|
+
peers = r.peers ?? [];
|
|
729
|
+
}
|
|
730
|
+
catch (err) {
|
|
731
|
+
peersErr = rewriteFederationStatusFetchFailed(err, baseUrl, urlSetting);
|
|
732
|
+
}
|
|
733
|
+
// Auth-shaped failures stay FATAL even when the other read succeeded:
|
|
734
|
+
// both endpoints sit behind the same allowAdmin gate, so a 401/403 is a
|
|
735
|
+
// property of the session's credentials, not of one endpoint — and
|
|
736
|
+
// degrading it to "unverifiable" would swallow the actionable remedy
|
|
737
|
+
// (flair#634's UX, kept). Only non-auth failures degrade independently.
|
|
738
|
+
// Both reads failed → nothing to render at all. Either way keep the
|
|
739
|
+
// classic failure UX (auth remedy when it's an auth problem), exit
|
|
740
|
+
// non-zero.
|
|
741
|
+
if ((instanceErr && peersErr) || isFederationStatusAuthFailure(instanceErr) || isFederationStatusAuthFailure(peersErr)) {
|
|
742
|
+
const primaryErr = instanceErr ?? peersErr;
|
|
743
|
+
const msg = String(primaryErr.message ?? primaryErr);
|
|
744
|
+
if (isFederationStatusAuthRemedy(primaryErr)) {
|
|
745
|
+
console.error(`${render.icons.error} federation status requires auth.`);
|
|
746
|
+
console.error(` ${render.wrap(render.c.dim, "Set one of:")}`);
|
|
747
|
+
console.error(` ${render.wrap(render.c.cyan, "FLAIR_AGENT_ID=<your-agent-id>")} ${render.wrap(render.c.dim, "(Ed25519 — uses ~/.flair/keys/<id>.key)")}`);
|
|
748
|
+
console.error(` ${render.wrap(render.c.cyan, "FLAIR_ADMIN_PASS=<admin-password>")} ${render.wrap(render.c.dim, "(admin Basic auth, remote targets)")}`);
|
|
749
|
+
console.error(` ${render.wrap(render.c.cyan, "FLAIR_TOKEN=<bearer>")} ${render.wrap(render.c.dim, "(legacy)")}`);
|
|
750
|
+
process.exit(1);
|
|
751
|
+
}
|
|
752
|
+
console.error(`${render.icons.error} ${msg}`);
|
|
753
|
+
process.exit(1);
|
|
754
|
+
}
|
|
755
|
+
// Driver state (flair#922). Computed from the LOCAL service manager, so
|
|
756
|
+
// it is only meaningful when the CLI is pointed at the local instance —
|
|
757
|
+
// and only when the peer read succeeded: the verdict is derived from
|
|
758
|
+
// peer contact, so with peers unverifiable it would be a confident claim
|
|
759
|
+
// built on no data.
|
|
760
|
+
let driver = null;
|
|
761
|
+
let assessment = null;
|
|
762
|
+
if (peers !== null && driverCheckAppliesTo(opts)) {
|
|
763
|
+
try {
|
|
764
|
+
const { schedulerStatus, assessDriver } = await import("../federation/scheduler.js");
|
|
765
|
+
driver = schedulerStatus();
|
|
766
|
+
let lastSyncAt = null;
|
|
767
|
+
for (const p of peers) {
|
|
768
|
+
if (!p?.lastSyncAt)
|
|
769
|
+
continue;
|
|
770
|
+
const t = Date.parse(p.lastSyncAt);
|
|
771
|
+
if (Number.isFinite(t) && (lastSyncAt === null || t > Date.parse(lastSyncAt))) {
|
|
772
|
+
lastSyncAt = new Date(t).toISOString();
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
assessment = assessDriver({
|
|
776
|
+
installed: driver.installed,
|
|
777
|
+
active: driver.active,
|
|
778
|
+
intervalSeconds: driver.intervalSeconds,
|
|
779
|
+
lastSyncAt,
|
|
780
|
+
now: Date.now(),
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
catch {
|
|
784
|
+
// An unsupported platform (neither darwin nor linux) or an
|
|
785
|
+
// unreadable unit must not take down `federation status` — the peer
|
|
786
|
+
// table is still the primary output.
|
|
787
|
+
driver = null;
|
|
788
|
+
assessment = null;
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
if (mode === "json") {
|
|
792
|
+
console.log(render.asJSON({
|
|
793
|
+
instance,
|
|
794
|
+
peers,
|
|
795
|
+
driver,
|
|
796
|
+
driverAssessment: assessment,
|
|
797
|
+
// flair#1233: name what could not be read, so a partial result is
|
|
798
|
+
// distinguishable from "verified absent" (instance/peers stay null
|
|
799
|
+
// when their read failed).
|
|
800
|
+
...(instanceErr || peersErr
|
|
801
|
+
? {
|
|
802
|
+
unverifiable: {
|
|
803
|
+
...(instanceErr ? { instance: String(instanceErr.message ?? instanceErr) } : {}),
|
|
804
|
+
...(peersErr ? { peers: String(peersErr.message ?? peersErr) } : {}),
|
|
805
|
+
},
|
|
806
|
+
}
|
|
807
|
+
: {}),
|
|
808
|
+
}));
|
|
809
|
+
return;
|
|
810
|
+
}
|
|
811
|
+
console.log(render.wrap(render.c.bold, "Federation"));
|
|
812
|
+
if (instance) {
|
|
813
|
+
const statusColor = instance.status === "active" ? render.c.green : render.c.yellow;
|
|
814
|
+
console.log(render.kv("Instance", `${instance.id} ${render.wrap(render.c.dim, `(${instance.role})`)}`));
|
|
815
|
+
console.log(render.kv("Public key", render.wrap(render.c.dim, instance.publicKey)));
|
|
816
|
+
console.log(render.kv("Status", render.wrap(statusColor, instance.status)));
|
|
817
|
+
// flair#1233 degraded marker — actor + state + remedy. The server sets
|
|
818
|
+
// signingKeyAvailable (runtime-only) on GET /FederationInstance; only
|
|
819
|
+
// an explicit false fires this. Older servers omit the field, which
|
|
820
|
+
// proves nothing either way, so no marker.
|
|
821
|
+
if (instance.signingKeyAvailable === false) {
|
|
822
|
+
console.log();
|
|
823
|
+
console.log(`${render.icons.warn} ${render.wrap(render.c.yellow, "Signing key unavailable — the server could not store or read this instance's private key.")}`);
|
|
824
|
+
console.log(` ${render.wrap(render.c.dim, "Reads (this status) still work; pair/sync signing on that instance will fail until the keystore is fixed.")}`);
|
|
825
|
+
console.log(` ${render.wrap(render.c.cyan, "Fix on the server: make $HOME/.flair/keys (under the Harper process's HOME) a directory writable by the Harper process, mode 0700. If the key was never stored, re-key: delete the Instance row and re-pair.")}`);
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
else {
|
|
829
|
+
// Instance read failed but peers succeeded — render what we have.
|
|
830
|
+
console.log(render.kv("Instance", `${render.icons.warn} unverifiable — ${render.wrap(render.c.dim, String(instanceErr?.message ?? instanceErr))}`));
|
|
831
|
+
}
|
|
832
|
+
if (peers === null) {
|
|
833
|
+
// Peer read failed but the instance rendered — say so explicitly
|
|
834
|
+
// instead of aborting: "unverifiable" is a different claim from "no
|
|
835
|
+
// peers", and conflating them is how hub state became unobservable.
|
|
836
|
+
console.log();
|
|
837
|
+
console.log(`${render.icons.warn} ${render.wrap(render.c.yellow, "Peers unverifiable — the peer read failed. This says nothing about whether peers exist or sync runs.")}`);
|
|
838
|
+
console.log(` ${render.wrap(render.c.dim, String(peersErr?.message ?? peersErr))}`);
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
if (peers.length === 0) {
|
|
842
|
+
console.log(`\n${render.icons.info} ${render.wrap(render.c.dim, "No peers configured. Use 'flair federation pair' to connect to a hub.")}`);
|
|
843
|
+
return;
|
|
844
|
+
}
|
|
845
|
+
// Print the driver line BEFORE the per-peer table: "is anything running
|
|
846
|
+
// sync at all" is the question that decides how to read everything
|
|
847
|
+
// below it.
|
|
848
|
+
if (assessment) {
|
|
849
|
+
const icon = assessment.verdict === "driving" || assessment.verdict === "external-driver"
|
|
850
|
+
? render.icons.ok
|
|
851
|
+
: assessment.verdict === "unknown"
|
|
852
|
+
? render.icons.info
|
|
853
|
+
: render.icons.warn;
|
|
854
|
+
const color = assessment.verdict === "driving" || assessment.verdict === "external-driver"
|
|
855
|
+
? render.c.green
|
|
856
|
+
: assessment.verdict === "unknown"
|
|
857
|
+
? render.c.dim
|
|
858
|
+
: render.c.yellow;
|
|
859
|
+
console.log();
|
|
860
|
+
console.log(`${icon} ${render.wrap(color, assessment.headline)}`);
|
|
861
|
+
console.log(` ${render.wrap(render.c.dim, assessment.detail)}`);
|
|
862
|
+
if (assessment.remedy)
|
|
863
|
+
console.log(` ${render.wrap(render.c.cyan, assessment.remedy)}`);
|
|
864
|
+
}
|
|
865
|
+
const now = Date.now();
|
|
866
|
+
const formatPeerAge = (iso, refNow, staleAfterMs) => {
|
|
867
|
+
if (!iso)
|
|
868
|
+
return render.wrap(render.c.red, "never");
|
|
869
|
+
const t = Date.parse(iso);
|
|
870
|
+
if (!Number.isFinite(t))
|
|
871
|
+
return render.wrap(render.c.red, "never");
|
|
872
|
+
const ageMs = refNow - t;
|
|
873
|
+
const ageStr = ageMs < 60_000 ? "<1m ago"
|
|
874
|
+
: ageMs < 3_600_000 ? `${Math.floor(ageMs / 60_000)}m ago`
|
|
875
|
+
: ageMs < 86_400_000 ? `${Math.floor(ageMs / 3_600_000)}h ago`
|
|
876
|
+
: `${Math.floor(ageMs / 86_400_000)}d ago`;
|
|
877
|
+
const stale = ageMs > staleAfterMs;
|
|
878
|
+
return render.wrap(stale ? render.c.yellow : render.c.dim, ageStr);
|
|
879
|
+
};
|
|
880
|
+
console.log();
|
|
881
|
+
const cols = [
|
|
882
|
+
{ label: "peer", key: "id" },
|
|
883
|
+
{ label: "role", key: "role", format: (v) => String(v ?? "—") },
|
|
884
|
+
{
|
|
885
|
+
label: "status",
|
|
886
|
+
key: "status",
|
|
887
|
+
format: (v) => {
|
|
888
|
+
const s = String(v ?? "—");
|
|
889
|
+
const color = s === "paired" || s === "connected" || s === "active" ? render.c.green : s === "revoked" ? render.c.red : render.c.yellow;
|
|
890
|
+
return render.wrap(color, s);
|
|
891
|
+
},
|
|
892
|
+
},
|
|
893
|
+
{
|
|
894
|
+
// Liveness: "did we hear from this peer recently?" Updates on every
|
|
895
|
+
// contact, even when 100% of records were skipped. See flair#444.
|
|
896
|
+
label: "last_sync",
|
|
897
|
+
key: "lastSyncAt",
|
|
898
|
+
format: (v) => formatPeerAge(v, now, 86_400_000),
|
|
899
|
+
},
|
|
900
|
+
{
|
|
901
|
+
// Progress: "did data actually flow in?" Updates only when merged>0.
|
|
902
|
+
// Diverging from last_sync means contact-yes but data-no — investigate.
|
|
903
|
+
label: "last_merge",
|
|
904
|
+
key: "lastMergeAt",
|
|
905
|
+
format: (v) => formatPeerAge(v, now, 86_400_000),
|
|
906
|
+
},
|
|
907
|
+
{
|
|
908
|
+
label: "relay",
|
|
909
|
+
key: "relayOnly",
|
|
910
|
+
format: (v) => (v ? render.wrap(render.c.cyan, "yes") : render.wrap(render.c.dim, "no")),
|
|
911
|
+
},
|
|
912
|
+
];
|
|
913
|
+
console.log(render.table(cols, peers));
|
|
914
|
+
// Stale warning is gated on lastMergeAt (real progress), not lastSyncAt.
|
|
915
|
+
// A peer that "syncs" every 5min but hasn't merged a record in 24h is
|
|
916
|
+
// exactly the failure mode we want surfaced.
|
|
917
|
+
const haveStale = peers.some((p) => {
|
|
918
|
+
const cursor = p.lastMergeAt ?? p.lastSyncAt;
|
|
919
|
+
if (!cursor)
|
|
920
|
+
return true;
|
|
921
|
+
const t = Date.parse(cursor);
|
|
922
|
+
return !Number.isFinite(t) || (now - t) > 86_400_000;
|
|
923
|
+
});
|
|
924
|
+
if (haveStale) {
|
|
925
|
+
console.log();
|
|
926
|
+
// The staleness warning used to fire identically whether sync was
|
|
927
|
+
// running and the peer was unreachable, or nothing had run sync since
|
|
928
|
+
// the day the spoke was paired (flair#922). Those need opposite
|
|
929
|
+
// actions, so the remedy is now chosen by the driver verdict instead
|
|
930
|
+
// of always pointing at SyncLog.
|
|
931
|
+
const noDriver = assessment?.verdict === "no-driver" || assessment?.verdict === "driver-inactive";
|
|
932
|
+
const remedy = noDriver
|
|
933
|
+
? "Nothing is driving sync — see the driver line above. Run 'flair federation sync enable'."
|
|
934
|
+
: "Check skippedReasons in SyncLog or run 'flair federation sync'.";
|
|
935
|
+
console.log(`${render.icons.warn} ${render.wrap(render.c.yellow, "One or more peers haven't merged a record in >24h.")} ${render.wrap(render.c.dim, remedy)}`);
|
|
936
|
+
}
|
|
937
|
+
const haveContactButNoMerge = peers.some((p) => {
|
|
938
|
+
if (!p.lastSyncAt || !Number.isFinite(Date.parse(p.lastSyncAt)))
|
|
939
|
+
return false;
|
|
940
|
+
if ((now - Date.parse(p.lastSyncAt)) > 3_600_000)
|
|
941
|
+
return false; // only recent contact
|
|
942
|
+
// Contact within the last hour, but no merge ever (or stale by >1h)
|
|
943
|
+
if (!p.lastMergeAt)
|
|
944
|
+
return true;
|
|
945
|
+
const tm = Date.parse(p.lastMergeAt);
|
|
946
|
+
return !Number.isFinite(tm) || (now - tm) > 3_600_000;
|
|
947
|
+
});
|
|
948
|
+
if (haveContactButNoMerge && !haveStale) {
|
|
949
|
+
console.log();
|
|
950
|
+
console.log(`${render.icons.warn} ${render.wrap(render.c.yellow, "Peer contact is fresh but no records merged in the last hour.")} ${render.wrap(render.c.dim, "Possible silent-skip scenario — check SyncLog.skippedReasons.")}`);
|
|
951
|
+
}
|
|
952
|
+
});
|
|
953
|
+
// `flair federation reachability` — probe local instance + all paired peers.
|
|
954
|
+
// Productizes flair#695: a single command that tells
|
|
955
|
+
// you whether memories CAN flow across the federation right now. Read-only;
|
|
956
|
+
// no mutations, no side effects beyond a single tagged status read per peer.
|
|
957
|
+
federation
|
|
958
|
+
.command("reachability")
|
|
959
|
+
.description("Probe local Flair + each paired peer for reachability (read-only)")
|
|
960
|
+
.option("--port <port>", "Harper HTTP port")
|
|
961
|
+
.option("--target <url>", "Remote Flair URL (env: FLAIR_TARGET)")
|
|
962
|
+
.option("--quiet", "Suppress output on full success")
|
|
963
|
+
.option("--json", "Emit machine-readable JSON instead of text")
|
|
964
|
+
.option("--peer-timeout <seconds>", "HTTP timeout per peer probe (default 5)", "5")
|
|
965
|
+
.action(async (opts) => {
|
|
966
|
+
const target = resolveTarget(opts);
|
|
967
|
+
const baseUrl = target ? target.replace(/\/$/, "") : undefined;
|
|
968
|
+
const timeoutMs = (Number(opts.peerTimeout) || 5) * 1000;
|
|
969
|
+
const results = [];
|
|
970
|
+
// 1. Local probe.
|
|
971
|
+
try {
|
|
972
|
+
const inst = await api("GET", "/FederationInstance", undefined, baseUrl ? { baseUrl } : undefined);
|
|
973
|
+
results.push({ host: "local", port: null, status: "ok", detail: `instance ${inst.id} (${inst.role}, ${inst.status})` });
|
|
974
|
+
}
|
|
975
|
+
catch (e) {
|
|
976
|
+
results.push({ host: "local", port: null, status: "fail", detail: e.message });
|
|
977
|
+
}
|
|
978
|
+
// 2. Per-peer probes. For each peer with an `endpoint` (URL), probe it.
|
|
979
|
+
// Peers without an endpoint are reverse-tunnel-paired (the spoke can't
|
|
980
|
+
// reach the hub directly without the tunnel) and we skip.
|
|
981
|
+
let peers = [];
|
|
982
|
+
try {
|
|
983
|
+
const r = await api("GET", "/FederationPeers", undefined, baseUrl ? { baseUrl } : undefined);
|
|
984
|
+
peers = r.peers ?? [];
|
|
985
|
+
}
|
|
986
|
+
catch (e) {
|
|
987
|
+
results.push({ host: "/FederationPeers", port: null, status: "fail", detail: e.message });
|
|
988
|
+
}
|
|
989
|
+
for (const p of peers) {
|
|
990
|
+
const endpoint = p.endpoint;
|
|
991
|
+
if (!endpoint) {
|
|
992
|
+
results.push({ host: p.id, port: null, status: "skip", detail: `${p.role ?? "—"} (no endpoint — needs tunnel)` });
|
|
993
|
+
continue;
|
|
994
|
+
}
|
|
995
|
+
// Any HTTP response (including 401) means the peer is reachable + responding.
|
|
996
|
+
// We're checking the network path, not auth; 401 is expected for unauth probes.
|
|
997
|
+
// Use new URL() to avoid path-swallowing when endpoint includes a query
|
|
998
|
+
// (Sherlock review on #314).
|
|
999
|
+
let probeUrl;
|
|
1000
|
+
try {
|
|
1001
|
+
probeUrl = new URL("/Health", endpoint);
|
|
1002
|
+
}
|
|
1003
|
+
catch {
|
|
1004
|
+
results.push({ host: p.id, port: null, status: "fail", detail: `${p.role ?? "—"} invalid endpoint URL` });
|
|
1005
|
+
continue;
|
|
1006
|
+
}
|
|
1007
|
+
if (probeUrl.protocol !== "http:" && probeUrl.protocol !== "https:") {
|
|
1008
|
+
results.push({ host: p.id, port: null, status: "fail", detail: `${p.role ?? "—"} unsupported protocol ${probeUrl.protocol}` });
|
|
1009
|
+
continue;
|
|
1010
|
+
}
|
|
1011
|
+
try {
|
|
1012
|
+
const ctrl = new AbortController();
|
|
1013
|
+
const t = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
1014
|
+
const res = await fetch(probeUrl, { signal: ctrl.signal });
|
|
1015
|
+
clearTimeout(t);
|
|
1016
|
+
results.push({ host: p.id, port: null, status: "ok", detail: `${p.role ?? "—"} HTTP ${res.status}` });
|
|
1017
|
+
}
|
|
1018
|
+
catch (e) {
|
|
1019
|
+
const msg = e.name === "AbortError" ? `timeout after ${opts.peerTimeout}s` : e.message;
|
|
1020
|
+
results.push({ host: p.id, port: null, status: "fail", detail: `${p.role ?? "—"} ${msg}` });
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
const failures = results.filter(r => r.status === "fail").length;
|
|
1024
|
+
if (opts.json) {
|
|
1025
|
+
console.log(JSON.stringify({ ts: new Date().toISOString(), failures, results }, null, 2));
|
|
1026
|
+
}
|
|
1027
|
+
else if (!(opts.quiet && failures === 0)) {
|
|
1028
|
+
console.log(`── Flair reachability — ${new Date().toISOString()} ──`);
|
|
1029
|
+
for (const r of results) {
|
|
1030
|
+
const tag = r.status === "ok" ? "OK " : r.status === "skip" ? "SKIP" : "FAIL";
|
|
1031
|
+
console.log(`${tag} ${r.host.padEnd(40)} ${r.detail}`);
|
|
1032
|
+
}
|
|
1033
|
+
if (failures > 0) {
|
|
1034
|
+
console.log(`── ${failures} path(s) FAILED ──`);
|
|
1035
|
+
}
|
|
1036
|
+
else {
|
|
1037
|
+
console.log("── all reachable ──");
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
if (failures > 0)
|
|
1041
|
+
process.exit(1);
|
|
1042
|
+
});
|
|
1043
|
+
federation
|
|
1044
|
+
.command("pair <hub-url>")
|
|
1045
|
+
.description("Pair this spoke with a hub instance")
|
|
1046
|
+
.option("--port <port>", "Harper HTTP port")
|
|
1047
|
+
.option("--admin-pass <pass>", "Admin password")
|
|
1048
|
+
.option("--admin-user <name>", "Admin username for Basic auth (env: FLAIR_ADMIN_USER; default: admin)")
|
|
1049
|
+
.option("--ops-port <port>", "Harper operations API port")
|
|
1050
|
+
.option("--token <token>", "One-time pairing token from hub admin (env: FLAIR_PAIRING_TOKEN) [deprecated: use --token-from]")
|
|
1051
|
+
.option("--token-from <file>", "Read bootstrap triple from JSON file (use '-' for stdin)")
|
|
1052
|
+
.option("--target <url>", "Remote Flair URL (env: FLAIR_TARGET)")
|
|
1053
|
+
.option("--ops-target <url>", "Explicit ops API URL (env: FLAIR_OPS_TARGET; bypasses port derivation)")
|
|
1054
|
+
.action(async (hubUrl, opts) => {
|
|
1055
|
+
const target = resolveTarget(opts);
|
|
1056
|
+
const baseUrl = target ? target.replace(/\/$/, "") : undefined;
|
|
1057
|
+
try {
|
|
1058
|
+
const instance = await api("GET", "/FederationInstance", undefined, baseUrl ? { baseUrl } : undefined);
|
|
1059
|
+
console.log(`${target ? "Remote" : "Local"} instance: ${instance.id} (${instance.role})`);
|
|
1060
|
+
// Determine token source: --token-from wins if both specified
|
|
1061
|
+
if (opts.tokenFrom && opts.token) {
|
|
1062
|
+
console.error("warning: --token-from takes precedence over --token. The --token flag is deprecated; use --token-from <file> instead.");
|
|
1063
|
+
}
|
|
1064
|
+
let pairingToken;
|
|
1065
|
+
let authHeader;
|
|
1066
|
+
if (opts.tokenFrom) {
|
|
1067
|
+
// ── Bootstrap triple path (--token-from) ──
|
|
1068
|
+
const triple = parseTokenFromFile(opts.tokenFrom);
|
|
1069
|
+
pairingToken = triple.token;
|
|
1070
|
+
authHeader = `Basic ${Buffer.from(`${triple.user}:${triple.password}`).toString("base64")}`;
|
|
1071
|
+
console.log(`Using bootstrap user: ${triple.user}`);
|
|
1072
|
+
}
|
|
1073
|
+
else if (opts.token) {
|
|
1074
|
+
// ── Bare token path (--token) — deprecated ──
|
|
1075
|
+
pairingToken = opts.token || process.env.FLAIR_PAIRING_TOKEN;
|
|
1076
|
+
console.error("warning: --token is deprecated. Use --token-from <file> to keep credentials out of shell history.");
|
|
1077
|
+
// Warning: inline token may leak to shell history.
|
|
1078
|
+
const tokenFromEnv = !opts.token && !!process.env.FLAIR_PAIRING_TOKEN;
|
|
1079
|
+
if (shouldShowInlineSecretWarning(opts.token, tokenFromEnv, new Set(["--token"]), "--token")) {
|
|
1080
|
+
console.error("warning: --token passed inline. Consider --token-from <file> or FLAIR_PAIRING_TOKEN env " +
|
|
1081
|
+
"to keep secrets out of shell history.");
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
else {
|
|
1085
|
+
console.error("Error: --token or --token-from is required. Ask the hub admin to run 'flair federation token' and provide the token.");
|
|
1086
|
+
process.exit(1);
|
|
1087
|
+
}
|
|
1088
|
+
// Load secret key and sign the pairing request.
|
|
1089
|
+
const secretKey = await loadInstanceSecretKey(instance.id, opts);
|
|
1090
|
+
const pairBody = {
|
|
1091
|
+
instanceId: instance.id,
|
|
1092
|
+
publicKey: instance.publicKey,
|
|
1093
|
+
role: "spoke",
|
|
1094
|
+
pairingToken,
|
|
1095
|
+
};
|
|
1096
|
+
const signedBody = signBodyFresh(pairBody, secretKey);
|
|
1097
|
+
const fetchHeaders = { "Content-Type": "application/json" };
|
|
1098
|
+
if (authHeader) {
|
|
1099
|
+
fetchHeaders.Authorization = authHeader;
|
|
1100
|
+
}
|
|
1101
|
+
const res = await fetch(`${hubUrl}/FederationPair`, {
|
|
1102
|
+
method: "POST",
|
|
1103
|
+
headers: fetchHeaders,
|
|
1104
|
+
body: JSON.stringify(signedBody),
|
|
1105
|
+
});
|
|
1106
|
+
if (!res.ok) {
|
|
1107
|
+
const text = await res.text().catch(() => "");
|
|
1108
|
+
console.error(`Pairing failed: ${res.status} ${text}`);
|
|
1109
|
+
process.exit(1);
|
|
1110
|
+
}
|
|
1111
|
+
const result = await res.json();
|
|
1112
|
+
// flair#822: fail-closed. Pair already returns instance.{id,publicKey}
|
|
1113
|
+
// when the hub has a FederationInstance row. A missing key means that
|
|
1114
|
+
// row was absent (#839) — ERROR, never store "". Do not GET
|
|
1115
|
+
// /FederationInstance: bootstrap Basic cannot read it (allowAdmin),
|
|
1116
|
+
// and a successful GET find-or-creates a hub Instance. A spoke Peer
|
|
1117
|
+
// write does not provision the hub row.
|
|
1118
|
+
const resolvedHub = resolveHubPeerIdentity(result);
|
|
1119
|
+
if (resolvedHub.ok === false) {
|
|
1120
|
+
console.error(`Error: ${resolvedHub.error}`);
|
|
1121
|
+
process.exit(1);
|
|
1122
|
+
}
|
|
1123
|
+
console.log(`✅ Paired with hub: ${resolvedHub.peer.id}`);
|
|
1124
|
+
// Record the hub as our local peer. This is REQUIRED, not optional:
|
|
1125
|
+
// `flair federation sync` reads the Peer table to find the hub, so
|
|
1126
|
+
// without this record sync reports "No hub peer configured" and silently
|
|
1127
|
+
// never runs. Previously this was gated on `if (adminPass)` and the write
|
|
1128
|
+
// result was never checked — pairing with only an agent key (or a failed
|
|
1129
|
+
// upsert) left no peer behind a misleadingly green "✅ Paired".
|
|
1130
|
+
const adminPass = opts.adminPass ?? process.env.FLAIR_ADMIN_PASS ?? process.env.HDB_ADMIN_PASSWORD ?? "";
|
|
1131
|
+
if (!adminPass) {
|
|
1132
|
+
console.error("Error: paired on the hub, but the local hub-peer record needs admin auth to write — " +
|
|
1133
|
+
"pass --admin-pass, or set FLAIR_ADMIN_PASS / HDB_ADMIN_PASSWORD, then re-run pair. " +
|
|
1134
|
+
"Without it, 'flair federation sync' will report 'No hub peer configured'.");
|
|
1135
|
+
process.exit(1);
|
|
1136
|
+
}
|
|
1137
|
+
const auth = `Basic ${Buffer.from(`${resolveAdminUser(opts.adminUser)}:${adminPass}`).toString("base64")}`;
|
|
1138
|
+
const opsEndpoint = resolveEffectiveOpsUrl(opts) ?? `http://127.0.0.1:${resolveOpsPort(opts)}`;
|
|
1139
|
+
const peerRes = await fetch(`${opsEndpoint}/`, {
|
|
1140
|
+
method: "POST",
|
|
1141
|
+
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
1142
|
+
body: JSON.stringify({
|
|
1143
|
+
operation: "upsert", database: "flair", table: "Peer",
|
|
1144
|
+
records: [{
|
|
1145
|
+
id: resolvedHub.peer.id,
|
|
1146
|
+
publicKey: resolvedHub.peer.publicKey,
|
|
1147
|
+
role: "hub", endpoint: hubUrl, status: "paired",
|
|
1148
|
+
pairedAt: new Date().toISOString(),
|
|
1149
|
+
createdAt: new Date().toISOString(),
|
|
1150
|
+
updatedAt: new Date().toISOString(),
|
|
1151
|
+
}],
|
|
1152
|
+
}),
|
|
1153
|
+
signal: AbortSignal.timeout(10_000),
|
|
1154
|
+
});
|
|
1155
|
+
if (!peerRes.ok) {
|
|
1156
|
+
const text = await peerRes.text().catch(() => "");
|
|
1157
|
+
console.error(`Error: paired with the hub but failed to write the local hub-peer record ` +
|
|
1158
|
+
`(${peerRes.status} ${text.slice(0, 200)}). Ops endpoint: ${opsEndpoint}. ` +
|
|
1159
|
+
`'flair federation sync' will not find the hub until this succeeds — check --admin-pass and the ops port.`);
|
|
1160
|
+
process.exit(1);
|
|
1161
|
+
}
|
|
1162
|
+
console.log(`✅ Recorded hub as local peer: ${resolvedHub.peer.id} → ${hubUrl}`);
|
|
1163
|
+
}
|
|
1164
|
+
catch (err) {
|
|
1165
|
+
console.error(`Error: ${err.message}`);
|
|
1166
|
+
process.exit(1);
|
|
1167
|
+
}
|
|
1168
|
+
});
|
|
1169
|
+
federation
|
|
1170
|
+
.command("token")
|
|
1171
|
+
.description("Generate a one-time pairing token (run on the hub)")
|
|
1172
|
+
.option("--port <port>", "Harper HTTP port")
|
|
1173
|
+
.option("--admin-pass <pass>", "Admin password")
|
|
1174
|
+
.option("--admin-user <name>", "Admin username for Basic auth (env: FLAIR_ADMIN_USER; default: admin)")
|
|
1175
|
+
.option("--ops-port <port>", "Harper operations API port")
|
|
1176
|
+
.option("--ttl <minutes>", "Token TTL in minutes (default: 60)", "60")
|
|
1177
|
+
.option("--target <url>", "Remote Flair URL (env: FLAIR_TARGET)")
|
|
1178
|
+
.option("--ops-target <url>", "Explicit ops API URL (env: FLAIR_OPS_TARGET; bypasses port derivation)")
|
|
1179
|
+
.option("--format <format>", "Output format: json (default) or text (bare token, deprecated)", "json")
|
|
1180
|
+
.action(async (opts) => {
|
|
1181
|
+
const target = resolveTarget(opts);
|
|
1182
|
+
const baseUrl = target ? target.replace(/\/$/, "") : undefined;
|
|
1183
|
+
try {
|
|
1184
|
+
const token = randomBytes(24).toString("base64url");
|
|
1185
|
+
const ttlMinutes = parseInt(opts.ttl, 10) || 60;
|
|
1186
|
+
const expiresAt = new Date(Date.now() + ttlMinutes * 60 * 1000).toISOString();
|
|
1187
|
+
const opsEndpoint = resolveEffectiveOpsUrl(opts) ?? `http://127.0.0.1:${resolveOpsPort(opts)}`;
|
|
1188
|
+
const adminPass = opts.adminPass ?? process.env.FLAIR_ADMIN_PASS ?? "";
|
|
1189
|
+
const auth = `Basic ${Buffer.from(`${resolveAdminUser(opts.adminUser)}:${adminPass}`).toString("base64")}`;
|
|
1190
|
+
// 1. Persist the PairingToken record
|
|
1191
|
+
const opsRes = await fetch(`${opsEndpoint}/`, {
|
|
1192
|
+
method: "POST",
|
|
1193
|
+
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
1194
|
+
body: JSON.stringify({
|
|
1195
|
+
operation: "upsert", database: "flair", table: "PairingToken",
|
|
1196
|
+
records: [{
|
|
1197
|
+
id: token,
|
|
1198
|
+
createdAt: new Date().toISOString(),
|
|
1199
|
+
expiresAt,
|
|
1200
|
+
}],
|
|
1201
|
+
}),
|
|
1202
|
+
signal: AbortSignal.timeout(10_000),
|
|
1203
|
+
});
|
|
1204
|
+
if (!opsRes.ok) {
|
|
1205
|
+
const detail = await opsRes.text().catch(() => "");
|
|
1206
|
+
throw new Error(`Failed to persist pairing token (${opsRes.status}): ${detail || "no body"}`);
|
|
1207
|
+
}
|
|
1208
|
+
// 2. Create bootstrap user for this token
|
|
1209
|
+
const bootstrapPassword = randomBytes(32).toString("base64url");
|
|
1210
|
+
const bootstrapUsername = `pair-bootstrap-${token.slice(0, 8)}`;
|
|
1211
|
+
let addUserRes;
|
|
1212
|
+
try {
|
|
1213
|
+
addUserRes = await fetch(`${opsEndpoint}/`, {
|
|
1214
|
+
method: "POST",
|
|
1215
|
+
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
1216
|
+
body: JSON.stringify({
|
|
1217
|
+
operation: "add_user",
|
|
1218
|
+
username: bootstrapUsername,
|
|
1219
|
+
password: bootstrapPassword,
|
|
1220
|
+
role: "flair_pair_initiator",
|
|
1221
|
+
active: true,
|
|
1222
|
+
}),
|
|
1223
|
+
signal: AbortSignal.timeout(10_000),
|
|
1224
|
+
});
|
|
1225
|
+
}
|
|
1226
|
+
catch (err) {
|
|
1227
|
+
// Network failure creating bootstrap user — roll back PairingToken
|
|
1228
|
+
await fetch(`${opsEndpoint}/`, {
|
|
1229
|
+
method: "POST",
|
|
1230
|
+
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
1231
|
+
body: JSON.stringify({
|
|
1232
|
+
operation: "delete",
|
|
1233
|
+
database: "flair",
|
|
1234
|
+
table: "PairingToken",
|
|
1235
|
+
hash_value: token,
|
|
1236
|
+
}),
|
|
1237
|
+
signal: AbortSignal.timeout(10_000),
|
|
1238
|
+
}).catch(() => { });
|
|
1239
|
+
throw new Error(`Failed to create bootstrap user (network): ${err.message}`);
|
|
1240
|
+
}
|
|
1241
|
+
if (!addUserRes.ok) {
|
|
1242
|
+
// add_user failed — roll back PairingToken so the two stay in sync
|
|
1243
|
+
await fetch(`${opsEndpoint}/`, {
|
|
1244
|
+
method: "POST",
|
|
1245
|
+
headers: { "Content-Type": "application/json", Authorization: auth },
|
|
1246
|
+
body: JSON.stringify({
|
|
1247
|
+
operation: "delete",
|
|
1248
|
+
database: "flair",
|
|
1249
|
+
table: "PairingToken",
|
|
1250
|
+
hash_value: token,
|
|
1251
|
+
}),
|
|
1252
|
+
signal: AbortSignal.timeout(10_000),
|
|
1253
|
+
}).catch(() => { });
|
|
1254
|
+
const detail = await addUserRes.text().catch(() => "");
|
|
1255
|
+
throw new Error(`Failed to create bootstrap user (${addUserRes.status}): ${detail || "no body"}`);
|
|
1256
|
+
}
|
|
1257
|
+
// 3. Output
|
|
1258
|
+
const format = (opts.format ?? "json").toLowerCase();
|
|
1259
|
+
if (format === "text") {
|
|
1260
|
+
process.stderr.write(`[DEPRECATION] --format text is deprecated. Default output is now JSON.\n`);
|
|
1261
|
+
console.log(token);
|
|
1262
|
+
}
|
|
1263
|
+
else {
|
|
1264
|
+
console.log(JSON.stringify({ token, user: bootstrapUsername, password: bootstrapPassword, expiresAt }, null, 2));
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
catch (err) {
|
|
1268
|
+
console.error(`Error: ${err.message}`);
|
|
1269
|
+
process.exit(1);
|
|
1270
|
+
}
|
|
1271
|
+
});
|
|
1272
|
+
const federationSync = addSharedCredentialOptions(federation
|
|
1273
|
+
.command("sync")
|
|
1274
|
+
.description("Push local changes to the hub (one-shot). Subcommands manage the scheduled driver.")
|
|
1275
|
+
.option("--port <port>", "Harper HTTP port")
|
|
1276
|
+
.option("--ops-port <port>", "Harper operations API port")
|
|
1277
|
+
.option("--target <url>", "Remote Flair URL (env: FLAIR_TARGET)")
|
|
1278
|
+
.option("--ops-target <url>", "Explicit ops API URL (env: FLAIR_OPS_TARGET; bypasses port derivation)")).action(async (opts) => {
|
|
1279
|
+
// --admin-pass-file resolves into the same `adminPass` slot the inline
|
|
1280
|
+
// flag uses, so the scheduler never has to embed a secret in a unit file.
|
|
1281
|
+
applyAdminPassFile(opts);
|
|
1282
|
+
const r = await runFederationSyncOnce(opts);
|
|
1283
|
+
if (r.error) {
|
|
1284
|
+
console.error(`Error: ${r.error.message}`);
|
|
1285
|
+
process.exit(1);
|
|
1286
|
+
}
|
|
1287
|
+
});
|
|
1288
|
+
// ─── flair federation sync enable | disable | status ────────────────────────
|
|
1289
|
+
// The supervised driver (flair#922). Federation had no automatic driver at
|
|
1290
|
+
// all: `sync` is one-shot and `watch` is a foreground loop that dies with its
|
|
1291
|
+
// terminal, so every operator paired a spoke, saw one successful sync, and
|
|
1292
|
+
// then silently stopped syncing.
|
|
1293
|
+
//
|
|
1294
|
+
// Shape deliberately mirrors `flair rem nightly enable|disable|status` —
|
|
1295
|
+
// same verbs, same platform coverage (launchd on macOS, systemd --user timer
|
|
1296
|
+
// on Linux), same "never claim success before activation succeeded" rule.
|
|
1297
|
+
// Strategy is a PERIODIC ONE-SHOT rather than a supervised long-lived
|
|
1298
|
+
// watcher; the reasoning is in src/federation/scheduler.ts's header.
|
|
1299
|
+
federationSync
|
|
1300
|
+
.command("enable")
|
|
1301
|
+
.description("Install the sync driver (launchd on macOS, systemd timer on Linux)")
|
|
1302
|
+
.option("--interval <seconds>", `Seconds between syncs (default ${FEDERATION_SYNC_DEFAULT_INTERVAL})`, String(FEDERATION_SYNC_DEFAULT_INTERVAL))
|
|
1303
|
+
// Deliberately NOT `--no-admin-pass-file`: commander treats a `--no-x` flag
|
|
1304
|
+
// as the negation of `--x`, and declaring both on one command makes the
|
|
1305
|
+
// POSITIVE option silently parse to undefined — `--admin-pass-file /path`
|
|
1306
|
+
// would be accepted and dropped, producing a driver that fails auth every
|
|
1307
|
+
// cycle with no error anywhere. Verified against commander 14.
|
|
1308
|
+
.option("--no-credentials", "Do not wire any credential file into the unit")
|
|
1309
|
+
// `--admin-pass-file` and `--target` are NOT redeclared here (flair#926).
|
|
1310
|
+
// The parent `flair federation sync` owns both, and commander matches an
|
|
1311
|
+
// option against the parent's list before dispatching — so a duplicate
|
|
1312
|
+
// declaration here never receives a value, it only makes the option LOOK
|
|
1313
|
+
// local. Both flags still work on this command; they arrive via
|
|
1314
|
+
// optsWithGlobals() below and are listed under "Global Options" in --help.
|
|
1315
|
+
.addHelpText("after", "\nCredentials:\n"
|
|
1316
|
+
+ " --admin-pass-file defaults to ~/.flair/admin-pass when that file exists.\n"
|
|
1317
|
+
+ " The PATH is stored in the unit — never the password.\n")
|
|
1318
|
+
.action(async (_opts, cmd) => {
|
|
1319
|
+
// optsWithGlobals(), NOT the action's first argument: `--admin-pass-file`
|
|
1320
|
+
// and `--target` are declared on the PARENT (`flair federation sync`), and
|
|
1321
|
+
// commander binds their values there. The subcommand's own opts() has no
|
|
1322
|
+
// entry for them at all. Reading only the local opts silently dropped
|
|
1323
|
+
// `--admin-pass-file <path>` here (flair#923), which installed a driver
|
|
1324
|
+
// that failed auth every cycle with no error anywhere. Verified against
|
|
1325
|
+
// commander 14; test/unit/cli-option-collisions.test.ts pins the rule.
|
|
1326
|
+
const opts = cmd.optsWithGlobals();
|
|
1327
|
+
const intervalSeconds = Number(opts.interval);
|
|
1328
|
+
if (!Number.isFinite(intervalSeconds)) {
|
|
1329
|
+
console.error(`Error: --interval must be a number of seconds (got: ${opts.interval})`);
|
|
1330
|
+
process.exit(1);
|
|
1331
|
+
}
|
|
1332
|
+
// Default to the canonical admin-pass file when it is actually there.
|
|
1333
|
+
// Silently wiring a path that does not exist would produce a driver that
|
|
1334
|
+
// runs and fails auth every interval — the failure mode this whole issue
|
|
1335
|
+
// is about, in a new costume.
|
|
1336
|
+
let adminPassFile;
|
|
1337
|
+
if (opts.credentials === false) {
|
|
1338
|
+
adminPassFile = undefined;
|
|
1339
|
+
}
|
|
1340
|
+
else if (typeof opts.adminPassFile === "string" && opts.adminPassFile) {
|
|
1341
|
+
adminPassFile = opts.adminPassFile;
|
|
1342
|
+
}
|
|
1343
|
+
else {
|
|
1344
|
+
const fallback = defaultAdminPassPath();
|
|
1345
|
+
adminPassFile = existsSync(fallback) ? fallback : undefined;
|
|
1346
|
+
}
|
|
1347
|
+
const { enableScheduler, formatEnableReport } = await import("../federation/scheduler.js");
|
|
1348
|
+
try {
|
|
1349
|
+
const r = enableScheduler({ intervalSeconds, adminPassFile, target: opts.target });
|
|
1350
|
+
const { lines, ok } = formatEnableReport(r, { adminPassFile, target: opts.target });
|
|
1351
|
+
for (const line of lines)
|
|
1352
|
+
console.log(line);
|
|
1353
|
+
if (!ok)
|
|
1354
|
+
process.exit(1);
|
|
1355
|
+
}
|
|
1356
|
+
catch (err) {
|
|
1357
|
+
console.error(`Error: ${err.message}`);
|
|
1358
|
+
process.exit(1);
|
|
1359
|
+
}
|
|
1360
|
+
});
|
|
1361
|
+
federationSync
|
|
1362
|
+
.command("disable")
|
|
1363
|
+
.description("Remove the sync driver (peers and sync history are preserved)")
|
|
1364
|
+
.option("--remove-shim", "Also delete the ~/.flair/bin/flair-federation-sync shim")
|
|
1365
|
+
.action(async (opts) => {
|
|
1366
|
+
const { disableScheduler } = await import("../federation/scheduler.js");
|
|
1367
|
+
try {
|
|
1368
|
+
const r = disableScheduler({ removeShim: !!opts.removeShim });
|
|
1369
|
+
if (r.removed.length === 0) {
|
|
1370
|
+
console.log(`(Federation sync driver was not installed on ${r.platform})`);
|
|
1371
|
+
return;
|
|
1372
|
+
}
|
|
1373
|
+
console.log(`✅ Federation sync driver disabled (${r.platform})`);
|
|
1374
|
+
console.log(` Removed:`);
|
|
1375
|
+
for (const p of r.removed)
|
|
1376
|
+
console.log(` ${p}`);
|
|
1377
|
+
if (r.unloadResult && r.unloadResult.code !== 0) {
|
|
1378
|
+
console.log(` Unload: ${r.unloadCommand.join(" ")} → code ${r.unloadResult.code}`);
|
|
1379
|
+
if (r.unloadResult.stderr)
|
|
1380
|
+
console.log(` stderr: ${r.unloadResult.stderr.trim()}`);
|
|
1381
|
+
}
|
|
1382
|
+
console.log(`\nPeers, keys and sync history are untouched. Nothing will sync until you`);
|
|
1383
|
+
console.log(`re-enable the driver or run \`flair federation sync\` by hand.`);
|
|
1384
|
+
}
|
|
1385
|
+
catch (err) {
|
|
1386
|
+
console.error(`Error: ${err.message}`);
|
|
1387
|
+
process.exit(1);
|
|
1388
|
+
}
|
|
1389
|
+
});
|
|
1390
|
+
federationSync
|
|
1391
|
+
.command("status")
|
|
1392
|
+
.description("Show whether a sync driver is installed and genuinely active")
|
|
1393
|
+
// `--port` and `--target` are NOT redeclared here (flair#926) — the parent
|
|
1394
|
+
// `flair federation sync` owns them and commander binds them there. They
|
|
1395
|
+
// still work on this command, via optsWithGlobals() below.
|
|
1396
|
+
.option("--json", "Emit JSON")
|
|
1397
|
+
.action(async (_opts, cmd) => {
|
|
1398
|
+
// See the comment on `enable` above: `--target`/`--port` live on the
|
|
1399
|
+
// parent, so commander binds them there and only optsWithGlobals() sees
|
|
1400
|
+
// them.
|
|
1401
|
+
const opts = cmd.optsWithGlobals();
|
|
1402
|
+
const { schedulerStatus, formatStatusReport, assessDriver } = await import("../federation/scheduler.js");
|
|
1403
|
+
try {
|
|
1404
|
+
const s = schedulerStatus();
|
|
1405
|
+
const lastSyncAt = await latestPeerContact(opts);
|
|
1406
|
+
const a = assessDriver({
|
|
1407
|
+
installed: s.installed,
|
|
1408
|
+
active: s.active,
|
|
1409
|
+
intervalSeconds: s.intervalSeconds,
|
|
1410
|
+
lastSyncAt,
|
|
1411
|
+
now: Date.now(),
|
|
1412
|
+
});
|
|
1413
|
+
if (render.resolveOutputMode(opts) === "json") {
|
|
1414
|
+
console.log(render.asJSON({ driver: s, assessment: a, lastSyncAt }));
|
|
1415
|
+
return;
|
|
1416
|
+
}
|
|
1417
|
+
const { lines } = formatStatusReport(s, a);
|
|
1418
|
+
for (const line of lines)
|
|
1419
|
+
console.log(line);
|
|
1420
|
+
}
|
|
1421
|
+
catch (err) {
|
|
1422
|
+
console.error(`Error: ${err.message}`);
|
|
1423
|
+
process.exit(1);
|
|
1424
|
+
}
|
|
1425
|
+
});
|
|
1426
|
+
federation
|
|
1427
|
+
.command("watch")
|
|
1428
|
+
.description("Run federation sync in a loop (foreground daemon)")
|
|
1429
|
+
.option("--interval <seconds>", "Seconds between syncs", "30")
|
|
1430
|
+
.option("--port <port>", "Harper HTTP port")
|
|
1431
|
+
.option("--admin-pass <pass>", "Admin password")
|
|
1432
|
+
.option("--admin-user <name>", "Admin username for Basic auth (env: FLAIR_ADMIN_USER; default: admin)")
|
|
1433
|
+
.option("--ops-port <port>", "Harper operations API port")
|
|
1434
|
+
.option("--target <url>", "Remote Flair URL")
|
|
1435
|
+
.option("--ops-target <url>", "Explicit ops API URL")
|
|
1436
|
+
.action(async (opts) => {
|
|
1437
|
+
await runFederationWatch(opts);
|
|
1438
|
+
});
|
|
1439
|
+
federation
|
|
1440
|
+
.command("prune")
|
|
1441
|
+
.description("Remove stale spoke peers (older than --older-than). Hub is never pruned. Default dry-run.")
|
|
1442
|
+
.option("--older-than <duration>", "Duration spec (e.g. 30d, 12h, 90m)", "30d")
|
|
1443
|
+
.option("--apply", "Actually delete (default is dry-run)")
|
|
1444
|
+
.option("--include <pattern>", "Only consider peer IDs starting with this prefix")
|
|
1445
|
+
.option("--port <port>", "Harper HTTP port")
|
|
1446
|
+
.option("--ops-port <port>", "Harper operations API port")
|
|
1447
|
+
.option("--target <url>", "Remote Flair URL")
|
|
1448
|
+
.option("--ops-target <url>", "Explicit ops API URL")
|
|
1449
|
+
.action(async (opts) => {
|
|
1450
|
+
const target = resolveTarget(opts);
|
|
1451
|
+
const baseUrl = target ? target.replace(/\/$/, "") : undefined;
|
|
1452
|
+
const olderThanMs = parseDuration(opts.olderThan);
|
|
1453
|
+
if (olderThanMs == null) {
|
|
1454
|
+
console.error(`Error: invalid or unsafe --older-than '${opts.olderThan}'. Use forms like 30d, 12h, 90m. Minimum 1 minute.`);
|
|
1455
|
+
process.exit(2);
|
|
1456
|
+
}
|
|
1457
|
+
const cutoff = Date.now() - olderThanMs;
|
|
1458
|
+
let peers = [];
|
|
1459
|
+
try {
|
|
1460
|
+
const r = await api("GET", "/FederationPeers", undefined, baseUrl ? { baseUrl } : undefined);
|
|
1461
|
+
peers = r.peers ?? [];
|
|
1462
|
+
}
|
|
1463
|
+
catch (e) {
|
|
1464
|
+
console.error(`Error fetching peers: ${e.message}`);
|
|
1465
|
+
process.exit(1);
|
|
1466
|
+
}
|
|
1467
|
+
const candidates = peers.filter(p => {
|
|
1468
|
+
// Hub-protection: never prune. Case-insensitive; null/undefined role is
|
|
1469
|
+
// treated as "unknown — refuse to prune to be safe" (Sherlock review on #314).
|
|
1470
|
+
const role = (p.role ?? "").toString().toLowerCase();
|
|
1471
|
+
if (role === "hub" || role === "")
|
|
1472
|
+
return false;
|
|
1473
|
+
// Include filter.
|
|
1474
|
+
if (opts.include && !String(p.id ?? "").startsWith(opts.include))
|
|
1475
|
+
return false;
|
|
1476
|
+
// Stale threshold: a peer with NO lastSyncAt is treated as having been
|
|
1477
|
+
// born and immediately abandoned — qualifies if it's older than the
|
|
1478
|
+
// threshold based on pairedAt instead.
|
|
1479
|
+
const ts = p.lastSyncAt ?? p.pairedAt;
|
|
1480
|
+
if (!ts)
|
|
1481
|
+
return true; // truly orphaned record — prune candidate.
|
|
1482
|
+
return new Date(ts).getTime() < cutoff;
|
|
1483
|
+
});
|
|
1484
|
+
if (candidates.length === 0) {
|
|
1485
|
+
console.log(`flair federation prune: no peers older than ${opts.olderThan} (and not hub) — nothing to do.`);
|
|
1486
|
+
return;
|
|
1487
|
+
}
|
|
1488
|
+
if (!opts.apply) {
|
|
1489
|
+
console.log(`── flair federation prune — dry-run (use --apply to delete) ──`);
|
|
1490
|
+
console.log(`Would delete ${candidates.length} peer(s) older than ${opts.olderThan}:`);
|
|
1491
|
+
for (const p of candidates) {
|
|
1492
|
+
const ts = p.lastSyncAt ?? p.pairedAt ?? "never";
|
|
1493
|
+
const age = ts === "never" ? "(never synced/paired)" : `${Math.floor((Date.now() - new Date(ts).getTime()) / (24 * 60 * 60 * 1000))}d ago`;
|
|
1494
|
+
console.log(` ${p.id} ${(p.role ?? "—").padEnd(8)} lastSyncAt ${ts} (${age})`);
|
|
1495
|
+
}
|
|
1496
|
+
console.log(`Run with --apply to actually delete.`);
|
|
1497
|
+
return;
|
|
1498
|
+
}
|
|
1499
|
+
// Apply path. Delete each peer via the Harper ops API. We use the
|
|
1500
|
+
// domain-socket form when local; otherwise we fall back to the resource
|
|
1501
|
+
// DELETE which requires admin auth.
|
|
1502
|
+
let deleted = 0;
|
|
1503
|
+
let errors = 0;
|
|
1504
|
+
for (const p of candidates) {
|
|
1505
|
+
try {
|
|
1506
|
+
const res = await api("DELETE", `/FederationPeers/${encodeURIComponent(p.id)}`, undefined, baseUrl ? { baseUrl } : undefined);
|
|
1507
|
+
const ok = res?.ok ?? res?.deleted ?? true;
|
|
1508
|
+
if (ok) {
|
|
1509
|
+
deleted++;
|
|
1510
|
+
const ts = p.lastSyncAt ?? p.pairedAt ?? "never";
|
|
1511
|
+
console.log(`Deleted ${p.id} (last seen ${ts}).`);
|
|
1512
|
+
}
|
|
1513
|
+
else {
|
|
1514
|
+
errors++;
|
|
1515
|
+
console.log(`Failed to delete ${p.id}: ${JSON.stringify(res)}`);
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
catch (e) {
|
|
1519
|
+
errors++;
|
|
1520
|
+
console.log(`Failed to delete ${p.id}: ${e.message}`);
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
console.log(`${deleted} peer(s) deleted; ${errors} error(s).`);
|
|
1524
|
+
if (errors > 0)
|
|
1525
|
+
process.exit(1);
|
|
1526
|
+
});
|
|
1527
|
+
// `flair federation verify` — end-to-end roundtrip: write a tagged memory
|
|
1528
|
+
// locally, push it (bring-up has no daemon yet), probe peers for the tag.
|
|
1529
|
+
// Same class as fleet-verify / flair#988: 401/403 and unreachable are
|
|
1530
|
+
// UNVERIFIABLE (warning), a reachable missing canary still FAILs (flair#823).
|
|
1531
|
+
addSharedCredentialOptions(addSharedIdentityOption(federation
|
|
1532
|
+
.command("verify")
|
|
1533
|
+
.description("End-to-end check: write a tagged memory, push it, and verify it shows up on each peer")
|
|
1534
|
+
.option("--peer <id>", "Verify only against this peer ID (default: all hubs + spokes)")
|
|
1535
|
+
.option("--wait <seconds>", "Post-push probe window in seconds (default 60)", "60")
|
|
1536
|
+
.option("--tag <prefix>", "Memory tag prefix (default: fed-verify)", "fed-verify")
|
|
1537
|
+
.option("--port <port>", "Harper HTTP port")
|
|
1538
|
+
.option("--ops-port <port>", "Harper operations API port")
|
|
1539
|
+
.option("--target <url>", "Remote Flair URL (env: FLAIR_TARGET)")
|
|
1540
|
+
.option("--ops-target <url>", "Explicit ops API URL (env: FLAIR_OPS_TARGET; bypasses port derivation)"))).action(async (opts) => {
|
|
1541
|
+
applyAdminPassFile(opts);
|
|
1542
|
+
const target = resolveTarget(opts);
|
|
1543
|
+
const baseUrl = target ? target.replace(/\/$/, "") : undefined;
|
|
1544
|
+
const waitSeconds = Number(opts.wait) || 60;
|
|
1545
|
+
const tag = `${opts.tag}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
1546
|
+
const agentId = opts.agent || process.env.FLAIR_AGENT_ID;
|
|
1547
|
+
if (!agentId) {
|
|
1548
|
+
console.error("Error: FLAIR_AGENT_ID not set. Set it, pass --agent, or use 'flair agent default <id>'.");
|
|
1549
|
+
process.exit(1);
|
|
1550
|
+
}
|
|
1551
|
+
const result = await runFederationVerify({
|
|
1552
|
+
agentId,
|
|
1553
|
+
waitMs: waitSeconds * 1000,
|
|
1554
|
+
waitSeconds,
|
|
1555
|
+
tag,
|
|
1556
|
+
peerId: opts.peer,
|
|
1557
|
+
baseUrl,
|
|
1558
|
+
syncOpts: opts,
|
|
1559
|
+
// GET /FederationPeers is allowAdmin. applyAdminPassFile already
|
|
1560
|
+
// folded --admin-pass-file into opts.adminPass; without this the
|
|
1561
|
+
// injected api only sees { baseUrl } and flag-only creds never
|
|
1562
|
+
// reach the listing (UNVERIFIABLE, check never runs).
|
|
1563
|
+
explicitAdminPass: opts.adminPass,
|
|
1564
|
+
adminUser: opts.adminUser,
|
|
1565
|
+
}, {
|
|
1566
|
+
api,
|
|
1567
|
+
syncOnce: runFederationSyncOnce,
|
|
1568
|
+
fetch,
|
|
1569
|
+
log: (msg) => console.log(msg),
|
|
1570
|
+
error: (msg) => console.error(msg),
|
|
1571
|
+
});
|
|
1572
|
+
if (result.exitCode !== 0)
|
|
1573
|
+
process.exit(result.exitCode);
|
|
1574
|
+
});
|
|
1575
|
+
}
|