@celilo/cli 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +38 -9
- package/drizzle/0019_backup_pid.sql +18 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +5 -5
- package/schemas/system_config.json +1 -1
- package/src/api/remote-client.test.ts +62 -0
- package/src/api/serve.ts +14 -6
- package/src/cli/command-tree-parser.ts +0 -1
- package/src/cli/commands/apt-upgrade.test.ts +20 -1
- package/src/cli/commands/apt-upgrade.ts +12 -2
- package/src/cli/commands/backup-sweep.ts +62 -0
- package/src/cli/commands/events.ts +90 -0
- package/src/cli/commands/module-operations.test.ts +45 -1
- package/src/cli/commands/module-operations.ts +35 -12
- package/src/cli/commands/module-show.ts +1 -0
- package/src/cli/commands/module-update.test.ts +72 -1
- package/src/cli/commands/module-update.ts +45 -22
- package/src/cli/commands/system-audit.ts +2 -0
- package/src/cli/commands/system-migrate.test.ts +56 -0
- package/src/cli/commands/system-migrate.ts +92 -4
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +19 -0
- package/src/cli/fuel-gauge.ts +0 -1
- package/src/cli/generate-zsh-completion.ts +1 -1
- package/src/cli/index.ts +5 -1
- package/src/cli/tui/audit-state.ts +4 -0
- package/src/cli/tui/audit-tui.test.tsx +0 -1
- package/src/db/migration-status.test.ts +114 -0
- package/src/db/migration-status.ts +78 -0
- package/src/db/schema-introspection.ts +8 -1
- package/src/db/schema.ts +53 -9
- package/src/hooks/capability-loader.ts +30 -1
- package/src/ipam/allocator.ts +13 -3
- package/src/services/alerting/builtin-monitors.test.ts +42 -0
- package/src/services/alerting/builtin-monitors.ts +2 -0
- package/src/services/alerting/builtin-source.ts +15 -0
- package/src/services/audit/abandoned-operations.test.ts +73 -0
- package/src/services/audit/abandoned-operations.ts +0 -0
- package/src/services/audit/disk-space.test.ts +111 -0
- package/src/services/audit/disk-space.ts +114 -0
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +9 -0
- package/src/services/audit/types.ts +2 -0
- package/src/services/backup-create.ts +4 -4
- package/src/services/backup-in-flight-refusal.test.ts +2 -0
- package/src/services/backup-metadata.ts +4 -0
- package/src/services/backup-staging.test.ts +134 -0
- package/src/services/backup-staging.ts +192 -0
- package/src/services/backup-sweep.test.ts +68 -0
- package/src/services/backup-sweep.ts +62 -0
- package/src/services/bus-interview.ts +11 -5
- package/src/services/config-interview.ts +1 -1
- package/src/services/deploy-ansible.ts +0 -1
- package/src/services/disk-probe.test.ts +74 -0
- package/src/services/disk-probe.ts +145 -0
- package/src/services/events-daemon.test.ts +244 -0
- package/src/services/events-daemon.ts +295 -8
- package/src/services/fleet-checks.test.ts +75 -4
- package/src/services/fleet-checks.ts +97 -12
- package/src/services/interview-errors.ts +20 -0
- package/src/services/module-operations.test.ts +22 -0
- package/src/services/module-operations.ts +48 -1
- package/src/services/module-subscriptions.test.ts +39 -6
- package/src/services/module-subscriptions.ts +6 -4
- package/src/services/module-types-generator.test.ts +6 -3
- package/src/services/module-types-generator.ts +12 -7
- package/src/services/remote-responder.test.ts +70 -0
- package/src/services/remote-responder.ts +27 -10
- package/src/services/responder-probe.ts +3 -1
- package/src/services/update/orchestrator.test.ts +1 -0
- package/src/variables/context.ts +6 -1
|
@@ -39,9 +39,9 @@ function seedHeartbeat(
|
|
|
39
39
|
);
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
/** Write a supervisor unit file so readInstalledUnit(
|
|
43
|
-
function installFakeUnit(home: string): void {
|
|
44
|
-
const path = getDaemonUnitPath('linux', home,
|
|
42
|
+
/** Write a supervisor unit file so readInstalledUnit(scope) sees it. */
|
|
43
|
+
function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', systemRoot = '/'): void {
|
|
44
|
+
const path = getDaemonUnitPath('linux', home, scope, systemRoot);
|
|
45
45
|
mkdirSync(dirname(path), { recursive: true });
|
|
46
46
|
writeFileSync(path, '[Unit]\nDescription=fake\n');
|
|
47
47
|
}
|
|
@@ -114,6 +114,51 @@ describe('checkDispatcher', () => {
|
|
|
114
114
|
expect(f.detail.join(' ')).toContain('not under a supervisor');
|
|
115
115
|
});
|
|
116
116
|
|
|
117
|
+
// #610 — celilo-mgr had a dead system unit and a live user-scope unit of the
|
|
118
|
+
// SAME name. The old file-exists test called that "supervised".
|
|
119
|
+
it('fails when the running dispatcher is not the pid any installed unit supervises', () => {
|
|
120
|
+
seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 588704 });
|
|
121
|
+
installFakeUnit(home);
|
|
122
|
+
const f = checkDispatcher(bus, {
|
|
123
|
+
now: now,
|
|
124
|
+
home,
|
|
125
|
+
platform: 'linux',
|
|
126
|
+
unitMainPid: () => 3639051, // systemd supervises a different process
|
|
127
|
+
});
|
|
128
|
+
expect(f.status).toBe('fail');
|
|
129
|
+
expect(f.detail.join(' ')).toContain('not the process any installed unit supervises');
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('fails when both a user-scope and a system-scope unit are installed', () => {
|
|
133
|
+
seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
|
|
134
|
+
installFakeUnit(home);
|
|
135
|
+
installFakeUnit(home, 'system', dir);
|
|
136
|
+
const f = checkDispatcher(bus, {
|
|
137
|
+
now: now,
|
|
138
|
+
home,
|
|
139
|
+
systemRoot: dir,
|
|
140
|
+
platform: 'linux',
|
|
141
|
+
unitMainPid: () => 4242,
|
|
142
|
+
});
|
|
143
|
+
expect(f.status).toBe('fail');
|
|
144
|
+
expect(f.detail.join(' ')).toContain('same unit name, different services');
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
// Null is ignorance, not evidence: a systemd probe that fails must not
|
|
148
|
+
// manufacture an orphan report.
|
|
149
|
+
it('makes no supervision claim when the unit pid cannot be determined', () => {
|
|
150
|
+
seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
|
|
151
|
+
installFakeUnit(home);
|
|
152
|
+
const f = checkDispatcher(bus, {
|
|
153
|
+
now: now,
|
|
154
|
+
home,
|
|
155
|
+
platform: 'linux',
|
|
156
|
+
installedCodeMtimeMs: now - 2 * MINUTE,
|
|
157
|
+
unitMainPid: () => null,
|
|
158
|
+
});
|
|
159
|
+
expect(f.status).toBe('ok');
|
|
160
|
+
});
|
|
161
|
+
|
|
117
162
|
it('warns when the dispatcher started before the installed code (stale)', () => {
|
|
118
163
|
seedHeartbeat(bus, { startedAt: now - 10 * MINUTE, lastHeartbeat: now - 1000 });
|
|
119
164
|
installFakeUnit(home);
|
|
@@ -425,7 +470,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
|
|
|
425
470
|
it('is ok when every schema table is present (fresh migrated DB)', () => {
|
|
426
471
|
const f = checkSchemaDrift(db);
|
|
427
472
|
expect(f.status).toBe('ok');
|
|
428
|
-
expect(f.summary).toContain('schema tables
|
|
473
|
+
expect(f.summary).toContain('schema tables');
|
|
429
474
|
});
|
|
430
475
|
|
|
431
476
|
it('fails and names a table the running CLI expects but the DB lacks', () => {
|
|
@@ -435,6 +480,32 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
|
|
|
435
480
|
expect(f.detail.join(' ')).toContain('dns_internal_records');
|
|
436
481
|
expect(f.remediation).toContain('migrations');
|
|
437
482
|
});
|
|
483
|
+
|
|
484
|
+
// celilo#604: this is the state the rollout could not check. Every table
|
|
485
|
+
// is present, one MIGRATED COLUMN is not, and the doctor must not call
|
|
486
|
+
// that "migrations applied".
|
|
487
|
+
it('fails and names a migrated COLUMN the DB lacks, with every table present', () => {
|
|
488
|
+
db.$client.run('ALTER TABLE backups DROP COLUMN pid');
|
|
489
|
+
const f = checkSchemaDrift(db);
|
|
490
|
+
expect(f.status).toBe('fail');
|
|
491
|
+
expect(f.detail.join(' ')).toContain('backups.pid');
|
|
492
|
+
expect(f.summary).not.toContain('present');
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
it('says it checked columns, not only tables', () => {
|
|
496
|
+
const f = checkSchemaDrift(db);
|
|
497
|
+
expect(f.status).toBe('ok');
|
|
498
|
+
expect(f.summary).toContain('columns present');
|
|
499
|
+
});
|
|
500
|
+
|
|
501
|
+
it('fails when a journal migration has not been applied on this box', () => {
|
|
502
|
+
db.$client.run(
|
|
503
|
+
'DELETE FROM `__drizzle_migrations` WHERE created_at = (SELECT MAX(created_at) FROM `__drizzle_migrations`)',
|
|
504
|
+
);
|
|
505
|
+
const f = checkSchemaDrift(db);
|
|
506
|
+
expect(f.status).toBe('fail');
|
|
507
|
+
expect(f.detail.join(' ')).toContain('unapplied migration');
|
|
508
|
+
});
|
|
438
509
|
});
|
|
439
510
|
});
|
|
440
511
|
|
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
import type { Bus } from '@celilo/event-bus';
|
|
21
21
|
import { inArray } from 'drizzle-orm';
|
|
22
22
|
import { getModuleStoragePath } from '../config/paths';
|
|
23
|
-
import type
|
|
23
|
+
import { type DbClient, findMigrationsFolder } from '../db/client';
|
|
24
|
+
import { getMigrationStatus } from '../db/migration-status';
|
|
24
25
|
import { capabilities as capabilitiesTable, modules } from '../db/schema';
|
|
25
26
|
import { findSchemaDrift } from '../db/schema-introspection';
|
|
26
27
|
import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
|
|
@@ -30,7 +31,13 @@ import type { ModuleManifest } from '../manifest/schema';
|
|
|
30
31
|
const CONTROL_PLANE_MODULE = 'celilo-mgmt';
|
|
31
32
|
import { getModuleSystems } from './deployed-systems';
|
|
32
33
|
import { listDnsInternalRecords } from './dns-internal-records';
|
|
33
|
-
import {
|
|
34
|
+
import {
|
|
35
|
+
SUPERVISOR_SCOPES,
|
|
36
|
+
type SupervisorPlatform,
|
|
37
|
+
type SupervisorScope,
|
|
38
|
+
readInstalledUnit,
|
|
39
|
+
unitMainPid,
|
|
40
|
+
} from './events-daemon';
|
|
34
41
|
import { resolveSubscription } from './module-subscriptions';
|
|
35
42
|
|
|
36
43
|
/**
|
|
@@ -107,26 +114,43 @@ function worst(statuses: FleetFindingStatus[]): FleetFindingStatus {
|
|
|
107
114
|
* its schema is current, so presence is the honest signal.
|
|
108
115
|
*/
|
|
109
116
|
export function checkSchemaDrift(db: DbClient): FleetFinding {
|
|
110
|
-
const { missingTables, missingColumns, tableCount } = findSchemaDrift(db.$client);
|
|
117
|
+
const { missingTables, missingColumns, tableCount, columnCount } = findSchemaDrift(db.$client);
|
|
111
118
|
|
|
112
119
|
const detail: string[] = [];
|
|
113
120
|
if (missingTables.length > 0) detail.push(`missing table(s): ${missingTables.join(', ')}`);
|
|
114
121
|
if (missingColumns.length > 0) detail.push(`missing column(s): ${missingColumns.join(', ')}`);
|
|
122
|
+
|
|
123
|
+
// Also name unapplied migrations. Presence is the honest signal for schema
|
|
124
|
+
// objects, but a migration can carry an index or a data fix that presence
|
|
125
|
+
// can't see — and an operator reading "migrations applied" deserves to know
|
|
126
|
+
// when some aren't (celilo#604). Best-effort: an install layout where the
|
|
127
|
+
// journal can't be found must not fail the check.
|
|
128
|
+
let pending: string[] = [];
|
|
129
|
+
try {
|
|
130
|
+
pending = getMigrationStatus(db.$client, findMigrationsFolder()).pending;
|
|
131
|
+
} catch {
|
|
132
|
+
// No journal reachable — the presence check above still stands.
|
|
133
|
+
}
|
|
134
|
+
if (pending.length > 0) detail.push(`unapplied migration(s): ${pending.join(', ')}`);
|
|
135
|
+
|
|
115
136
|
const status: FleetFindingStatus = detail.length > 0 ? 'fail' : 'ok';
|
|
116
137
|
|
|
117
138
|
return {
|
|
118
139
|
id: 'schema',
|
|
119
140
|
title: 'database schema matches the running CLI (migrations applied)',
|
|
120
141
|
status,
|
|
142
|
+
// Say tables AND columns: "all 35 tables present" reads as though columns
|
|
143
|
+
// went unchecked, which is what sent a rollout to sqlite3 over SSH to
|
|
144
|
+
// confirm a column migration the doctor had in fact already verified.
|
|
121
145
|
summary:
|
|
122
146
|
status === 'ok'
|
|
123
|
-
? `all ${tableCount} schema tables present`
|
|
147
|
+
? `all ${tableCount} schema tables and ${columnCount} columns present, no unapplied migrations`
|
|
124
148
|
: 'database schema is behind the running CLI — migrations not applied',
|
|
125
149
|
detail,
|
|
126
150
|
remediation:
|
|
127
151
|
status === 'ok'
|
|
128
152
|
? null
|
|
129
|
-
: 'run `celilo system migrate` to apply pending migrations on this box — see ISS-0100',
|
|
153
|
+
: 'run `celilo system migrate` to apply pending migrations on this box (`celilo system migrate --status` names them) — see ISS-0100',
|
|
130
154
|
autoFixable: false,
|
|
131
155
|
};
|
|
132
156
|
}
|
|
@@ -152,6 +176,14 @@ export interface DispatcherCheckOptions {
|
|
|
152
176
|
/** Override for readInstalledUnit — tests point this at a temp home. */
|
|
153
177
|
home?: string;
|
|
154
178
|
platform?: SupervisorPlatform;
|
|
179
|
+
/** Prefix for system-scope unit paths. Test seam — see getDaemonUnitPath. */
|
|
180
|
+
systemRoot?: string;
|
|
181
|
+
/**
|
|
182
|
+
* Which pid each scope's unit supervises. Injected so the check is testable
|
|
183
|
+
* without systemd. Returning null means "can't tell" — the check then makes
|
|
184
|
+
* no supervision claim rather than guessing.
|
|
185
|
+
*/
|
|
186
|
+
unitMainPid?: (scope: SupervisorScope) => number | null;
|
|
155
187
|
}
|
|
156
188
|
|
|
157
189
|
/**
|
|
@@ -208,16 +240,69 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
|
|
|
208
240
|
remediations.push('`celilo events repair` to sweep stuck deliveries');
|
|
209
241
|
}
|
|
210
242
|
|
|
211
|
-
// (
|
|
212
|
-
//
|
|
213
|
-
//
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
243
|
+
// (1b) sole — a duplicate dispatcher can no longer START (it refuses), but one
|
|
244
|
+
// stranded before that shipped keeps running, and it makes every other check
|
|
245
|
+
// here ambiguous: `hb` is whichever of them wrote last. Fail rather than warn —
|
|
246
|
+
// celilo-mgr ran two for 40 days precisely because nothing reported it (#580).
|
|
247
|
+
if (health.dispatcherCount > 1) {
|
|
248
|
+
statuses.push('fail');
|
|
249
|
+
const pids = health.dispatchers.map((d) => d.pid).join(', ');
|
|
250
|
+
detail.push(
|
|
251
|
+
`${health.dispatcherCount} dispatchers are live on this bus (pids ${pids}) — only one may run`,
|
|
252
|
+
);
|
|
253
|
+
remediations.push(
|
|
254
|
+
'stop the unsupervised one: compare `systemctl show celilo-events.service -p MainPID` against those pids and kill the pid systemd does not own',
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// (2) supervised — not just "a unit file exists on disk", but "the process
|
|
259
|
+
// that is actually running IS the one an installed unit supervises".
|
|
260
|
+
//
|
|
261
|
+
// The file-exists test this replaces reported green on celilo-mgr while the
|
|
262
|
+
// system unit was dead and a user-scope unit of the SAME NAME served
|
|
263
|
+
// production (#610). A check whose entire job is catching an unsupervised
|
|
264
|
+
// dispatcher cannot be satisfied by a file nobody is running.
|
|
265
|
+
const installedScopes = SUPERVISOR_SCOPES.filter(
|
|
266
|
+
(scope) =>
|
|
267
|
+
readInstalledUnit({
|
|
268
|
+
scope,
|
|
269
|
+
home: opts.home,
|
|
270
|
+
platform: opts.platform,
|
|
271
|
+
systemRoot: opts.systemRoot,
|
|
272
|
+
}).exists,
|
|
273
|
+
);
|
|
274
|
+
if (installedScopes.length === 0) {
|
|
218
275
|
statuses.push('warn');
|
|
219
276
|
detail.push('not under a supervisor unit — will not survive a reboot (orphan process)');
|
|
220
277
|
remediations.push('`celilo events install-daemon` then enable the unit so it is supervised');
|
|
278
|
+
} else {
|
|
279
|
+
if (installedScopes.length > 1) {
|
|
280
|
+
statuses.push('fail');
|
|
281
|
+
detail.push(
|
|
282
|
+
'both a user-scope AND a system-scope unit are installed — same unit name, different services; ' +
|
|
283
|
+
'one will lose the race on every boot and retry forever',
|
|
284
|
+
);
|
|
285
|
+
remediations.push(
|
|
286
|
+
'keep exactly one: `celilo events uninstall-daemon` (user) or `celilo events uninstall-daemon --system`, and disable it in systemd',
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
// Only accuse when systemd actually answered. A null probe is ignorance,
|
|
290
|
+
// not evidence of an orphan.
|
|
291
|
+
const probe =
|
|
292
|
+
opts.unitMainPid ?? ((scope: SupervisorScope) => unitMainPid(scope, opts.platform));
|
|
293
|
+
const supervisedPids = installedScopes
|
|
294
|
+
.map((scope) => ({ scope, pid: probe(scope) }))
|
|
295
|
+
.filter((entry): entry is { scope: SupervisorScope; pid: number } => entry.pid !== null);
|
|
296
|
+
if (supervisedPids.length > 0 && !supervisedPids.some((entry) => entry.pid === hb.pid)) {
|
|
297
|
+
statuses.push('fail');
|
|
298
|
+
detail.push(
|
|
299
|
+
`the running dispatcher (pid ${hb.pid}) is not the process any installed unit supervises ` +
|
|
300
|
+
`(${supervisedPids.map((e) => `${e.scope}=${e.pid}`).join(', ')}) — restarting the unit will not restart it`,
|
|
301
|
+
);
|
|
302
|
+
remediations.push(
|
|
303
|
+
'stop the unsupervised process and let the unit own the dispatcher, or reinstall the unit for the scope that is actually running it',
|
|
304
|
+
);
|
|
305
|
+
}
|
|
221
306
|
}
|
|
222
307
|
|
|
223
308
|
// (3) current — started before the installed code was last written ⇒
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one error type that means "this interview question could not be
|
|
3
|
+
* answered". Its own module so `responder-probe` (no responder listening) and
|
|
4
|
+
* `bus-interview` (a responder replied that it couldn't decide) can both throw
|
|
5
|
+
* it without importing each other.
|
|
6
|
+
*
|
|
7
|
+
* Callers catch this to distinguish an *unanswered* question from an answered
|
|
8
|
+
* one — the distinction `module update` conflated when it reported a breaking
|
|
9
|
+
* update as "operator declined" that no operator had ever seen.
|
|
10
|
+
*/
|
|
11
|
+
export class InterviewUnansweredError extends Error {
|
|
12
|
+
constructor(
|
|
13
|
+
/** The interview event type that went unanswered, e.g. `interview.required.<scope>.<key>`. */
|
|
14
|
+
readonly queryType: string,
|
|
15
|
+
message: string,
|
|
16
|
+
) {
|
|
17
|
+
super(message);
|
|
18
|
+
this.name = 'InterviewUnansweredError';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
@@ -7,9 +7,12 @@ import { closeDb } from '../db/client';
|
|
|
7
7
|
import { runMigrations } from '../db/migrate';
|
|
8
8
|
import {
|
|
9
9
|
InFlightError,
|
|
10
|
+
OPERATIONS_SWEEP_PATTERN,
|
|
11
|
+
OPERATIONS_SWEEP_SUBSCRIBER,
|
|
10
12
|
OPERATION_TTL_MS,
|
|
11
13
|
checkInFlight,
|
|
12
14
|
completeOperation,
|
|
15
|
+
ensureOperationsSweepSubscriber,
|
|
13
16
|
failOperation,
|
|
14
17
|
isPidRunnable,
|
|
15
18
|
refuseIfInFlight,
|
|
@@ -213,3 +216,22 @@ describe('module-operations', () => {
|
|
|
213
216
|
});
|
|
214
217
|
});
|
|
215
218
|
});
|
|
219
|
+
|
|
220
|
+
describe('ensureOperationsSweepSubscriber', () => {
|
|
221
|
+
it('registers the hourly reclaim against the existing clear command', () => {
|
|
222
|
+
const calls: Array<{ name: string; pattern: string; handler: string; registeredBy?: string }> =
|
|
223
|
+
[];
|
|
224
|
+
ensureOperationsSweepSubscriber({ subscribe: (options) => calls.push(options) });
|
|
225
|
+
|
|
226
|
+
expect(calls).toEqual([
|
|
227
|
+
{
|
|
228
|
+
name: OPERATIONS_SWEEP_SUBSCRIBER,
|
|
229
|
+
pattern: OPERATIONS_SWEEP_PATTERN,
|
|
230
|
+
handler: 'celilo module operations clear',
|
|
231
|
+
registeredBy: 'celilo-module-operations',
|
|
232
|
+
},
|
|
233
|
+
]);
|
|
234
|
+
// Finer than the TTL, so a wedged row never survives long.
|
|
235
|
+
expect(OPERATIONS_SWEEP_PATTERN).toBe('timer.tick.1h');
|
|
236
|
+
});
|
|
237
|
+
});
|
|
@@ -26,7 +26,8 @@
|
|
|
26
26
|
* lock for 20 days and blocked every backup on the fleet.
|
|
27
27
|
*
|
|
28
28
|
* Stale rows are ignored rather than deleted; `celilo module operations`
|
|
29
|
-
* lists them and `... clear` sweeps them
|
|
29
|
+
* lists them and `... clear` sweeps them — on an hourly bus tick, not
|
|
30
|
+
* only when a human remembers (see `ensureOperationsSweepSubscriber`).
|
|
30
31
|
*/
|
|
31
32
|
|
|
32
33
|
import { spawnSync } from 'node:child_process';
|
|
@@ -119,6 +120,52 @@ export function isPidRunnable(pid: number): boolean {
|
|
|
119
120
|
return state !== 'T' && state !== 'Z';
|
|
120
121
|
}
|
|
121
122
|
|
|
123
|
+
/**
|
|
124
|
+
* The `errorMessage` written when a row is released as abandoned.
|
|
125
|
+
*
|
|
126
|
+
* Load-bearing, not cosmetic: releasing marks rows `failed` rather than
|
|
127
|
+
* deleting them, and this exact string is what later distinguishes "the
|
|
128
|
+
* operation reported a failure" from "the operation never reported
|
|
129
|
+
* anything and the sweep reclaimed it". The abandoned-operations audit
|
|
130
|
+
* counts rows by it (`services/audit/abandoned-operations.ts`).
|
|
131
|
+
*/
|
|
132
|
+
export const ABANDONED_RELEASE_MESSAGE = 'abandoned — released by "celilo module operations clear"';
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Reclaim abandoned rows on a schedule instead of when a human remembers.
|
|
136
|
+
*
|
|
137
|
+
* Registered as an ordinary bus subscriber whose handler is the existing
|
|
138
|
+
* `celilo module operations clear`, exactly like the backup sweep
|
|
139
|
+
* (`services/backup-sweep.ts`) — no new command and no new scheduler.
|
|
140
|
+
* Hourly is far finer than the two-hour TTL, so a wedge never survives
|
|
141
|
+
* long, and clearing is idempotent: a pass with nothing abandoned is a
|
|
142
|
+
* single read.
|
|
143
|
+
*
|
|
144
|
+
* `clear` without `--all` only touches rows that `checkInFlight` already
|
|
145
|
+
* ignores, so the sweep can never release a lock a live operation holds.
|
|
146
|
+
*/
|
|
147
|
+
export const OPERATIONS_SWEEP_SUBSCRIBER = 'celilo-operations-sweep';
|
|
148
|
+
export const OPERATIONS_SWEEP_PATTERN = 'timer.tick.1h';
|
|
149
|
+
|
|
150
|
+
export interface SubscriberRegistrar {
|
|
151
|
+
subscribe(options: {
|
|
152
|
+
name: string;
|
|
153
|
+
pattern: string;
|
|
154
|
+
handler: string;
|
|
155
|
+
registeredBy?: string;
|
|
156
|
+
}): unknown;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Idempotent: `bus.subscribe` upserts by name. */
|
|
160
|
+
export function ensureOperationsSweepSubscriber(bus: SubscriberRegistrar): void {
|
|
161
|
+
bus.subscribe({
|
|
162
|
+
name: OPERATIONS_SWEEP_SUBSCRIBER,
|
|
163
|
+
pattern: OPERATIONS_SWEEP_PATTERN,
|
|
164
|
+
handler: 'celilo module operations clear',
|
|
165
|
+
registeredBy: 'celilo-module-operations',
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
|
|
122
169
|
export interface InFlightConflict {
|
|
123
170
|
operation: ModuleOperation;
|
|
124
171
|
/** A short, operator-readable description: "deploy of homebridge (pid 12345)". */
|
|
@@ -173,7 +173,7 @@ describe('register / unregister roundtrip', () => {
|
|
|
173
173
|
try {
|
|
174
174
|
const row = bus.db
|
|
175
175
|
.query<{ name: string; pattern: string; handler: string }, []>(
|
|
176
|
-
|
|
176
|
+
"SELECT name, pattern, handler FROM subscribers WHERE name = 'celilo-backup-sweep'",
|
|
177
177
|
)
|
|
178
178
|
.get();
|
|
179
179
|
expect(row).toEqual({
|
|
@@ -186,6 +186,28 @@ describe('register / unregister roundtrip', () => {
|
|
|
186
186
|
}
|
|
187
187
|
});
|
|
188
188
|
|
|
189
|
+
// Any module can hold the operation lock, so registering ANY module — even
|
|
190
|
+
// one with no subscriptions and no backup hook — is enough to arm the sweep
|
|
191
|
+
// that reclaims abandoned rows (#581).
|
|
192
|
+
it('arms the abandoned-operations sweep for any module', () => {
|
|
193
|
+
registerModuleSubscriptions(baseManifest({}), '/p');
|
|
194
|
+
|
|
195
|
+
const bus = openBus({ dbPath, events: defineEvents({}) });
|
|
196
|
+
try {
|
|
197
|
+
const row = bus.db
|
|
198
|
+
.query<{ pattern: string; handler: string }, []>(
|
|
199
|
+
"SELECT pattern, handler FROM subscribers WHERE name = 'celilo-operations-sweep'",
|
|
200
|
+
)
|
|
201
|
+
.get();
|
|
202
|
+
expect(row).toEqual({
|
|
203
|
+
pattern: 'timer.tick.1h',
|
|
204
|
+
handler: 'celilo module operations clear',
|
|
205
|
+
});
|
|
206
|
+
} finally {
|
|
207
|
+
bus.close();
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
|
|
189
211
|
it('registers each subscription as a row, names scoped to module id', () => {
|
|
190
212
|
const result = registerModuleSubscriptions(
|
|
191
213
|
baseManifest({
|
|
@@ -210,7 +232,7 @@ describe('register / unregister roundtrip', () => {
|
|
|
210
232
|
try {
|
|
211
233
|
const rows = bus.db
|
|
212
234
|
.query<{ name: string; pattern: string; handler: string }, []>(
|
|
213
|
-
|
|
235
|
+
"SELECT name, pattern, handler FROM subscribers WHERE name LIKE '%.%' ORDER BY name",
|
|
214
236
|
)
|
|
215
237
|
.all();
|
|
216
238
|
expect(rows).toEqual([
|
|
@@ -243,10 +265,14 @@ describe('register / unregister roundtrip', () => {
|
|
|
243
265
|
const bus = openBus({ dbPath, events: defineEvents({}) });
|
|
244
266
|
try {
|
|
245
267
|
const rows = bus.db
|
|
246
|
-
.query<{ count: number }, []>(
|
|
268
|
+
.query<{ count: number }, []>(
|
|
269
|
+
"SELECT COUNT(*) AS count FROM subscribers WHERE name LIKE '%.%'",
|
|
270
|
+
)
|
|
247
271
|
.get();
|
|
248
272
|
expect(rows?.count).toBe(1);
|
|
249
|
-
const row = bus.db
|
|
273
|
+
const row = bus.db
|
|
274
|
+
.query<{ handler: string }, []>("SELECT handler FROM subscribers WHERE name LIKE '%.%'")
|
|
275
|
+
.get();
|
|
250
276
|
expect(row?.handler).toBe('echo second');
|
|
251
277
|
} finally {
|
|
252
278
|
bus.close();
|
|
@@ -279,7 +305,9 @@ describe('register / unregister roundtrip', () => {
|
|
|
279
305
|
const bus = openBus({ dbPath, events: defineEvents({}) });
|
|
280
306
|
try {
|
|
281
307
|
const rows = bus.db
|
|
282
|
-
.query<{ name: string }, []>(
|
|
308
|
+
.query<{ name: string }, []>(
|
|
309
|
+
"SELECT name FROM subscribers WHERE name LIKE '%.%' ORDER BY name",
|
|
310
|
+
)
|
|
283
311
|
.all();
|
|
284
312
|
expect(rows).toEqual([{ name: 'authentik.a' }]);
|
|
285
313
|
} finally {
|
|
@@ -337,8 +365,13 @@ describe('resyncAllSubscriptions (ISS-0088)', () => {
|
|
|
337
365
|
function subscriberNames(): string[] {
|
|
338
366
|
const bus = openBus({ dbPath: busPath, events: defineEvents({}) });
|
|
339
367
|
try {
|
|
368
|
+
// Module subscriptions are dot-scoped (`<module-id>.<sub-name>`); celilo's
|
|
369
|
+
// own housekeeping subscribers (the backup and operations sweeps) are not,
|
|
370
|
+
// and are not what these tests are about.
|
|
340
371
|
return bus.db
|
|
341
|
-
.query<{ name: string }, []>(
|
|
372
|
+
.query<{ name: string }, []>(
|
|
373
|
+
"SELECT name FROM subscribers WHERE name LIKE '%.%' ORDER BY name",
|
|
374
|
+
)
|
|
342
375
|
.all()
|
|
343
376
|
.map((r) => r.name);
|
|
344
377
|
} finally {
|
|
@@ -22,6 +22,7 @@ import { getDb } from '../db/client';
|
|
|
22
22
|
import { modules } from '../db/schema';
|
|
23
23
|
import type { ModuleManifest, ModuleSubscription } from '../manifest/schema';
|
|
24
24
|
import { ensureBackupSweepSubscriber } from './backup-sweep';
|
|
25
|
+
import { ensureOperationsSweepSubscriber } from './module-operations';
|
|
25
26
|
|
|
26
27
|
/**
|
|
27
28
|
* The bus is opened by the celilo CLI without an event registry — the
|
|
@@ -82,9 +83,6 @@ function resolveHandler(sub: ModuleSubscription, moduleId: string, modulePath: s
|
|
|
82
83
|
/**
|
|
83
84
|
* Register all of a module's subscriptions on the bus. Idempotent —
|
|
84
85
|
* re-running with the same manifest updates existing rows in place.
|
|
85
|
-
*
|
|
86
|
-
* If the module's manifest declares no subscriptions, this is a
|
|
87
|
-
* cheap no-op (the bus DB isn't even touched).
|
|
88
86
|
*/
|
|
89
87
|
export function registerModuleSubscriptions(
|
|
90
88
|
manifest: ModuleManifest,
|
|
@@ -92,10 +90,14 @@ export function registerModuleSubscriptions(
|
|
|
92
90
|
): { registered: number } {
|
|
93
91
|
const subs = manifest.subscriptions ?? [];
|
|
94
92
|
const backupSweep = Boolean(manifest.hooks?.on_backup);
|
|
95
|
-
if (subs.length === 0 && !backupSweep) return { registered: 0 };
|
|
96
93
|
|
|
97
94
|
const bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
|
|
98
95
|
try {
|
|
96
|
+
// Unconditional: every module can hold the operation lock (a deploy at
|
|
97
|
+
// minimum), so the first module on a fleet is what arms the sweep that
|
|
98
|
+
// reclaims abandoned rows. Idempotent.
|
|
99
|
+
ensureOperationsSweepSubscriber(bus);
|
|
100
|
+
|
|
99
101
|
// A module that can be backed up is also what switches the scheduled
|
|
100
102
|
// backup sweep on. Registering here rather than at system init means the
|
|
101
103
|
// sweep appears the moment the fleet has something to back up, and — since
|
|
@@ -63,17 +63,20 @@ describe('variableTypeToTs', () => {
|
|
|
63
63
|
});
|
|
64
64
|
|
|
65
65
|
describe('generateModuleTypes', () => {
|
|
66
|
-
test('emits a file header and
|
|
66
|
+
test('emits a file header and a no-keys alias for a manifest with no variables', () => {
|
|
67
67
|
const out = generateModuleTypes(baseManifest({ id: 'empty', name: 'Empty Module' }));
|
|
68
68
|
expect(out).toContain('// Generated from manifest.yml');
|
|
69
69
|
expect(out).toContain('Do not edit by hand');
|
|
70
|
-
|
|
70
|
+
// `= {}` means "any non-nullish value", which is the opposite of an empty
|
|
71
|
+
// config surface — and biome bans it (lint/complexity/noBannedTypes).
|
|
72
|
+
expect(out).toContain('export type EmptyConfig = Record<string, never>;');
|
|
73
|
+
expect(out).not.toContain('export type EmptyConfig = {');
|
|
71
74
|
expect(out).toContain('(No variables declared — module has no typed config surface)');
|
|
72
75
|
});
|
|
73
76
|
|
|
74
77
|
test('produces the right type-alias name from a kebab-case module ID', () => {
|
|
75
78
|
const out = generateModuleTypes(baseManifest({ id: 'dns-external', name: 'DNS External' }));
|
|
76
|
-
expect(out).toContain('export type DnsExternalConfig =
|
|
79
|
+
expect(out).toContain('export type DnsExternalConfig =');
|
|
77
80
|
});
|
|
78
81
|
|
|
79
82
|
test('renders required fields as non-optional', () => {
|
|
@@ -145,8 +145,6 @@ export function generateModuleTypes(manifest: ModuleManifest): string {
|
|
|
145
145
|
lines.push(' * gives type aliases an implicit index signature but withholds one from');
|
|
146
146
|
lines.push(' * interfaces (which can be declaration-merged). See v2/issues.');
|
|
147
147
|
lines.push(' */');
|
|
148
|
-
lines.push(`export type ${typeName} = {`);
|
|
149
|
-
|
|
150
148
|
const ownsFields: string[] = [];
|
|
151
149
|
const importsFields: string[] = [];
|
|
152
150
|
|
|
@@ -172,6 +170,17 @@ export function generateModuleTypes(manifest: ModuleManifest): string {
|
|
|
172
170
|
importsFields.push(...rendered);
|
|
173
171
|
}
|
|
174
172
|
|
|
173
|
+
if (ownsFields.length === 0 && importsFields.length === 0) {
|
|
174
|
+
// `= {}` is the banned "any non-nullish value" type, not "no keys" — and an
|
|
175
|
+
// empty config surface means exactly no keys.
|
|
176
|
+
lines.push('// (No variables declared — module has no typed config surface)');
|
|
177
|
+
lines.push(`export type ${typeName} = Record<string, never>;`);
|
|
178
|
+
lines.push('');
|
|
179
|
+
return lines.map((line) => line.trimEnd()).join('\n');
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
lines.push(`export type ${typeName} = {`);
|
|
183
|
+
|
|
175
184
|
if (ownsFields.length > 0) {
|
|
176
185
|
lines.push(' // Module-owned variables (from variables.owns)');
|
|
177
186
|
lines.push(...ownsFields);
|
|
@@ -183,12 +192,8 @@ export function generateModuleTypes(manifest: ModuleManifest): string {
|
|
|
183
192
|
lines.push(...importsFields);
|
|
184
193
|
}
|
|
185
194
|
|
|
186
|
-
if (ownsFields.length === 0 && importsFields.length === 0) {
|
|
187
|
-
lines.push(' // (No variables declared — module has no typed config surface)');
|
|
188
|
-
}
|
|
189
|
-
|
|
190
195
|
lines.push('};');
|
|
191
196
|
lines.push('');
|
|
192
197
|
|
|
193
|
-
return lines.join('\n');
|
|
198
|
+
return lines.map((line) => line.trimEnd()).join('\n');
|
|
194
199
|
}
|
|
@@ -76,3 +76,73 @@ test('answers responder.probe with kind daemon', async () => {
|
|
|
76
76
|
responder.close();
|
|
77
77
|
}
|
|
78
78
|
}, 15_000);
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* When the client can't reach a decider it says so. The responder must relay
|
|
82
|
+
* that as an error reply — the waiting command then fails loudly instead of the
|
|
83
|
+
* question quietly resolving to `defaultValue`, which is how a breaking update
|
|
84
|
+
* came to be recorded as "operator declined".
|
|
85
|
+
*/
|
|
86
|
+
test('an ask that cannot be answered replies with an error, not the default', async () => {
|
|
87
|
+
const responder = startRemoteResponder({
|
|
88
|
+
busDbPath,
|
|
89
|
+
ask: async () => {
|
|
90
|
+
throw new Error("stdin isn't a terminal and no answer was pre-staged");
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
|
|
95
|
+
try {
|
|
96
|
+
const replies = (await bus.query(
|
|
97
|
+
'interview.required.module-upgrade:iptables.apply_breaking' as never,
|
|
98
|
+
{
|
|
99
|
+
scope: 'module-upgrade:iptables',
|
|
100
|
+
key: 'apply_breaking',
|
|
101
|
+
kind: 'confirm',
|
|
102
|
+
message: 'Apply breaking update for iptables?',
|
|
103
|
+
required: true,
|
|
104
|
+
defaultValue: 'false',
|
|
105
|
+
} as never,
|
|
106
|
+
{ timeoutMs: 8000, pollIntervalMs: 100, expect: 'first' } as never,
|
|
107
|
+
)) as BusEvent[];
|
|
108
|
+
|
|
109
|
+
expect(replies).toHaveLength(1);
|
|
110
|
+
const payload = replies[0].payload as { value?: unknown; error?: string };
|
|
111
|
+
expect(payload.value).toBeUndefined();
|
|
112
|
+
expect(payload.error).toContain('terminal');
|
|
113
|
+
} finally {
|
|
114
|
+
bus.close();
|
|
115
|
+
responder.close();
|
|
116
|
+
}
|
|
117
|
+
}, 15_000);
|
|
118
|
+
|
|
119
|
+
test('forwards the question scope/key so a client can pre-stage an answer', async () => {
|
|
120
|
+
const asked: WireInterview[] = [];
|
|
121
|
+
const responder = startRemoteResponder({
|
|
122
|
+
busDbPath,
|
|
123
|
+
ask: async (iv) => {
|
|
124
|
+
asked.push(iv);
|
|
125
|
+
return true;
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
|
|
130
|
+
try {
|
|
131
|
+
await bus.query(
|
|
132
|
+
'interview.required.module-upgrade:iptables.apply_breaking' as never,
|
|
133
|
+
{
|
|
134
|
+
scope: 'module-upgrade:iptables',
|
|
135
|
+
key: 'apply_breaking',
|
|
136
|
+
kind: 'confirm',
|
|
137
|
+
message: 'Apply breaking update for iptables?',
|
|
138
|
+
required: true,
|
|
139
|
+
} as never,
|
|
140
|
+
{ timeoutMs: 8000, pollIntervalMs: 100, expect: 'first' } as never,
|
|
141
|
+
);
|
|
142
|
+
expect(asked[0].scope).toBe('module-upgrade:iptables');
|
|
143
|
+
expect(asked[0].key).toBe('apply_breaking');
|
|
144
|
+
} finally {
|
|
145
|
+
bus.close();
|
|
146
|
+
responder.close();
|
|
147
|
+
}
|
|
148
|
+
}, 15_000);
|