@celilo/cli 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +38 -9
- package/drizzle/0019_backup_pid.sql +18 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +5 -5
- package/schemas/system_config.json +1 -1
- package/src/api/remote-client.test.ts +62 -0
- package/src/api/serve.ts +14 -6
- package/src/cli/command-tree-parser.ts +0 -1
- package/src/cli/commands/apt-upgrade.test.ts +20 -1
- package/src/cli/commands/apt-upgrade.ts +12 -2
- package/src/cli/commands/backup-sweep.ts +62 -0
- package/src/cli/commands/events.ts +90 -0
- package/src/cli/commands/module-operations.test.ts +45 -1
- package/src/cli/commands/module-operations.ts +35 -12
- package/src/cli/commands/module-show.ts +1 -0
- package/src/cli/commands/module-update.test.ts +72 -1
- package/src/cli/commands/module-update.ts +45 -22
- package/src/cli/commands/system-audit.ts +2 -0
- package/src/cli/commands/system-migrate.test.ts +56 -0
- package/src/cli/commands/system-migrate.ts +92 -4
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +19 -0
- package/src/cli/fuel-gauge.ts +0 -1
- package/src/cli/generate-zsh-completion.ts +1 -1
- package/src/cli/index.ts +5 -1
- package/src/cli/tui/audit-state.ts +4 -0
- package/src/cli/tui/audit-tui.test.tsx +0 -1
- package/src/db/migration-status.test.ts +114 -0
- package/src/db/migration-status.ts +78 -0
- package/src/db/schema-introspection.ts +8 -1
- package/src/db/schema.ts +53 -9
- package/src/hooks/capability-loader.ts +30 -1
- package/src/ipam/allocator.ts +13 -3
- package/src/services/alerting/builtin-monitors.test.ts +42 -0
- package/src/services/alerting/builtin-monitors.ts +2 -0
- package/src/services/alerting/builtin-source.ts +15 -0
- package/src/services/audit/abandoned-operations.test.ts +73 -0
- package/src/services/audit/abandoned-operations.ts +0 -0
- package/src/services/audit/disk-space.test.ts +111 -0
- package/src/services/audit/disk-space.ts +114 -0
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +9 -0
- package/src/services/audit/types.ts +2 -0
- package/src/services/backup-create.ts +4 -4
- package/src/services/backup-in-flight-refusal.test.ts +2 -0
- package/src/services/backup-metadata.ts +4 -0
- package/src/services/backup-staging.test.ts +134 -0
- package/src/services/backup-staging.ts +192 -0
- package/src/services/backup-sweep.test.ts +68 -0
- package/src/services/backup-sweep.ts +62 -0
- package/src/services/bus-interview.ts +11 -5
- package/src/services/config-interview.ts +1 -1
- package/src/services/deploy-ansible.ts +0 -1
- package/src/services/disk-probe.test.ts +74 -0
- package/src/services/disk-probe.ts +145 -0
- package/src/services/events-daemon.test.ts +244 -0
- package/src/services/events-daemon.ts +295 -8
- package/src/services/fleet-checks.test.ts +75 -4
- package/src/services/fleet-checks.ts +97 -12
- package/src/services/interview-errors.ts +20 -0
- package/src/services/module-operations.test.ts +22 -0
- package/src/services/module-operations.ts +48 -1
- package/src/services/module-subscriptions.test.ts +39 -6
- package/src/services/module-subscriptions.ts +6 -4
- package/src/services/module-types-generator.test.ts +6 -3
- package/src/services/module-types-generator.ts +12 -7
- package/src/services/remote-responder.test.ts +70 -0
- package/src/services/remote-responder.ts +27 -10
- package/src/services/responder-probe.ts +3 -1
- package/src/services/update/orchestrator.test.ts +1 -0
- package/src/variables/context.ts +6 -1
|
@@ -6,7 +6,7 @@ import { eq } from 'drizzle-orm';
|
|
|
6
6
|
import { closeDb, getDb } from '../../db/client';
|
|
7
7
|
import { runMigrations } from '../../db/migrate';
|
|
8
8
|
import { moduleOperations } from '../../db/schema';
|
|
9
|
-
import { OPERATION_TTL_MS } from '../../services/module-operations';
|
|
9
|
+
import { ABANDONED_RELEASE_MESSAGE, OPERATION_TTL_MS } from '../../services/module-operations';
|
|
10
10
|
import { handleModuleOperations } from './module-operations';
|
|
11
11
|
|
|
12
12
|
describe('celilo module operations', () => {
|
|
@@ -85,6 +85,50 @@ describe('celilo module operations', () => {
|
|
|
85
85
|
expect(statusOf('expired')).toBe('in_progress');
|
|
86
86
|
});
|
|
87
87
|
|
|
88
|
+
// The recurrence gate for #581: the sweep runs on every hourly tick, so an
|
|
89
|
+
// abandoned row must be reclaimed once and then stop being work. Before
|
|
90
|
+
// this, rows only ever accumulated — 85 of them, the oldest 62 days old.
|
|
91
|
+
it('reclaims an abandoned row and does not re-collect it on the next sweep', () => {
|
|
92
|
+
insert('expired', process.pid, OPERATION_TTL_MS + 60_000);
|
|
93
|
+
|
|
94
|
+
const first = handleModuleOperations(['clear'], {});
|
|
95
|
+
if (!first.success) throw new Error(`expected success, got: ${first.error}`);
|
|
96
|
+
expect(first.message).toContain('Released 1');
|
|
97
|
+
expect(statusOf('expired')).toBe('failed');
|
|
98
|
+
|
|
99
|
+
const second = handleModuleOperations(['clear'], {});
|
|
100
|
+
if (!second.success) throw new Error(`expected success, got: ${second.error}`);
|
|
101
|
+
expect(second.message).toContain('no operations in progress');
|
|
102
|
+
|
|
103
|
+
// Reclaimed, not deleted: the released row is the evidence the
|
|
104
|
+
// abandoned-operations audit reads.
|
|
105
|
+
const row = getDb()
|
|
106
|
+
.select()
|
|
107
|
+
.from(moduleOperations)
|
|
108
|
+
.where(eq(moduleOperations.id, 'expired'))
|
|
109
|
+
.get();
|
|
110
|
+
expect(row?.errorMessage).toBe(ABANDONED_RELEASE_MESSAGE);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it('hides abandoned rows from list by default and shows them with --abandoned', () => {
|
|
114
|
+
insert('expired', process.pid, OPERATION_TTL_MS + 60_000);
|
|
115
|
+
|
|
116
|
+
const lines: string[] = [];
|
|
117
|
+
const original = console.log;
|
|
118
|
+
console.log = (msg?: unknown) => lines.push(String(msg));
|
|
119
|
+
try {
|
|
120
|
+
handleModuleOperations(['list'], {});
|
|
121
|
+
expect(lines.join('\n')).not.toContain('pid');
|
|
122
|
+
expect(lines.join('\n')).toContain('1 abandoned row(s) hidden');
|
|
123
|
+
|
|
124
|
+
lines.length = 0;
|
|
125
|
+
handleModuleOperations(['list'], { abandoned: true });
|
|
126
|
+
expect(lines.join('\n')).toContain('abandoned (expired)');
|
|
127
|
+
} finally {
|
|
128
|
+
console.log = original;
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
|
|
88
132
|
it('rejects an unknown action rather than silently listing', () => {
|
|
89
133
|
const result = handleModuleOperations(['nuke'], {});
|
|
90
134
|
if (result.success) throw new Error('expected an unknown action to fail');
|
|
@@ -10,12 +10,22 @@
|
|
|
10
10
|
*
|
|
11
11
|
* `clear` marks rows failed rather than deleting them — the history of
|
|
12
12
|
* what was abandoned, and when, is worth more than a tidy table.
|
|
13
|
+
*
|
|
14
|
+
* That is load-bearing now, not merely tidy: `clear` runs hourly off the
|
|
15
|
+
* bus, and `services/audit/abandoned-operations.ts` reads exactly these
|
|
16
|
+
* released rows to notice that one module's backup is being killed over
|
|
17
|
+
* and over. Turning this into a DELETE would tidy the table and silently
|
|
18
|
+
* destroy the only signal that a repeatedly-dying operation ever leaves.
|
|
13
19
|
*/
|
|
14
20
|
|
|
15
21
|
import { and, eq } from 'drizzle-orm';
|
|
16
22
|
import { getDb } from '../../db/client';
|
|
17
23
|
import { type ModuleOperation, moduleOperations } from '../../db/schema';
|
|
18
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
ABANDONED_RELEASE_MESSAGE,
|
|
26
|
+
OPERATION_TTL_MS,
|
|
27
|
+
isPidRunnable,
|
|
28
|
+
} from '../../services/module-operations';
|
|
19
29
|
import type { CommandResult } from '../types';
|
|
20
30
|
|
|
21
31
|
/** Why a row is not holding the lock, or null when it still is. */
|
|
@@ -41,36 +51,49 @@ function inProgressRows(): ModuleOperation[] {
|
|
|
41
51
|
.all();
|
|
42
52
|
}
|
|
43
53
|
|
|
44
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Abandoned rows are summarised, not listed, unless `--abandoned` asks for
|
|
56
|
+
* them. The question this command answers is "what holds the lock right
|
|
57
|
+
* now", and on a fleet where something is dying repeatedly the answer was
|
|
58
|
+
* buried under 85 corpses. The count still prints, so they never become
|
|
59
|
+
* invisible — the audit is what reads them as a symptom
|
|
60
|
+
* (`services/audit/abandoned-operations.ts`).
|
|
61
|
+
*/
|
|
62
|
+
function handleList(flags: Record<string, boolean | string>): CommandResult {
|
|
45
63
|
const now = Date.now();
|
|
46
64
|
const rows = inProgressRows();
|
|
65
|
+
const showAbandoned = flags.abandoned === true || flags.all === true;
|
|
47
66
|
|
|
48
67
|
if (rows.length === 0) {
|
|
49
68
|
console.log('\nNo module operations in progress.\n');
|
|
50
69
|
return { success: true, message: 'no operations in progress' };
|
|
51
70
|
}
|
|
52
71
|
|
|
72
|
+
const holding = rows.filter((row) => abandonedReason(row, now) === null);
|
|
73
|
+
const abandoned = rows.length - holding.length;
|
|
74
|
+
const shown = showAbandoned ? rows : holding;
|
|
75
|
+
|
|
53
76
|
console.log('\nModule operations in progress:\n');
|
|
54
|
-
|
|
55
|
-
for (const row of rows) {
|
|
77
|
+
for (const row of shown) {
|
|
56
78
|
const reason = abandonedReason(row, now);
|
|
57
|
-
if (!reason) holding++;
|
|
58
79
|
const age = formatAge(now - row.startedAt.getTime());
|
|
59
80
|
const status = reason ? `abandoned (${reason})` : 'HOLDING LOCK';
|
|
60
81
|
console.log(
|
|
61
82
|
` ${row.operation.padEnd(9)} ${row.moduleId.padEnd(16)} pid ${String(row.pid).padEnd(8)} ${age.padStart(4)} ago ${status}`,
|
|
62
83
|
);
|
|
63
84
|
}
|
|
85
|
+
if (shown.length === 0) console.log(' (nothing is holding the lock)');
|
|
64
86
|
|
|
65
|
-
const abandoned = rows.length - holding;
|
|
66
87
|
console.log('');
|
|
67
|
-
if (abandoned > 0) {
|
|
68
|
-
console.log(
|
|
88
|
+
if (abandoned > 0 && !showAbandoned) {
|
|
89
|
+
console.log(
|
|
90
|
+
`${abandoned} abandoned row(s) hidden — "--abandoned" lists them, the hourly sweep clears them.\n`,
|
|
91
|
+
);
|
|
69
92
|
}
|
|
70
93
|
|
|
71
94
|
return {
|
|
72
95
|
success: true,
|
|
73
|
-
message: `${rows.length} in progress (${holding} holding the lock, ${abandoned} abandoned)`,
|
|
96
|
+
message: `${rows.length} in progress (${holding.length} holding the lock, ${abandoned} abandoned)`,
|
|
74
97
|
};
|
|
75
98
|
}
|
|
76
99
|
|
|
@@ -108,7 +131,7 @@ function handleClear(flags: Record<string, boolean | string>): CommandResult {
|
|
|
108
131
|
.set({
|
|
109
132
|
status: 'failed',
|
|
110
133
|
completedAt: new Date(),
|
|
111
|
-
errorMessage:
|
|
134
|
+
errorMessage: ABANDONED_RELEASE_MESSAGE,
|
|
112
135
|
})
|
|
113
136
|
.where(and(eq(moduleOperations.id, row.id), eq(moduleOperations.status, 'in_progress')))
|
|
114
137
|
.run();
|
|
@@ -124,11 +147,11 @@ export function handleModuleOperations(
|
|
|
124
147
|
): CommandResult {
|
|
125
148
|
const action = args[0];
|
|
126
149
|
|
|
127
|
-
if (!action || action === 'list') return handleList();
|
|
150
|
+
if (!action || action === 'list') return handleList(flags);
|
|
128
151
|
if (action === 'clear') return handleClear(flags);
|
|
129
152
|
|
|
130
153
|
return {
|
|
131
154
|
success: false,
|
|
132
|
-
error: `Unknown action "${action}"\n\nUsage: celilo module operations [list|clear] [--all]`,
|
|
155
|
+
error: `Unknown action "${action}"\n\nUsage: celilo module operations [list|clear] [--abandoned] [--all]`,
|
|
133
156
|
};
|
|
134
157
|
}
|
|
@@ -171,6 +171,7 @@ export async function handleModuleShowZone(args: string[]): Promise<CommandResul
|
|
|
171
171
|
secure: 'Secure (Authentication/Database)',
|
|
172
172
|
'secure-mgmt': "Secure-Mgmt (celilo's own control plane)",
|
|
173
173
|
external: 'External (VPS/Cloud)',
|
|
174
|
+
'control-plane-vpn': 'Control-plane VPN (administrative remote access)',
|
|
174
175
|
};
|
|
175
176
|
|
|
176
177
|
// Cast at the lookup, not the declaration: `zone` comes from config and may be
|
|
@@ -11,7 +11,7 @@ import { join } from 'node:path';
|
|
|
11
11
|
import { eq } from 'drizzle-orm';
|
|
12
12
|
import { type DbClient, getDb } from '../../db/client';
|
|
13
13
|
import { modules } from '../../db/schema';
|
|
14
|
-
import { classifyVersionChange, updateOne } from './module-update';
|
|
14
|
+
import { classifyVersionChange, handleModuleUpdate, updateOne } from './module-update';
|
|
15
15
|
|
|
16
16
|
describe('classifyVersionChange', () => {
|
|
17
17
|
test('identical versions are up-to-date', () => {
|
|
@@ -250,3 +250,74 @@ subscriptions:
|
|
|
250
250
|
}
|
|
251
251
|
});
|
|
252
252
|
});
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Regression for the fabricated decline: driven headlessly with no responder,
|
|
256
|
+
* the registry sweep reported a breaking update as "operator declined" — a
|
|
257
|
+
* decision nobody was asked to make. An unanswerable question is not a "no".
|
|
258
|
+
*/
|
|
259
|
+
describe('registry sweep — an unanswered breaking update is not a decline', () => {
|
|
260
|
+
let tempDir: string;
|
|
261
|
+
let server: ReturnType<typeof Bun.serve>;
|
|
262
|
+
let registryUrl: string;
|
|
263
|
+
|
|
264
|
+
beforeEach(() => {
|
|
265
|
+
tempDir = mkdtempSync(join(tmpdir(), 'celilo-sweep-'));
|
|
266
|
+
process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
|
|
267
|
+
process.env.CELILO_ORIGINAL_CWD = tempDir;
|
|
268
|
+
// Isolated bus with no responder attached — the headless case.
|
|
269
|
+
process.env.EVENT_BUS_DB = join(tempDir, 'events.db');
|
|
270
|
+
|
|
271
|
+
getDb()
|
|
272
|
+
.insert(modules)
|
|
273
|
+
.values({
|
|
274
|
+
id: 'iptables',
|
|
275
|
+
name: 'iptables',
|
|
276
|
+
sourcePath: join(tempDir, 'installed'),
|
|
277
|
+
version: '1.0.2+9',
|
|
278
|
+
manifestData: {
|
|
279
|
+
celilo_contract: '1.0',
|
|
280
|
+
id: 'iptables',
|
|
281
|
+
name: 'iptables',
|
|
282
|
+
version: '1.0.2',
|
|
283
|
+
},
|
|
284
|
+
})
|
|
285
|
+
.run();
|
|
286
|
+
|
|
287
|
+
// Minimal sparse-index server offering a major bump for `iptables`.
|
|
288
|
+
server = Bun.serve({
|
|
289
|
+
port: 0,
|
|
290
|
+
fetch(req) {
|
|
291
|
+
const path = new URL(req.url).pathname;
|
|
292
|
+
if (path === '/index/ip/ta/iptables') {
|
|
293
|
+
return new Response(
|
|
294
|
+
`${JSON.stringify({ name: 'iptables', vers: '2.0.0+1', deps: [], cksum: 'x' })}\n`,
|
|
295
|
+
);
|
|
296
|
+
}
|
|
297
|
+
return new Response('not found', { status: 404 });
|
|
298
|
+
},
|
|
299
|
+
});
|
|
300
|
+
registryUrl = `http://localhost:${server.port}`;
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
afterEach(() => {
|
|
304
|
+
server.stop(true);
|
|
305
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
306
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
307
|
+
process.env.CELILO_ORIGINAL_CWD = undefined;
|
|
308
|
+
process.env.EVENT_BUS_DB = undefined;
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
test('reports it as unanswered, never as declined, and fails the sweep', async () => {
|
|
312
|
+
const result = await handleModuleUpdate([], { registry: registryUrl });
|
|
313
|
+
|
|
314
|
+
const report = result.success ? (result.message ?? '') : (result.error ?? '');
|
|
315
|
+
expect(report).not.toContain('operator declined');
|
|
316
|
+
expect(report).toContain('NOT declined');
|
|
317
|
+
expect(report).toContain('iptables');
|
|
318
|
+
// A breaking update that silently didn't land must not read as success.
|
|
319
|
+
expect(result.success).toBe(false);
|
|
320
|
+
// And the module is still on the old version — no accidental upgrade.
|
|
321
|
+
expect(getDb().select().from(modules).all()[0].version).toBe('1.0.2+9');
|
|
322
|
+
});
|
|
323
|
+
});
|
|
@@ -23,6 +23,7 @@ import type { ModuleManifest } from '../../manifest/schema';
|
|
|
23
23
|
import { cleanupTempDir, extractPackage } from '../../module/packaging/extract';
|
|
24
24
|
import { RegistryClient } from '../../registry/client';
|
|
25
25
|
import { askConfirm, withInterviewSession } from '../../services/bus-interview';
|
|
26
|
+
import { InterviewUnansweredError } from '../../services/interview-errors';
|
|
26
27
|
import { getFlag } from '../parser';
|
|
27
28
|
import { log } from '../prompts';
|
|
28
29
|
import type { CommandResult } from '../types';
|
|
@@ -496,7 +497,8 @@ async function runRegistrySweep(
|
|
|
496
497
|
}
|
|
497
498
|
|
|
498
499
|
let appliedBreaking = 0;
|
|
499
|
-
let
|
|
500
|
+
let declinedBreaking = 0;
|
|
501
|
+
const unanswered: Array<{ moduleId: string; error: string }> = [];
|
|
500
502
|
|
|
501
503
|
if (breaking.length > 0) {
|
|
502
504
|
log.info('\nBreaking updates available — review required (semver-major bump):');
|
|
@@ -508,16 +510,29 @@ async function runRegistrySweep(
|
|
|
508
510
|
log.message('Each breaking update will be applied only on explicit confirmation.\n');
|
|
509
511
|
|
|
510
512
|
for (const plan of breaking) {
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
513
|
+
let proceed: boolean;
|
|
514
|
+
try {
|
|
515
|
+
proceed = await withInterviewSession(() =>
|
|
516
|
+
askConfirm({
|
|
517
|
+
scope: `module-upgrade:${plan.moduleId}`,
|
|
518
|
+
key: 'apply_breaking',
|
|
519
|
+
message: `Apply breaking update for ${plan.moduleId} (${plan.installedVersion} → ${plan.targetVersion})?`,
|
|
520
|
+
defaultValue: false,
|
|
521
|
+
}),
|
|
522
|
+
);
|
|
523
|
+
} catch (err) {
|
|
524
|
+
// Nobody could answer. That is NOT a decline — record it as unanswered
|
|
525
|
+
// so the summary says so and the sweep exits non-zero, and keep going
|
|
526
|
+
// so the updates already applied aren't thrown away with a hard abort.
|
|
527
|
+
if (!(err instanceof InterviewUnansweredError)) throw err;
|
|
528
|
+
unanswered.push({ moduleId: plan.moduleId, error: err.message });
|
|
529
|
+
console.log(
|
|
530
|
+
` ? ${plan.moduleId.padEnd(30)} ${plan.installedVersion} → ${plan.targetVersion} (major, UNANSWERED)`,
|
|
531
|
+
);
|
|
532
|
+
continue;
|
|
533
|
+
}
|
|
519
534
|
if (!proceed) {
|
|
520
|
-
|
|
535
|
+
declinedBreaking++;
|
|
521
536
|
continue;
|
|
522
537
|
}
|
|
523
538
|
const result = await fetchAndUpdate(client, plan.moduleId, plan.targetVersion, db, flags);
|
|
@@ -545,8 +560,17 @@ async function runRegistrySweep(
|
|
|
545
560
|
} else {
|
|
546
561
|
summary.push('No updates applied.');
|
|
547
562
|
}
|
|
548
|
-
if (
|
|
549
|
-
summary.push(`Skipped ${
|
|
563
|
+
if (declinedBreaking > 0) {
|
|
564
|
+
summary.push(`Skipped ${declinedBreaking} breaking update(s) (operator declined).`);
|
|
565
|
+
}
|
|
566
|
+
if (unanswered.length > 0) {
|
|
567
|
+
summary.push(
|
|
568
|
+
`Skipped ${unanswered.length} breaking update(s) — NOT declined: the confirmation could not be answered (${unanswered
|
|
569
|
+
.map((u) => u.moduleId)
|
|
570
|
+
.join(
|
|
571
|
+
', ',
|
|
572
|
+
)}). Re-run with a responder attached, or pre-stage the answer under "module-upgrade:<module>.apply_breaking".`,
|
|
573
|
+
);
|
|
550
574
|
}
|
|
551
575
|
if (notInRegistry.length > 0) {
|
|
552
576
|
summary.push(`Not in registry (${notInRegistry.length}): ${notInRegistry.join(', ')}`);
|
|
@@ -556,16 +580,15 @@ async function runRegistrySweep(
|
|
|
556
580
|
`Registry errors (${errored.length}): ${errored.map((e) => `${e.moduleId} — ${e.error}`).join('; ')}`,
|
|
557
581
|
);
|
|
558
582
|
}
|
|
559
|
-
if (failed.length > 0) {
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
};
|
|
583
|
+
if (failed.length > 0 || unanswered.length > 0) {
|
|
584
|
+
const detail: string[] = [];
|
|
585
|
+
if (failed.length > 0) {
|
|
586
|
+
detail.push('', 'Failures:', ...failed.map((f) => ` ${f.moduleId}: ${f.error}`));
|
|
587
|
+
}
|
|
588
|
+
if (unanswered.length > 0) {
|
|
589
|
+
detail.push('', 'Unanswered:', ...unanswered.map((u) => ` ${u.moduleId}: ${u.error}`));
|
|
590
|
+
}
|
|
591
|
+
return { success: false, error: [...summary, ...detail].join('\n') };
|
|
569
592
|
}
|
|
570
593
|
return { success: true, message: summary.join('\n') };
|
|
571
594
|
}
|
|
@@ -38,6 +38,7 @@ import { getOrCreateMasterKey } from '../../secrets/master-key';
|
|
|
38
38
|
import { readAllTransportStatuses } from '../../services/alerting/read-records';
|
|
39
39
|
import { runAudit } from '../../services/audit';
|
|
40
40
|
import type { DriftFinding, SystemAuditReport } from '../../services/audit';
|
|
41
|
+
import { loadAbandonedOperations } from '../../services/audit/abandoned-operations';
|
|
41
42
|
import { loadBackupAuditInfo } from '../../services/audit/backup-source';
|
|
42
43
|
import {
|
|
43
44
|
type LatestCliVersionFetcher,
|
|
@@ -363,6 +364,7 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
|
|
|
363
364
|
moduleConfigs: { modules: installedConfigs },
|
|
364
365
|
health: { results: healthResults },
|
|
365
366
|
backups: { modules: installedBackupInfo },
|
|
367
|
+
abandonedOperations: { records: loadAbandonedOperations(db) },
|
|
366
368
|
undeployedModules: {
|
|
367
369
|
modules: installed.map((m) => ({ id: m.id, state: m.state })),
|
|
368
370
|
},
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Database } from 'bun:sqlite';
|
|
1
2
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
3
|
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
4
|
import { tmpdir } from 'node:os';
|
|
@@ -37,4 +38,59 @@ describe('handleSystemMigrate', () => {
|
|
|
37
38
|
const second = await handleSystemMigrate();
|
|
38
39
|
expect(second.success).toBe(true);
|
|
39
40
|
});
|
|
41
|
+
|
|
42
|
+
// celilo#604: the runbook asserts "applied 19 -> 20, backups.pid present".
|
|
43
|
+
// Before this, the only answer was a table COUNT, which cannot see a column.
|
|
44
|
+
describe('--status', () => {
|
|
45
|
+
it('names the applied count and the latest applied migration', async () => {
|
|
46
|
+
await handleSystemMigrate();
|
|
47
|
+
closeDb();
|
|
48
|
+
|
|
49
|
+
const result = await handleSystemMigrate([], { status: true });
|
|
50
|
+
|
|
51
|
+
expect(result.success).toBe(true);
|
|
52
|
+
if (result.success) {
|
|
53
|
+
expect(result.message).toMatch(/Applied migrations: \d+/);
|
|
54
|
+
expect(result.message).toContain('0019_backup_pid');
|
|
55
|
+
expect(result.message).toContain('Pending: none');
|
|
56
|
+
expect(result.message).toContain('columns');
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('reports a pending migration WITHOUT applying it', async () => {
|
|
61
|
+
await handleSystemMigrate();
|
|
62
|
+
closeDb();
|
|
63
|
+
// Rewind one migration, the way an upgrade that never ran would look.
|
|
64
|
+
const raw = new Database(process.env.CELILO_DB_PATH as string);
|
|
65
|
+
raw.run(
|
|
66
|
+
'DELETE FROM `__drizzle_migrations` WHERE created_at = (SELECT MAX(created_at) FROM `__drizzle_migrations`)',
|
|
67
|
+
);
|
|
68
|
+
raw.run('ALTER TABLE backups DROP COLUMN pid');
|
|
69
|
+
const countBefore = raw
|
|
70
|
+
.query<{ c: number }, []>('SELECT COUNT(*) AS c FROM `__drizzle_migrations`')
|
|
71
|
+
.get()?.c;
|
|
72
|
+
raw.close();
|
|
73
|
+
|
|
74
|
+
const result = await handleSystemMigrate([], { status: true });
|
|
75
|
+
|
|
76
|
+
expect(result.success).toBe(false);
|
|
77
|
+
if (!result.success) {
|
|
78
|
+
expect(result.error).toContain('0019_backup_pid');
|
|
79
|
+
expect(result.error).toContain('backups.pid');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// A status that repaired what it reports would always read clean — the
|
|
83
|
+
// exact placebo this command exists to replace.
|
|
84
|
+
const after = new Database(process.env.CELILO_DB_PATH as string);
|
|
85
|
+
const countAfter = after
|
|
86
|
+
.query<{ c: number }, []>('SELECT COUNT(*) AS c FROM `__drizzle_migrations`')
|
|
87
|
+
.get()?.c;
|
|
88
|
+
const cols = after
|
|
89
|
+
.query<{ name: string }, []>("SELECT name FROM pragma_table_info('backups')")
|
|
90
|
+
.all();
|
|
91
|
+
after.close();
|
|
92
|
+
expect(countAfter).toBe(countBefore as number);
|
|
93
|
+
expect(cols.map((c) => c.name)).not.toContain('pid');
|
|
94
|
+
});
|
|
95
|
+
});
|
|
40
96
|
});
|
|
@@ -8,11 +8,50 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import type { Database } from 'bun:sqlite';
|
|
11
|
-
import {
|
|
11
|
+
import { defineEvents, openBus } from '@celilo/event-bus';
|
|
12
|
+
import { getEventBusPath } from '../../config/paths';
|
|
13
|
+
import { createDbClient, findMigrationsFolder, getDb } from '../../db/client';
|
|
12
14
|
import { runMigrationsOn } from '../../db/migrate';
|
|
15
|
+
import { getMigrationStatus } from '../../db/migration-status';
|
|
13
16
|
import { findSchemaDrift } from '../../db/schema-introspection';
|
|
17
|
+
import { ensureBackupSweepSubscriber } from '../../services/backup-sweep';
|
|
18
|
+
import { ensureOperationsSweepSubscriber } from '../../services/module-operations';
|
|
14
19
|
import type { CommandResult } from '../types';
|
|
15
20
|
|
|
21
|
+
/**
|
|
22
|
+
* Arm celilo's own housekeeping subscribers.
|
|
23
|
+
*
|
|
24
|
+
* Here because this command is what the .deb postinst runs on every apt
|
|
25
|
+
* upgrade — the one moment guaranteed to happen after new CLI code lands.
|
|
26
|
+
* A subscriber registered only from module install/update would not appear
|
|
27
|
+
* until some module happened to be touched next, which can be weeks and
|
|
28
|
+
* looks exactly like a feature that shipped and silently does nothing.
|
|
29
|
+
*
|
|
30
|
+
* Best-effort: a bus that can't be opened must not fail a schema migration.
|
|
31
|
+
*/
|
|
32
|
+
function ensureCoreSubscribers(): void {
|
|
33
|
+
try {
|
|
34
|
+
const bus = openBus({ dbPath: getEventBusPath(), events: defineEvents({}) });
|
|
35
|
+
try {
|
|
36
|
+
ensureOperationsSweepSubscriber(bus);
|
|
37
|
+
// The backup sweep is armed here for the same reason, plus a sharper one:
|
|
38
|
+
// its row already exists on every deployed fleet, carrying the 60s bus
|
|
39
|
+
// default that made scheduled backups impossible. Correcting the default
|
|
40
|
+
// in code does nothing until something re-registers, and module
|
|
41
|
+
// install/update can be weeks away. This is the upgrade path.
|
|
42
|
+
//
|
|
43
|
+
// This also re-arms a sweep an operator paused by hand. Deliberate: the
|
|
44
|
+
// pause exists only because staging leaked, and the reaper that stops it
|
|
45
|
+
// leaking ships in this same binary.
|
|
46
|
+
ensureBackupSweepSubscriber(bus);
|
|
47
|
+
} finally {
|
|
48
|
+
bus.close();
|
|
49
|
+
}
|
|
50
|
+
} catch {
|
|
51
|
+
// Nothing to do — the next module install/update arms it.
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
16
55
|
function countApplied(sqlite: Database): number {
|
|
17
56
|
try {
|
|
18
57
|
const row = sqlite
|
|
@@ -24,7 +63,50 @@ function countApplied(sqlite: Database): number {
|
|
|
24
63
|
}
|
|
25
64
|
}
|
|
26
65
|
|
|
27
|
-
|
|
66
|
+
/**
|
|
67
|
+
* `celilo system migrate --status` — read-only interrogation (celilo#604).
|
|
68
|
+
*
|
|
69
|
+
* A rollout runbook that says "assert applied 19 → 20 and `backups.pid`
|
|
70
|
+
* present" needs a product surface to assert against; the table count that used
|
|
71
|
+
* to be the only answer cannot see a column migration at all.
|
|
72
|
+
*/
|
|
73
|
+
export function migrationStatusResult(sqlite: Database): CommandResult {
|
|
74
|
+
const status = getMigrationStatus(sqlite, findMigrationsFolder());
|
|
75
|
+
const missing = [...status.missingTables, ...status.missingColumns];
|
|
76
|
+
const lines = [
|
|
77
|
+
`Applied migrations: ${status.appliedCount}`,
|
|
78
|
+
`Latest applied: ${status.latestApplied ?? '(none)'}`,
|
|
79
|
+
status.pending.length > 0
|
|
80
|
+
? `Pending: ${status.pending.join(', ')}`
|
|
81
|
+
: 'Pending: none',
|
|
82
|
+
`Schema present: ${status.tableCount} tables, ${status.columnCount} columns`,
|
|
83
|
+
...(missing.length > 0 ? [`Missing: ${missing.join(', ')}`] : []),
|
|
84
|
+
];
|
|
85
|
+
if (status.pending.length > 0 || missing.length > 0) {
|
|
86
|
+
return {
|
|
87
|
+
success: false,
|
|
88
|
+
error: `${lines.join('\n')}\n\nRun \`celilo system migrate\` to apply pending migrations on this box.`,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
return { success: true, message: lines.join('\n'), data: status };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export async function handleSystemMigrate(
|
|
95
|
+
_args: string[] = [],
|
|
96
|
+
flags: Record<string, string | boolean> = {},
|
|
97
|
+
): Promise<CommandResult> {
|
|
98
|
+
// --status must NOT migrate. getDb() auto-migrates on open, so a status that
|
|
99
|
+
// went through it would repair the very state it claims to be reporting and
|
|
100
|
+
// could never say "pending" — the placebo shape this command exists to end.
|
|
101
|
+
if (flags.status) {
|
|
102
|
+
const ro = createDbClient({ readonly: true });
|
|
103
|
+
try {
|
|
104
|
+
return migrationStatusResult(ro.$client);
|
|
105
|
+
} finally {
|
|
106
|
+
ro.$client.close();
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
28
110
|
// getDb() auto-migrates on open; do it inside try so an existing DB that
|
|
29
111
|
// predates the drizzle-authoritative change fails with an actionable message
|
|
30
112
|
// instead of a raw migrator error.
|
|
@@ -57,9 +139,15 @@ export async function handleSystemMigrate(): Promise<CommandResult> {
|
|
|
57
139
|
};
|
|
58
140
|
}
|
|
59
141
|
|
|
142
|
+
ensureCoreSubscribers();
|
|
143
|
+
|
|
144
|
+
// Name the latest migration, not just a table count: "35 tables" reads the
|
|
145
|
+
// same whether a column migration applied or silently did nothing (celilo#604).
|
|
146
|
+
const status = getMigrationStatus(sqlite, findMigrationsFolder());
|
|
60
147
|
const lines = [
|
|
61
148
|
applied > 0 ? `Applied ${applied} migration(s).` : 'Schema already up to date.',
|
|
62
|
-
`
|
|
149
|
+
`Applied migrations: ${status.appliedCount} (latest: ${status.latestApplied ?? 'none'})`,
|
|
150
|
+
`Schema current: ${drift.tableCount} tables, ${drift.columnCount} columns.`,
|
|
63
151
|
];
|
|
64
|
-
return { success: true, message: lines.join('\n') };
|
|
152
|
+
return { success: true, message: lines.join('\n'), data: status };
|
|
65
153
|
}
|
|
@@ -26,6 +26,7 @@ import { backups, moduleConfigs as moduleConfigsTbl, modules } from '../../db/sc
|
|
|
26
26
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
27
27
|
import { RegistryClient } from '../../registry/client';
|
|
28
28
|
import { runAudit } from '../../services/audit';
|
|
29
|
+
import { loadAbandonedOperations } from '../../services/audit/abandoned-operations';
|
|
29
30
|
import { fetchLatestCliVersion } from '../../services/audit/cli-version';
|
|
30
31
|
import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema';
|
|
31
32
|
import { createModuleBackup, createSystemStateBackup } from '../../services/backup-create';
|
|
@@ -567,6 +568,7 @@ export async function handleSystemUpdate(
|
|
|
567
568
|
lastSuccessfulBackupAt: latestBackupByModule.get(m.id) ?? null,
|
|
568
569
|
})),
|
|
569
570
|
},
|
|
571
|
+
abandonedOperations: { records: loadAbandonedOperations(db) },
|
|
570
572
|
undeployedModules: {
|
|
571
573
|
modules: installed.map((m) => ({ id: m.id, state: m.state })),
|
|
572
574
|
},
|
|
@@ -778,6 +780,9 @@ export function rebuildAuditDepsForRerun(
|
|
|
778
780
|
configs: configsByModule.get(m.id) ?? {},
|
|
779
781
|
})),
|
|
780
782
|
},
|
|
783
|
+
// Unchanged across an orchestrator run — an upgrade does not reclaim
|
|
784
|
+
// abandoned operations, so re-reading them would be the same rows.
|
|
785
|
+
abandonedOperations: original.abandonedOperations,
|
|
781
786
|
backups: {
|
|
782
787
|
...original.backups,
|
|
783
788
|
modules: upgradable.map((m) => ({
|
package/src/cli/completion.ts
CHANGED
|
@@ -7,6 +7,7 @@ import { eq } from 'drizzle-orm';
|
|
|
7
7
|
import { getDb } from '../db/client';
|
|
8
8
|
import { capabilities, modules } from '../db/schema';
|
|
9
9
|
import type { ModuleManifest } from '../manifest/schema';
|
|
10
|
+
import { SCHEDULABLE_BUILTIN_CHECKS } from '../services/alerting/builtin-source';
|
|
10
11
|
import { listPrincipals } from '../services/api-access';
|
|
11
12
|
import { listBackups } from '../services/backup-metadata';
|
|
12
13
|
import { listBackupStorages } from '../services/backup-storage';
|
|
@@ -124,6 +125,7 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
124
125
|
'respond',
|
|
125
126
|
'install-daemon',
|
|
126
127
|
'uninstall-daemon',
|
|
128
|
+
'restart-daemon',
|
|
127
129
|
'show-daemon',
|
|
128
130
|
];
|
|
129
131
|
return filterSuggestions(subcommands, args[1] || '');
|
|
@@ -479,6 +481,23 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
479
481
|
return filterSuggestions(['list', 'add', 'run', 'enable', 'disable'], args[1] || '');
|
|
480
482
|
}
|
|
481
483
|
|
|
484
|
+
// Monitor targets - a module ID or one of celilo's own schedulable checks.
|
|
485
|
+
// Sourced from SCHEDULABLE_BUILTIN_CHECKS rather than a hand-copied list, so
|
|
486
|
+
// a new built-in check is completable the moment it is schedulable.
|
|
487
|
+
if (
|
|
488
|
+
command === 'monitor' &&
|
|
489
|
+
(args[1] === 'add' || args[1] === 'run' || args[1] === 'enable' || args[1] === 'disable') &&
|
|
490
|
+
currentIndex === 2
|
|
491
|
+
) {
|
|
492
|
+
const db = getDb();
|
|
493
|
+
const moduleIds = db
|
|
494
|
+
.select({ id: modules.id })
|
|
495
|
+
.from(modules)
|
|
496
|
+
.all()
|
|
497
|
+
.map((m) => m.id);
|
|
498
|
+
return filterSuggestions([...SCHEDULABLE_BUILTIN_CHECKS, ...moduleIds], args[2] || '');
|
|
499
|
+
}
|
|
500
|
+
|
|
482
501
|
// Alerts subcommands
|
|
483
502
|
if (command === 'alerts' && currentIndex === 1) {
|
|
484
503
|
return filterSuggestions(['list', 'ack', 'silence', 'resolve', 'sweep', 'poll'], args[1] || '');
|
package/src/cli/fuel-gauge.ts
CHANGED
|
@@ -334,7 +334,6 @@ export class FuelGauge {
|
|
|
334
334
|
* Strip ANSI codes from text
|
|
335
335
|
*/
|
|
336
336
|
private stripAnsi(text: string): string {
|
|
337
|
-
// biome-ignore lint/suspicious/noControlCharactersInRegex: ESC character needed for ANSI code matching
|
|
338
337
|
return text.replace(/\u001b\[[0-9;]*m/g, '');
|
|
339
338
|
}
|
|
340
339
|
|
|
@@ -361,7 +361,7 @@ _celilo_system_config_keys() {
|
|
|
361
361
|
'network.internal.gateway:Internal gateway IP'
|
|
362
362
|
'network.secure-mgmt.subnet:Control-plane subnet (celilo-mgr own network)'
|
|
363
363
|
'network.secure-mgmt.gateway:Control-plane gateway IP'
|
|
364
|
-
'network.vpn.subnet:VPN client subnet (WireGuard remote access)'
|
|
364
|
+
'network.control-plane-vpn.subnet:Administrative VPN client subnet (WireGuard remote access)'
|
|
365
365
|
'firewall.trusted_subnets:Extra subnets that reach every managed zone (comma-separated CIDRs)'
|
|
366
366
|
'dns.primary:Primary DNS server'
|
|
367
367
|
'dns.fallback:Fallback DNS servers'
|