@celilo/cli 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +61 -9
- package/drizzle/0024_module_pause.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +7 -7
- package/src/__integration__/container-services-cli.integration.test.ts +8 -2
- package/src/api/remote-client.test.ts +6 -5
- package/src/api/serve.ts +41 -7
- package/src/api-clients/proxmox.ts +34 -0
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/alerts-sweep.ts +2 -0
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/events.ts +34 -3
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +64 -2
- package/src/cli/commands/module-config.ts +159 -8
- package/src/cli/commands/module-deploy.ts +2 -2
- package/src/cli/commands/module-health.ts +1 -0
- package/src/cli/commands/module-import.ts +3 -3
- package/src/cli/commands/module-list.ts +12 -1
- package/src/cli/commands/module-pause.ts +317 -0
- package/src/cli/commands/module-remove.ts +78 -40
- package/src/cli/commands/module-status.ts +127 -4
- package/src/cli/commands/module-update.test.ts +1 -1
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/proxmox-template-selection.ts +1 -1
- package/src/cli/commands/status.ts +25 -3
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +39 -9
- package/src/cli/fuel-gauge.ts +4 -4
- package/src/cli/index.ts +104 -22
- package/src/cli/json-output.test.ts +162 -0
- package/src/cli/prompts.ts +53 -74
- package/src/cli/service-credential.ts +3 -3
- package/src/cli/stdout-is-undecorated.test.ts +94 -0
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/cli/types.ts +7 -2
- package/src/db/schema.ts +73 -15
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/run-named-hook.ts +28 -0
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/suppression.test.ts +5 -0
- package/src/services/alerting/suppression.ts +18 -1
- package/src/services/alerting/sweep-runner.test.ts +52 -3
- package/src/services/alerting/sweep-runner.ts +41 -8
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/bus-interview.ts +2 -2
- package/src/services/bus-secret-flow.test.ts +1 -1
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/fleet-checks.ts +48 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/module-deploy.ts +1 -1
- package/src/services/module-pause-observability.test.ts +224 -0
- package/src/services/module-pause-quiescence.test.ts +163 -0
- package/src/services/module-pause.test.ts +573 -0
- package/src/services/module-pause.ts +544 -0
- package/src/services/remove-guard.test.ts +175 -0
- package/src/services/remove-guard.ts +109 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/terminal-responder.ts +16 -16
- package/src/services/update/dep-graph.test.ts +33 -4
- package/src/services/update/dep-graph.ts +39 -17
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +32 -49
- package/src/test-utils/cli.ts +15 -14
- package/src/test-utils/integration-guard.ts +26 -0
- package/src/test-utils/setup-test-db.ts +13 -23
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The per-module recovery summary under `celilo backup list`.
|
|
3
|
+
*
|
|
4
|
+
* The listing itself is a log of attempts, and on celilo-mgr a column of
|
|
5
|
+
* `✗ 0 B` rows on a tidy hourly rhythm was true for a day while forgejo had no
|
|
6
|
+
* usable backup at all (celilo#685). These assert the two facts that column
|
|
7
|
+
* structurally cannot carry: `never`, and how much has failed since.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { describe, expect, test } from 'bun:test';
|
|
11
|
+
import type { Backup } from '../../db/schema';
|
|
12
|
+
import { backedUpModuleIds, lastSuccessSummaryLines } from './backup-list';
|
|
13
|
+
|
|
14
|
+
const NOW = Date.UTC(2026, 7, 13, 12, 0, 0);
|
|
15
|
+
const ago = (ms: number) => new Date(NOW - ms);
|
|
16
|
+
const HOUR = 60 * 60 * 1000;
|
|
17
|
+
|
|
18
|
+
function backup(partial: Partial<Backup>): Backup {
|
|
19
|
+
return { backupType: 'module_data', moduleId: 'forgejo', ...partial } as Backup;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
describe('backedUpModuleIds', () => {
|
|
23
|
+
test('deduplicates and sorts the modules a listing names', () => {
|
|
24
|
+
expect(
|
|
25
|
+
backedUpModuleIds([
|
|
26
|
+
backup({ moduleId: 'forgejo' }),
|
|
27
|
+
backup({ moduleId: 'authentik' }),
|
|
28
|
+
backup({ moduleId: 'forgejo' }),
|
|
29
|
+
]),
|
|
30
|
+
).toEqual(['authentik', 'forgejo']);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test('ignores system backups, which have no module and no cadence', () => {
|
|
34
|
+
expect(
|
|
35
|
+
backedUpModuleIds([
|
|
36
|
+
backup({ backupType: 'system_state', moduleId: null }),
|
|
37
|
+
backup({ moduleId: 'forgejo' }),
|
|
38
|
+
]),
|
|
39
|
+
).toEqual(['forgejo']);
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
describe('lastSuccessSummaryLines', () => {
|
|
44
|
+
test('says never, out loud, when a module has never been captured', () => {
|
|
45
|
+
// `signal` on celilo-mgr. A listing of its attempts shows rows on a
|
|
46
|
+
// schedule; none of them is a backup.
|
|
47
|
+
const [line] = lastSuccessSummaryLines(
|
|
48
|
+
[{ moduleId: 'signal', lastSuccessAt: null, consecutiveFailures: 8 }],
|
|
49
|
+
NOW,
|
|
50
|
+
);
|
|
51
|
+
expect(line).toContain('signal');
|
|
52
|
+
expect(line).toContain('never');
|
|
53
|
+
expect(line).toContain('8 failed attempts since');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test('reports how stale a real success is, with the failures since it', () => {
|
|
57
|
+
const [line] = lastSuccessSummaryLines(
|
|
58
|
+
[{ moduleId: 'forgejo', lastSuccessAt: ago(7 * 24 * HOUR), consecutiveFailures: 20 }],
|
|
59
|
+
NOW,
|
|
60
|
+
);
|
|
61
|
+
// The existing relative formatter buckets 7 days as "last week".
|
|
62
|
+
expect(line).toContain('last week');
|
|
63
|
+
expect(line).toContain('20 failed attempts since');
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test('a healthy module carries no failure clause at all', () => {
|
|
67
|
+
const [line] = lastSuccessSummaryLines(
|
|
68
|
+
[{ moduleId: 'authentik', lastSuccessAt: ago(3 * HOUR), consecutiveFailures: 0 }],
|
|
69
|
+
NOW,
|
|
70
|
+
);
|
|
71
|
+
expect(line).toContain('3h ago');
|
|
72
|
+
expect(line).not.toContain('failed');
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test('counts one failure in the singular', () => {
|
|
76
|
+
const [line] = lastSuccessSummaryLines(
|
|
77
|
+
[{ moduleId: 'caddy', lastSuccessAt: ago(2 * HOUR), consecutiveFailures: 1 }],
|
|
78
|
+
NOW,
|
|
79
|
+
);
|
|
80
|
+
expect(line).toContain('1 failed attempt since');
|
|
81
|
+
expect(line).not.toContain('attempts');
|
|
82
|
+
});
|
|
83
|
+
});
|
|
@@ -3,7 +3,13 @@
|
|
|
3
3
|
* Lists available backups with metadata.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import {
|
|
6
|
+
import type { Backup } from '../../db/schema';
|
|
7
|
+
import {
|
|
8
|
+
formatSize,
|
|
9
|
+
getBackup,
|
|
10
|
+
listBackups,
|
|
11
|
+
loadBackupHistory,
|
|
12
|
+
} from '../../services/backup-metadata';
|
|
7
13
|
import { getBackupStorage } from '../../services/backup-storage';
|
|
8
14
|
import { celiloIntro } from '../prompts';
|
|
9
15
|
import type { CommandResult } from '../types';
|
|
@@ -12,8 +18,7 @@ import type { CommandResult } from '../types';
|
|
|
12
18
|
* Format a timestamp as a human-friendly relative string.
|
|
13
19
|
* Uses relative terms for recent backups and dates for older ones.
|
|
14
20
|
*/
|
|
15
|
-
function formatRelativeDate(date: Date): string {
|
|
16
|
-
const now = Date.now();
|
|
21
|
+
function formatRelativeDate(date: Date, now: number = Date.now()): string {
|
|
17
22
|
const diffMs = now - date.getTime();
|
|
18
23
|
const diffMins = Math.floor(diffMs / 60_000);
|
|
19
24
|
const diffHours = Math.floor(diffMs / 3_600_000);
|
|
@@ -73,6 +78,64 @@ function showBackupDetail(backupId: string): CommandResult {
|
|
|
73
78
|
return { success: true, message: `Backup detail: ${backupId}` };
|
|
74
79
|
}
|
|
75
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Say, per module, when its data was last actually captured.
|
|
83
|
+
*
|
|
84
|
+
* The listing above is a log of ATTEMPTS, and an attempt log is the one thing
|
|
85
|
+
* that cannot answer the question an operator is really asking. On celilo-mgr
|
|
86
|
+
* the forgejo rows read:
|
|
87
|
+
*
|
|
88
|
+
* ✗ 17cedba4 forgejo 0 B 1h ago
|
|
89
|
+
* ✗ 99a9f819 forgejo 0 B 2h ago
|
|
90
|
+
* ✗ fbb4d355 forgejo 0 B 3h ago
|
|
91
|
+
*
|
|
92
|
+
* — a tidy hourly cadence, every row on time, and not one byte of backup
|
|
93
|
+
* anywhere in it. Regular failure and regular success have the same silhouette
|
|
94
|
+
* at a glance; the `✗` is one character wide and the rhythm is what the eye
|
|
95
|
+
* reads. That listing was true for a day while production had no forgejo
|
|
96
|
+
* backup at all (celilo#685).
|
|
97
|
+
*
|
|
98
|
+
* So the recovery question gets answered directly rather than left to be
|
|
99
|
+
* inferred from a column of marks. `never` and a failure count are the two
|
|
100
|
+
* facts a column of attempts structurally cannot show.
|
|
101
|
+
*
|
|
102
|
+
* System backups are skipped: they are not per-module and have no cadence.
|
|
103
|
+
*/
|
|
104
|
+
function printLastSuccessSummary(backupList: Backup[]): void {
|
|
105
|
+
const moduleIds = backedUpModuleIds(backupList);
|
|
106
|
+
if (moduleIds.length === 0) return;
|
|
107
|
+
|
|
108
|
+
const lines = lastSuccessSummaryLines(
|
|
109
|
+
moduleIds.map((moduleId) => ({ moduleId, ...loadBackupHistory(moduleId) })),
|
|
110
|
+
Date.now(),
|
|
111
|
+
);
|
|
112
|
+
console.log('\nLast successful backup:');
|
|
113
|
+
for (const line of lines) console.log(line);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** The modules named by a listing, deduplicated. System rows have no module. */
|
|
117
|
+
export function backedUpModuleIds(backupList: Backup[]): string[] {
|
|
118
|
+
const ids = backupList
|
|
119
|
+
.filter((b) => b.backupType === 'module_data' && b.moduleId)
|
|
120
|
+
.map((b) => b.moduleId as string);
|
|
121
|
+
return [...new Set(ids)].sort();
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** One line per module: when it was last captured, and what has failed since. */
|
|
125
|
+
export function lastSuccessSummaryLines(
|
|
126
|
+
entries: Array<{ moduleId: string; lastSuccessAt: Date | null; consecutiveFailures: number }>,
|
|
127
|
+
now: number,
|
|
128
|
+
): string[] {
|
|
129
|
+
return entries.map(({ moduleId, lastSuccessAt, consecutiveFailures }) => {
|
|
130
|
+
const when = lastSuccessAt ? formatRelativeDate(lastSuccessAt, now) : 'never';
|
|
131
|
+
const since =
|
|
132
|
+
consecutiveFailures > 0
|
|
133
|
+
? ` (${consecutiveFailures} failed attempt${consecutiveFailures === 1 ? '' : 's'} since)`
|
|
134
|
+
: '';
|
|
135
|
+
return ` ${moduleId.padEnd(20)} ${when}${since}`;
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
76
139
|
export async function handleBackupList(
|
|
77
140
|
args: string[],
|
|
78
141
|
flags: Record<string, boolean | string> = {},
|
|
@@ -120,6 +183,7 @@ export async function handleBackupList(
|
|
|
120
183
|
}
|
|
121
184
|
|
|
122
185
|
console.log(`\n${backupList.length} backup${backupList.length === 1 ? '' : 's'} shown.`);
|
|
186
|
+
printLastSuccessSummary(backupList);
|
|
123
187
|
console.log('Run "celilo backup list <id>" for details.\n');
|
|
124
188
|
|
|
125
189
|
return { success: true, message: `Found ${backupList.length} backup(s)` };
|
|
@@ -8,7 +8,12 @@ import { getDb } from '../../db/client';
|
|
|
8
8
|
import { modules } from '../../db/schema';
|
|
9
9
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
10
10
|
import { findBackupEligibleModules } from '../../services/backup-create';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
effectiveBackupRetention,
|
|
13
|
+
pruneBackupsForModule,
|
|
14
|
+
prunesNothing,
|
|
15
|
+
} from '../../services/backup-retention';
|
|
16
|
+
import { loadModuleConfigs } from '../../services/module-config';
|
|
12
17
|
import { celiloIntro, celiloOutro } from '../prompts';
|
|
13
18
|
import type { CommandResult } from '../types';
|
|
14
19
|
|
|
@@ -37,17 +42,16 @@ export async function handleBackupPrune(
|
|
|
37
42
|
}
|
|
38
43
|
|
|
39
44
|
const manifest = mod.manifestData as unknown as ModuleManifest;
|
|
40
|
-
const
|
|
41
|
-
if (
|
|
42
|
-
console.log(
|
|
45
|
+
const policy = effectiveBackupRetention(manifest, loadModuleConfigs(db, specificModule));
|
|
46
|
+
if (prunesNothing(policy)) {
|
|
47
|
+
console.log(
|
|
48
|
+
`Module '${specificModule}' has no retention policy — every backup is kept.\n` +
|
|
49
|
+
`Set one with: celilo module config set ${specificModule} backup_retention_count <n>`,
|
|
50
|
+
);
|
|
43
51
|
return { success: true, message: 'No retention policy' };
|
|
44
52
|
}
|
|
45
53
|
|
|
46
|
-
const result = await pruneBackupsForModule(
|
|
47
|
-
specificModule,
|
|
48
|
-
{ count: retention.count, maxAgeDays: retention.max_age_days },
|
|
49
|
-
dryRun,
|
|
50
|
-
);
|
|
54
|
+
const result = await pruneBackupsForModule(specificModule, policy, dryRun);
|
|
51
55
|
|
|
52
56
|
if (result.deleted > 0) {
|
|
53
57
|
const verb = dryRun ? 'Would delete' : 'Deleted';
|
|
@@ -63,15 +67,11 @@ export async function handleBackupPrune(
|
|
|
63
67
|
// Prune all modules with retention policies
|
|
64
68
|
const eligible = findBackupEligibleModules();
|
|
65
69
|
|
|
66
|
-
for (const { module: mod, manifest } of eligible) {
|
|
67
|
-
const
|
|
68
|
-
if (
|
|
70
|
+
for (const { module: mod, manifest, configs } of eligible) {
|
|
71
|
+
const policy = effectiveBackupRetention(manifest, configs);
|
|
72
|
+
if (prunesNothing(policy)) continue;
|
|
69
73
|
|
|
70
|
-
const result = await pruneBackupsForModule(
|
|
71
|
-
mod.id,
|
|
72
|
-
{ count: retention.count, maxAgeDays: retention.max_age_days },
|
|
73
|
-
dryRun,
|
|
74
|
-
);
|
|
74
|
+
const result = await pruneBackupsForModule(mod.id, policy, dryRun);
|
|
75
75
|
|
|
76
76
|
if (result.deleted > 0) {
|
|
77
77
|
const verb = dryRun ? 'Would delete' : 'Deleted';
|
|
@@ -18,13 +18,19 @@ import {
|
|
|
18
18
|
isBackupDue,
|
|
19
19
|
} from '../../services/backup-create';
|
|
20
20
|
import { failBackup, getBackup, listInProgressBackups } from '../../services/backup-metadata';
|
|
21
|
-
import {
|
|
21
|
+
import {
|
|
22
|
+
effectiveBackupRetention,
|
|
23
|
+
pruneBackupsForModule,
|
|
24
|
+
prunesNothing,
|
|
25
|
+
} from '../../services/backup-retention';
|
|
26
|
+
import { BACKUP_SCHEDULE_CONFIG_KEY } from '../../services/backup-schedule';
|
|
22
27
|
import {
|
|
23
28
|
STAGING_PREFIX,
|
|
24
29
|
reapOrphanedStaging,
|
|
25
30
|
resolveAbandonedBackups,
|
|
26
31
|
} from '../../services/backup-staging';
|
|
27
32
|
import { type BackupSweepReport, runBackupSweep } from '../../services/backup-sweep';
|
|
33
|
+
import { configOverride } from '../../services/module-config';
|
|
28
34
|
import { isPidRunnable } from '../../services/module-operations';
|
|
29
35
|
import type { CommandResult } from '../types';
|
|
30
36
|
|
|
@@ -79,20 +85,26 @@ function resolveAbandonedRecords() {
|
|
|
79
85
|
}
|
|
80
86
|
|
|
81
87
|
export async function handleBackupSweep(): Promise<CommandResult> {
|
|
88
|
+
// Read once, so the cadence the sweep schedules on and the retention it
|
|
89
|
+
// prunes with come from the same snapshot of the operator's config.
|
|
90
|
+
const eligible = findBackupEligibleModules();
|
|
91
|
+
const configsByModule = new Map(eligible.map(({ module, configs }) => [module.id, configs]));
|
|
92
|
+
|
|
82
93
|
const report = await runBackupSweep({
|
|
83
94
|
reapStaging,
|
|
84
95
|
resolveAbandonedRecords,
|
|
85
96
|
listEligible: () =>
|
|
86
|
-
|
|
97
|
+
eligible.map(({ module, manifest, configs }) => ({
|
|
98
|
+
id: module.id,
|
|
99
|
+
manifest,
|
|
100
|
+
scheduleOverride: configOverride(configs, BACKUP_SCHEDULE_CONFIG_KEY),
|
|
101
|
+
})),
|
|
87
102
|
isDue: (moduleId, schedule) => isBackupDue(moduleId, schedule),
|
|
88
103
|
backup: (moduleId) => createModuleBackup(moduleId),
|
|
89
104
|
prune: async ({ id, manifest }) => {
|
|
90
|
-
const
|
|
91
|
-
if (
|
|
92
|
-
await pruneBackupsForModule(id,
|
|
93
|
-
count: retention.count,
|
|
94
|
-
maxAgeDays: retention.max_age_days,
|
|
95
|
-
});
|
|
105
|
+
const policy = effectiveBackupRetention(manifest, configsByModule.get(id));
|
|
106
|
+
if (prunesNothing(policy)) return;
|
|
107
|
+
await pruneBackupsForModule(id, policy);
|
|
96
108
|
},
|
|
97
109
|
});
|
|
98
110
|
|
|
@@ -55,7 +55,7 @@ import {
|
|
|
55
55
|
unitInstalledInAnyScope,
|
|
56
56
|
} from '../../services/events-daemon';
|
|
57
57
|
import { getArg, hasFlag } from '../parser';
|
|
58
|
-
import type { CommandResult } from '../types';
|
|
58
|
+
import type { CommandResult, CommandSuccess } from '../types';
|
|
59
59
|
|
|
60
60
|
const NO_SCHEMAS = defineEvents({});
|
|
61
61
|
|
|
@@ -63,10 +63,16 @@ function openCliBus() {
|
|
|
63
63
|
return openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
/**
|
|
67
|
+
* Every `celilo events` JSON verb returns through here, so `rawOutput` is set
|
|
68
|
+
* once: without it the payload goes through the CLI's decorating renderer and
|
|
69
|
+
* reaches stdout prefixed and re-wrapped, and no longer parses.
|
|
70
|
+
*/
|
|
66
71
|
function jsonResult(data: unknown): CommandResult {
|
|
67
72
|
return {
|
|
68
73
|
success: true,
|
|
69
74
|
message: JSON.stringify(data, null, 2),
|
|
75
|
+
rawOutput: true,
|
|
70
76
|
data,
|
|
71
77
|
};
|
|
72
78
|
}
|
|
@@ -123,6 +129,26 @@ export async function handleEventsRunHook(args: string[]): Promise<CommandResult
|
|
|
123
129
|
const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
|
|
124
130
|
if (!module) return { success: false, error: `Module not found: ${moduleId}` };
|
|
125
131
|
|
|
132
|
+
// Quiescence (openspec/changes/module-pause-lifecycle, task 2.1/2.2). Pausing
|
|
133
|
+
// drops the module's bus subscriptions, so ordinarily nothing reaches here at
|
|
134
|
+
// all; this is the second line, and it is load-bearing rather than belt-and-
|
|
135
|
+
// braces. The subscribers table lives in a DIFFERENT database (events.db) to
|
|
136
|
+
// the module state, so the two can disagree — `events resync-subscriptions`
|
|
137
|
+
// rebuilds subscribers from celilo.db, a restore starts events.db empty, and
|
|
138
|
+
// a hand-written row is always possible. Every one of those paths ends here,
|
|
139
|
+
// where the module's actual state is readable.
|
|
140
|
+
//
|
|
141
|
+
// Success, not failure: the event was delivered correctly and the module is
|
|
142
|
+
// deliberately not listening. Reporting a failure would retry it up to
|
|
143
|
+
// max_attempts and then surface as an alert about the pause the operator
|
|
144
|
+
// themselves took.
|
|
145
|
+
if (module.state === 'PAUSED') {
|
|
146
|
+
return {
|
|
147
|
+
success: true,
|
|
148
|
+
message: `Skipped ${moduleId}.${subName}: module is paused`,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
126
152
|
const manifest = module.manifestData as ModuleManifest;
|
|
127
153
|
const sub = (manifest.subscriptions ?? []).find((s) => s.name === subName);
|
|
128
154
|
if (!sub) {
|
|
@@ -882,7 +908,7 @@ async function runProgrammaticResponder(
|
|
|
882
908
|
|
|
883
909
|
let stopping = false;
|
|
884
910
|
let stopReason: 'idle' | 'max-duration' | 'signal' = 'idle';
|
|
885
|
-
const buildSummary = ():
|
|
911
|
+
const buildSummary = (): CommandSuccess => {
|
|
886
912
|
if (!stopping) {
|
|
887
913
|
stopping = true;
|
|
888
914
|
handle.close();
|
|
@@ -893,7 +919,12 @@ async function runProgrammaticResponder(
|
|
|
893
919
|
answered: handle.answered(),
|
|
894
920
|
missed: handle.missed(),
|
|
895
921
|
};
|
|
896
|
-
return {
|
|
922
|
+
return {
|
|
923
|
+
success: true,
|
|
924
|
+
message: JSON.stringify(summary, null, 2),
|
|
925
|
+
rawOutput: true,
|
|
926
|
+
data: summary,
|
|
927
|
+
};
|
|
897
928
|
};
|
|
898
929
|
|
|
899
930
|
process.on('SIGINT', () => {
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* §8.2 — the classification, on demand.
|
|
3
|
+
*
|
|
4
|
+
* The value of this command is entirely in what it SAYS, so that is what these
|
|
5
|
+
* assert: the role of every interface, and — for anything unaccounted for —
|
|
6
|
+
* what the next converge will actually do about it. "1 alien interface" with no
|
|
7
|
+
* further guidance would repeat the failure this whole change is about, which
|
|
8
|
+
* was never a lack of information but a lack of celilo saying it.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
12
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
13
|
+
import { tmpdir } from 'node:os';
|
|
14
|
+
import { join } from 'node:path';
|
|
15
|
+
|
|
16
|
+
let testDir: string;
|
|
17
|
+
let logs: string[];
|
|
18
|
+
const originalLog = console.log;
|
|
19
|
+
|
|
20
|
+
/** Capture stdout — the command's output IS its contract. */
|
|
21
|
+
function captureLogs() {
|
|
22
|
+
logs = [];
|
|
23
|
+
console.log = (...args: unknown[]) => {
|
|
24
|
+
logs.push(args.map(String).join(' '));
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
beforeEach(() => {
|
|
29
|
+
testDir = mkdtempSync(join(tmpdir(), 'celilo-fw-iface-'));
|
|
30
|
+
process.env.CELILO_DB_PATH = join(testDir, 'test.db');
|
|
31
|
+
process.env.CELILO_DATA_DIR = testDir;
|
|
32
|
+
captureLogs();
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
console.log = originalLog;
|
|
37
|
+
rmSync(testDir, { recursive: true, force: true });
|
|
38
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
describe('celilo firewall interface list', () => {
|
|
42
|
+
test('an empty pool says so, and says what makes a machine a firewall', async () => {
|
|
43
|
+
const { handleFirewallInterfaceList } = await import('./firewall-interface-list');
|
|
44
|
+
const result = await handleFirewallInterfaceList([], {});
|
|
45
|
+
|
|
46
|
+
expect(result.success).toBe(false);
|
|
47
|
+
if (result.success) throw new Error('unreachable — asserted above');
|
|
48
|
+
expect(result.error).toContain('No firewalls');
|
|
49
|
+
// Not a dead end: it explains the rule rather than leaving the operator to
|
|
50
|
+
// guess why their box is not listed.
|
|
51
|
+
expect(result.error).toContain('router');
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test('a named machine that does not exist names it, and points at machine list', async () => {
|
|
55
|
+
const { handleFirewallInterfaceList } = await import('./firewall-interface-list');
|
|
56
|
+
const result = await handleFirewallInterfaceList(['no-such-box'], {});
|
|
57
|
+
|
|
58
|
+
expect(result.success).toBe(false);
|
|
59
|
+
if (result.success) throw new Error('unreachable — asserted above');
|
|
60
|
+
expect(result.error).toContain('no-such-box');
|
|
61
|
+
expect(result.error).toContain('celilo machine list');
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The classification itself is `@celilo/capabilities`' job and is tested there
|
|
67
|
+
* against far more cases than a CLI test should duplicate.
|
|
68
|
+
*
|
|
69
|
+
* What is worth pinning here is STRUCTURAL. Asserting the wording of the output
|
|
70
|
+
* by grepping this command's own source would just be reading my strings back to
|
|
71
|
+
* me — it would pass no matter what they said. The property below is different:
|
|
72
|
+
* it is a claim about what the command may DO, and it can genuinely fail.
|
|
73
|
+
*/
|
|
74
|
+
describe('read-only by construction', () => {
|
|
75
|
+
test('never touches the box', async () => {
|
|
76
|
+
// Safe to run against a firewall whose converge is currently refusing —
|
|
77
|
+
// which is exactly when an operator reaches for it. If this ever grows an
|
|
78
|
+
// SSH call it is a different command and should be named like one.
|
|
79
|
+
const src = await Bun.file(join(import.meta.dir, 'firewall-interface-list.ts')).text();
|
|
80
|
+
|
|
81
|
+
expect(src).not.toContain('runAppCommand');
|
|
82
|
+
expect(src).not.toContain('execFileSync');
|
|
83
|
+
expect(src).not.toContain('ssh ');
|
|
84
|
+
});
|
|
85
|
+
});
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo firewall interface list [<hostname>]`
|
|
3
|
+
*
|
|
4
|
+
* The interface classification, on demand, with no side effects.
|
|
5
|
+
*
|
|
6
|
+
* A converge already refuses or isolates on what it finds — but only when it
|
|
7
|
+
* runs, and only in the middle of a deploy's output. An operator about to
|
|
8
|
+
* onboard a firewall, or wondering why one refused, needs to be able to ASK.
|
|
9
|
+
* The condition that produced `fw-keeper.sh` was not that celilo lacked the
|
|
10
|
+
* information; it was that celilo never said it.
|
|
11
|
+
*
|
|
12
|
+
* Read-only by construction: it reads the stored interface table and the
|
|
13
|
+
* declarations, and classifies in memory. It never touches the box, so it is
|
|
14
|
+
* safe to run against a firewall that is currently refusing.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { classifyInterfaces, isPubliclyRoutable } from '@celilo/capabilities';
|
|
18
|
+
import { getDb } from '../../db/client';
|
|
19
|
+
import { listFirewallIps, readDeclaredNetworks } from '../../hooks/capability-loader';
|
|
20
|
+
import { listMachines } from '../../services/machine-pool';
|
|
21
|
+
import { celiloIntro } from '../prompts';
|
|
22
|
+
import type { CommandResult } from '../types';
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Every network a subnet is declared for — THE same reader the converge uses.
|
|
26
|
+
*
|
|
27
|
+
* This command's whole value is telling an operator what the next converge will
|
|
28
|
+
* do, so reading declarations a second way is not a duplication smell, it is a
|
|
29
|
+
* correctness bug: this file walked `NETWORK_ZONES` and so could not see
|
|
30
|
+
* `network.control-plane-vpn.subnet`. It would have reported `wg0` as ALIEN —
|
|
31
|
+
* "this will be isolated" — about an interface the converge attributes and
|
|
32
|
+
* leaves alone. The operator's most likely response to that reading is to go
|
|
33
|
+
* and remove their own admin VPN.
|
|
34
|
+
*/
|
|
35
|
+
async function declaredZones(): Promise<Array<{ zone: string; subnet: string }>> {
|
|
36
|
+
return readDeclaredNetworks(getDb());
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** One line per interface, explaining the role rather than just naming it. */
|
|
40
|
+
function describe(role: string, zone: string | undefined, ip: string): string {
|
|
41
|
+
if (role === 'zone') return `zone:${zone}`;
|
|
42
|
+
if (role === 'external') return 'external — the WAN edge';
|
|
43
|
+
return isPubliclyRoutable(ip)
|
|
44
|
+
? 'ALIEN — publicly routable but no declared zone claims it'
|
|
45
|
+
: 'ALIEN — no declared subnet contains it';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function handleFirewallInterfaceList(
|
|
49
|
+
args: string[],
|
|
50
|
+
_flags: Record<string, boolean | string> = {},
|
|
51
|
+
): Promise<CommandResult> {
|
|
52
|
+
celiloIntro('Firewall interfaces');
|
|
53
|
+
|
|
54
|
+
const wanted = args[0];
|
|
55
|
+
const machines = await listMachines();
|
|
56
|
+
// A firewall is a machine a firewall provider MANAGES. `role === 'router'` is
|
|
57
|
+
// kept as a second way in, but it cannot be the only one: the role is decided
|
|
58
|
+
// by `machine add` from the zones declared at that moment, and the normal
|
|
59
|
+
// order is to add the machine and THEN deploy iptables, whose `on_install`
|
|
60
|
+
// writes the zone subnets. So a working firewall is recorded as a plain host,
|
|
61
|
+
// and this command — whose entire purpose is to report on firewalls — answered
|
|
62
|
+
// "No firewalls in the machine pool" on a fleet that had one.
|
|
63
|
+
//
|
|
64
|
+
// Named explicitly, the hostname wins, so an operator can inspect any box.
|
|
65
|
+
const firewallIps = new Set(await listFirewallIps(getDb()));
|
|
66
|
+
const targets = wanted
|
|
67
|
+
? machines.filter((m) => m.hostname === wanted)
|
|
68
|
+
: machines.filter((m) => firewallIps.has(m.ipAddress) || m.role === 'router');
|
|
69
|
+
|
|
70
|
+
if (targets.length === 0) {
|
|
71
|
+
return {
|
|
72
|
+
success: false,
|
|
73
|
+
error: wanted
|
|
74
|
+
? `No machine named "${wanted}". Run \`celilo machine list\` to see the pool.`
|
|
75
|
+
: 'No firewalls in the machine pool. celilo looks for a machine managed by a firewall provider, or one it classified as a router.',
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const zones = await declaredZones();
|
|
80
|
+
if (zones.length === 0) {
|
|
81
|
+
console.log(
|
|
82
|
+
'No zone subnets are declared, so every interface will read as unaccounted for.\n' +
|
|
83
|
+
'Declare them with: celilo system config set network.<zone>.subnet <cidr>\n',
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
let alienTotal = 0;
|
|
88
|
+
for (const machine of targets) {
|
|
89
|
+
console.log(`\n${machine.hostname} (${machine.ipAddress})`);
|
|
90
|
+
|
|
91
|
+
const interfaces = (machine.interfaces ?? []).map((i) => ({
|
|
92
|
+
name: i.name,
|
|
93
|
+
ip: i.ipAddress,
|
|
94
|
+
}));
|
|
95
|
+
if (interfaces.length === 0) {
|
|
96
|
+
console.log(' no interfaces recorded — re-run `celilo machine add` to detect them');
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
for (const c of classifyInterfaces(interfaces, zones)) {
|
|
101
|
+
if (c.role === 'alien') alienTotal += 1;
|
|
102
|
+
console.log(` ${c.name.padEnd(8)} ${c.ip.padEnd(16)} ${describe(c.role, c.zone, c.ip)}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
console.log('');
|
|
107
|
+
if (alienTotal > 0) {
|
|
108
|
+
// Say what will HAPPEN, not merely what was found — the answer differs by
|
|
109
|
+
// whether celilo has a baseline for the box, and that is the thing an
|
|
110
|
+
// operator most needs to know before the next converge.
|
|
111
|
+
const consequence = [
|
|
112
|
+
'On a firewall celilo has not yet converged cleanly, the next converge will REFUSE and change nothing.',
|
|
113
|
+
'On one with a recorded baseline, an interface that appeared since will be isolated.',
|
|
114
|
+
'Resolve either by declaring a zone: celilo system config set network.<zone>.subnet <cidr>',
|
|
115
|
+
].join('\n');
|
|
116
|
+
console.log(`${alienTotal} interface(s) celilo cannot attribute.\n${consequence}`);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
success: true,
|
|
121
|
+
message: alienTotal === 0 ? 'every interface accounted for' : `${alienTotal} unaccounted`,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { existsSync } from 'node:fs';
|
|
7
7
|
import { readFileSync } from 'node:fs';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
|
+
import { isPubliclyRoutable } from '@celilo/capabilities';
|
|
9
10
|
import { getDb } from '../../db/client';
|
|
10
11
|
import type { NetworkZone } from '../../db/schema';
|
|
11
12
|
import { askText, withInterviewSession } from '../../services/bus-interview';
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
detectNetworkInterfacesLocal,
|
|
17
18
|
testSshConnection,
|
|
18
19
|
} from '../../services/machine-detector';
|
|
20
|
+
import { describeInterfaceZone } from '../../services/machine-detector';
|
|
19
21
|
import { addMachine, getMachineByIp } from '../../services/machine-pool';
|
|
20
22
|
import { loadExistingConfiguration } from '../../services/system-init';
|
|
21
23
|
import { detectZoneFromIp } from '../../services/zone-detector';
|
|
@@ -242,8 +244,34 @@ export async function handleMachineAdd(
|
|
|
242
244
|
console.log(` Disk: ${detectedInfo.hardware.disk_gb} GB\n`);
|
|
243
245
|
|
|
244
246
|
// Zone: explicit override, else infer from IP.
|
|
247
|
+
//
|
|
248
|
+
// `detectZoneFromIp` answers containment only, and now says `'unknown'`
|
|
249
|
+
// rather than claiming `external` when nothing matches. Resolving that
|
|
250
|
+
// is a SECOND question — is this address one the internet can route to?
|
|
251
|
+
// — and the two were conflated before, which is how a private address in
|
|
252
|
+
// no declared subnet got labelled as facing the internet.
|
|
245
253
|
console.log('Detecting network zone...');
|
|
246
|
-
|
|
254
|
+
if (zoneOverride) {
|
|
255
|
+
zone = zoneOverride;
|
|
256
|
+
} else {
|
|
257
|
+
const detected = await detectZoneFromIp(ipAddress);
|
|
258
|
+
if (detected !== 'unknown') {
|
|
259
|
+
zone = detected;
|
|
260
|
+
} else if (isPubliclyRoutable(ipAddress)) {
|
|
261
|
+
// No declared subnet contains it and the internet can route to it:
|
|
262
|
+
// that is what `external` means — a cloud/VPS box.
|
|
263
|
+
zone = 'external';
|
|
264
|
+
} else {
|
|
265
|
+
// A private address in no declared subnet is UN-ZONEABLE, not
|
|
266
|
+
// external. Guessing here is the original defect; ask instead.
|
|
267
|
+
const fix =
|
|
268
|
+
'Fix: pass --zone <zone>, or declare the subnet with `celilo system config set network.<zone>.subnet <cidr>` and retry.';
|
|
269
|
+
return {
|
|
270
|
+
success: false,
|
|
271
|
+
error: `Cannot infer a zone for ${ipAddress}: it is not publicly routable and no declared network.<zone>.subnet contains it.\n${fix}`,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
}
|
|
247
275
|
console.log(`✓ Zone: ${zone}\n`);
|
|
248
276
|
|
|
249
277
|
// Detect network interfaces and classify machine
|
|
@@ -254,7 +282,7 @@ export async function handleMachineAdd(
|
|
|
254
282
|
|
|
255
283
|
console.log(`✓ Role: ${role}`);
|
|
256
284
|
for (const iface of interfaces) {
|
|
257
|
-
console.log(` ${iface.name}: ${iface.ipAddress} (${iface
|
|
285
|
+
console.log(` ${iface.name}: ${iface.ipAddress} (${describeInterfaceZone(iface)})`);
|
|
258
286
|
}
|
|
259
287
|
console.log('');
|
|
260
288
|
}
|