@celilo/cli 0.16.2 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +39 -9
- package/drizzle/0019_backup_pid.sql +18 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +5 -5
- package/schemas/system_config.json +1 -1
- package/src/cli/command-tree-parser.ts +0 -1
- package/src/cli/commands/alerts-poll.ts +26 -1
- package/src/cli/commands/backup-sweep.ts +62 -0
- package/src/cli/commands/module-operations.test.ts +45 -1
- package/src/cli/commands/module-operations.ts +35 -12
- package/src/cli/commands/module-show.ts +1 -0
- package/src/cli/commands/storage-set-path.test.ts +281 -0
- package/src/cli/commands/storage-set-path.ts +190 -0
- package/src/cli/commands/system-audit.ts +14 -0
- package/src/cli/commands/system-migrate.ts +40 -0
- package/src/cli/commands/system-update.ts +6 -0
- package/src/cli/completion.ts +24 -3
- package/src/cli/fuel-gauge.ts +0 -1
- package/src/cli/generate-zsh-completion.ts +1 -1
- package/src/cli/index.ts +12 -0
- package/src/cli/tui/audit-state.test.ts +15 -1
- package/src/cli/tui/audit-state.ts +6 -0
- package/src/cli/tui/audit-tui.test.tsx +0 -1
- package/src/db/schema.ts +53 -9
- package/src/hooks/capability-loader.ts +30 -1
- package/src/ipam/allocator.ts +13 -3
- package/src/services/alerting/builtin-monitors.test.ts +42 -0
- package/src/services/alerting/builtin-monitors.ts +3 -0
- package/src/services/alerting/builtin-source.ts +15 -0
- package/src/services/alerting/inbound-poller.test.ts +63 -1
- package/src/services/alerting/inbound-poller.ts +42 -0
- package/src/services/alerting/read-records.ts +85 -0
- package/src/services/audit/abandoned-operations.test.ts +73 -0
- package/src/services/audit/abandoned-operations.ts +0 -0
- package/src/services/audit/disk-space.test.ts +111 -0
- package/src/services/audit/disk-space.ts +114 -0
- package/src/services/audit/index.test.ts +2 -0
- package/src/services/audit/index.ts +12 -0
- package/src/services/audit/transport-reads.test.ts +113 -0
- package/src/services/audit/transport-reads.ts +120 -0
- package/src/services/audit/types.ts +3 -0
- package/src/services/backup-create.ts +4 -4
- package/src/services/backup-in-flight-refusal.test.ts +2 -0
- package/src/services/backup-metadata.ts +4 -0
- package/src/services/backup-staging.test.ts +134 -0
- package/src/services/backup-staging.ts +192 -0
- package/src/services/backup-storage.ts +29 -0
- package/src/services/backup-sweep.test.ts +68 -0
- package/src/services/backup-sweep.ts +62 -0
- package/src/services/config-interview.ts +1 -1
- package/src/services/deploy-ansible.ts +0 -1
- package/src/services/disk-probe.test.ts +74 -0
- package/src/services/disk-probe.ts +145 -0
- package/src/services/fleet-checks.ts +15 -0
- package/src/services/module-operations.test.ts +22 -0
- package/src/services/module-operations.ts +48 -1
- package/src/services/module-subscriptions.test.ts +39 -6
- package/src/services/module-subscriptions.ts +6 -4
- package/src/services/module-types-generator.test.ts +6 -3
- package/src/services/module-types-generator.ts +12 -7
- package/src/services/storage-providers/local.ts +2 -1
- package/src/services/update/orchestrator.test.ts +2 -0
- package/src/variables/context.ts +6 -1
package/src/cli/index.ts
CHANGED
|
@@ -839,6 +839,9 @@ Subcommands:
|
|
|
839
839
|
list List all configured storage destinations
|
|
840
840
|
verify <storage-id> Test storage connectivity and permissions
|
|
841
841
|
set-default <id> Set the default backup storage destination
|
|
842
|
+
set-path <id> <path> Relocate a local destination to a new directory
|
|
843
|
+
Options:
|
|
844
|
+
--no-migrate Change the path without moving existing archives
|
|
842
845
|
remove <storage-id> Remove a storage destination
|
|
843
846
|
Options:
|
|
844
847
|
--force Skip confirmation prompts
|
|
@@ -860,6 +863,10 @@ Examples:
|
|
|
860
863
|
# Set default destination
|
|
861
864
|
celilo storage set-default local-backups
|
|
862
865
|
|
|
866
|
+
# Move a local destination (archives at the old path are moved too;
|
|
867
|
+
# if that path is gone, the change proceeds with nothing to migrate)
|
|
868
|
+
celilo storage set-path local-backups /var/lib/celilo/backups
|
|
869
|
+
|
|
863
870
|
# Remove storage
|
|
864
871
|
celilo storage remove local-backups --force
|
|
865
872
|
|
|
@@ -1836,6 +1843,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1836
1843
|
return handleStorageSetDefault(parsed.args, parsed.flags);
|
|
1837
1844
|
}
|
|
1838
1845
|
|
|
1846
|
+
if (parsed.subcommand === 'set-path') {
|
|
1847
|
+
const { handleStorageSetPath } = await import('./commands/storage-set-path');
|
|
1848
|
+
return handleStorageSetPath(parsed.args, parsed.flags);
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1839
1851
|
return {
|
|
1840
1852
|
success: false,
|
|
1841
1853
|
error: `Unknown storage subcommand: ${parsed.subcommand}\n\nRun "celilo storage --help" for usage`,
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
|
-
import type { DriftFinding, SystemAuditReport } from '../../services/audit/types';
|
|
2
|
+
import type { DriftCategory, DriftFinding, SystemAuditReport } from '../../services/audit/types';
|
|
3
3
|
import {
|
|
4
|
+
ALL_CATEGORIES,
|
|
4
5
|
type AuditTuiState,
|
|
6
|
+
CATEGORY_LABELS,
|
|
5
7
|
groupFindings,
|
|
6
8
|
initState,
|
|
7
9
|
reducer,
|
|
@@ -244,3 +246,15 @@ describe('selectedFinding', () => {
|
|
|
244
246
|
expect(selectedFinding(initState(report([])))).toBeNull();
|
|
245
247
|
});
|
|
246
248
|
});
|
|
249
|
+
|
|
250
|
+
// ALL_CATEGORIES is a plain array, so the type system cannot require every
|
|
251
|
+
// DriftCategory to appear in it — a new category compiles fine and then never
|
|
252
|
+
// shows up in the TUI. CATEGORY_LABELS is a Record and IS exhaustive, so it is
|
|
253
|
+
// the honest source of truth to compare against.
|
|
254
|
+
describe('ALL_CATEGORIES covers every category', () => {
|
|
255
|
+
test('every labelled category is listed, and vice versa', () => {
|
|
256
|
+
expect([...ALL_CATEGORIES].sort()).toEqual(
|
|
257
|
+
(Object.keys(CATEGORY_LABELS) as DriftCategory[]).sort(),
|
|
258
|
+
);
|
|
259
|
+
});
|
|
260
|
+
});
|
|
@@ -77,12 +77,15 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
|
|
|
77
77
|
'module_configs',
|
|
78
78
|
'health',
|
|
79
79
|
'backups',
|
|
80
|
+
'abandoned_operations',
|
|
80
81
|
'undeployed_modules',
|
|
81
82
|
'unconfigured_modules',
|
|
82
83
|
'services_credentials',
|
|
83
84
|
'secrets_decryptable',
|
|
84
85
|
'services_reachable',
|
|
85
86
|
'machines_reachable',
|
|
87
|
+
'disk_space',
|
|
88
|
+
'transport_reads',
|
|
86
89
|
'trusted_sources',
|
|
87
90
|
];
|
|
88
91
|
|
|
@@ -95,12 +98,15 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
|
|
|
95
98
|
module_configs: 'Module configs',
|
|
96
99
|
health: 'Module health',
|
|
97
100
|
backups: 'Backups',
|
|
101
|
+
abandoned_operations: 'Abandoned operations',
|
|
98
102
|
undeployed_modules: 'Undeployed modules',
|
|
99
103
|
unconfigured_modules: 'Unconfigured modules',
|
|
100
104
|
services_credentials: 'Service credentials',
|
|
101
105
|
secrets_decryptable: 'Secrets',
|
|
102
106
|
services_reachable: 'Service reachability',
|
|
103
107
|
machines_reachable: 'Machine reachability',
|
|
108
|
+
disk_space: 'Disk space',
|
|
109
|
+
transport_reads: 'Transport readability',
|
|
104
110
|
trusted_sources: 'Trusted networks',
|
|
105
111
|
};
|
|
106
112
|
|
package/src/db/schema.ts
CHANGED
|
@@ -249,14 +249,31 @@ export const moduleBuilds = sqliteTable('module_builds', {
|
|
|
249
249
|
});
|
|
250
250
|
|
|
251
251
|
/**
|
|
252
|
-
* Network zones
|
|
253
|
-
*
|
|
254
|
-
*
|
|
255
|
-
*
|
|
256
|
-
*
|
|
252
|
+
* Network zones — the DEFINED vocabulary, not the active set.
|
|
253
|
+
*
|
|
254
|
+
* A zone being listed here means celilo knows the name and will accept
|
|
255
|
+
* `network.<zone>.*` config for it. It does NOT mean the zone exists on a given
|
|
256
|
+
* fleet: per openspec/specs/progressive-zone-disclosure/spec.md, "deployable
|
|
257
|
+
* zones are those with a configured subnet", so a zone becomes REAL only when
|
|
258
|
+
* something declares `network.<zone>.subnet` — normally the module that supplies
|
|
259
|
+
* the network. Defining a zone here costs nothing and activates nothing.
|
|
260
|
+
*
|
|
261
|
+
* - internal: the semi-trusted LAN the management server sits on
|
|
262
|
+
* - dmz: public-facing services
|
|
263
|
+
* - app: internal application services
|
|
264
|
+
* - secure: authentication and database services
|
|
257
265
|
* - secure-mgmt: celilo's own control plane (management server + management-plane
|
|
258
266
|
* modules). Outside the data-plane tier chain; reaches every tier by trust.
|
|
259
|
-
* - external:
|
|
267
|
+
* - external: internet-hosted services — a cloud/VPS provider's network, NOT a
|
|
268
|
+
* VPN. Addressed by the provider, which is why it is excluded from IPAM below.
|
|
269
|
+
* - control-plane-vpn: the ADMINISTRATIVE remote-access client subnet, activated
|
|
270
|
+
* by the module that terminates the tunnel (`wireguard` writes
|
|
271
|
+
* `network.control-plane-vpn.subnet`). Named for its purpose rather than its
|
|
272
|
+
* technology because a fleet may run more than one VPN — a site-to-site link
|
|
273
|
+
* or a user VPN is a different network with different trust, and `vpn` would
|
|
274
|
+
* have been the wrong name to have to share. Distinct from `external` too:
|
|
275
|
+
* that is someone else's cloud, this is a network the fleet's own firewall
|
|
276
|
+
* holds a leg on and must translate for.
|
|
260
277
|
*/
|
|
261
278
|
export const NETWORK_ZONES = [
|
|
262
279
|
'internal',
|
|
@@ -265,6 +282,7 @@ export const NETWORK_ZONES = [
|
|
|
265
282
|
'secure',
|
|
266
283
|
'secure-mgmt',
|
|
267
284
|
'external',
|
|
285
|
+
'control-plane-vpn',
|
|
268
286
|
] as const;
|
|
269
287
|
|
|
270
288
|
/**
|
|
@@ -276,15 +294,36 @@ export const NETWORK_ZONES = [
|
|
|
276
294
|
export type NetworkZone = (typeof NETWORK_ZONES)[number];
|
|
277
295
|
|
|
278
296
|
/**
|
|
279
|
-
* Zones an IP allocation or reservation can name: every NetworkZone
|
|
280
|
-
*
|
|
297
|
+
* Zones an IP allocation or reservation can name: every NetworkZone whose
|
|
298
|
+
* addresses celilo hands out.
|
|
299
|
+
*
|
|
300
|
+
* Two are excluded, for the same underlying reason — somebody else is the
|
|
301
|
+
* address authority:
|
|
302
|
+
* - `external`, whose systems are addressed by the cloud/VPS provider;
|
|
303
|
+
* - `vpn`, whose client addresses are assigned by the module terminating the
|
|
304
|
+
* tunnel. celilo allocating into that subnet would collide with the VPN
|
|
305
|
+
* server's own assignments.
|
|
281
306
|
*
|
|
282
307
|
* Derived rather than hand-written for the same reason as NetworkZone above —
|
|
283
308
|
* the previous hand-written union was copied into two column definitions and a
|
|
284
309
|
* cast in machine-pool.ts, and the cast had already drifted (it was missing
|
|
285
310
|
* `secure-mgmt`, and its comment claimed the only difference was `external`).
|
|
286
311
|
*/
|
|
287
|
-
export type AllocatableZone = Exclude<NetworkZone, 'external'>;
|
|
312
|
+
export type AllocatableZone = Exclude<NetworkZone, 'external' | 'control-plane-vpn'>;
|
|
313
|
+
|
|
314
|
+
/** The zones whose addresses celilo hands out, as a runtime list. */
|
|
315
|
+
export const ALLOCATABLE_ZONES: AllocatableZone[] = NETWORK_ZONES.filter(
|
|
316
|
+
(zone): zone is AllocatableZone => zone !== 'external' && zone !== 'control-plane-vpn',
|
|
317
|
+
);
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* Is this a zone celilo allocates addresses in? Use this rather than testing
|
|
321
|
+
* `zone !== 'external'` by hand — that check predates `vpn` and read as "the one
|
|
322
|
+
* externally-addressed zone" when there are now two.
|
|
323
|
+
*/
|
|
324
|
+
export function isAllocatableZone(zone: NetworkZone): zone is AllocatableZone {
|
|
325
|
+
return (ALLOCATABLE_ZONES as string[]).includes(zone);
|
|
326
|
+
}
|
|
288
327
|
|
|
289
328
|
/**
|
|
290
329
|
* Container services table
|
|
@@ -679,6 +718,11 @@ export const backups = sqliteTable('backups', {
|
|
|
679
718
|
status: text('status').$type<BackupStatus>().notNull().default('in_progress'),
|
|
680
719
|
errorMessage: text('error_message'),
|
|
681
720
|
name: text('name'), // optional human-readable name/annotation
|
|
721
|
+
// The process assembling this backup's staging directory. Lets the staging
|
|
722
|
+
// reaper (services/backup-staging.ts) tell a live backup from one whose
|
|
723
|
+
// process was killed before its `finally` could clean up. Nullable: rows
|
|
724
|
+
// written before this column existed have no pid and age out via the TTL.
|
|
725
|
+
pid: integer('pid'),
|
|
682
726
|
startedAt: integer('started_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
683
727
|
completedAt: integer('completed_at', { mode: 'timestamp' }),
|
|
684
728
|
});
|
|
@@ -28,7 +28,14 @@ import type {
|
|
|
28
28
|
} from '@celilo/capabilities';
|
|
29
29
|
import { and, eq } from 'drizzle-orm';
|
|
30
30
|
import type { DbClient } from '../db/client';
|
|
31
|
-
import {
|
|
31
|
+
import {
|
|
32
|
+
NETWORK_ZONES,
|
|
33
|
+
capabilities,
|
|
34
|
+
modules,
|
|
35
|
+
secrets,
|
|
36
|
+
systemConfig,
|
|
37
|
+
webRoutes,
|
|
38
|
+
} from '../db/schema';
|
|
32
39
|
import { decryptSecret } from '../secrets/encryption';
|
|
33
40
|
import { getOrCreateMasterKey } from '../secrets/master-key';
|
|
34
41
|
import { emitWebRoutesChangedAndWait } from '../services/celilo-events';
|
|
@@ -593,6 +600,7 @@ function buildCapabilityInterface(
|
|
|
593
600
|
zoneTiers: zones?.zoneTiers ?? [],
|
|
594
601
|
trustedSubnets: zones?.trustedSubnets ?? [],
|
|
595
602
|
controlPlaneSubnet: zones?.controlPlaneSubnet,
|
|
603
|
+
frontedSubnets: zones?.frontedSubnets ?? [],
|
|
596
604
|
},
|
|
597
605
|
store,
|
|
598
606
|
undefined, // no upstream — the chain path handles that
|
|
@@ -699,6 +707,22 @@ interface FirewallZones {
|
|
|
699
707
|
trustedSubnets: string[];
|
|
700
708
|
/** celilo's control-plane network, as a DESTINATION for trusted sources. */
|
|
701
709
|
controlPlaneSubnet?: string;
|
|
710
|
+
/**
|
|
711
|
+
* Every declared zone subnet — read from
|
|
712
|
+
* `network.<zone>.subnet` across all of `NETWORK_ZONES`, not just the three
|
|
713
|
+
* data-plane tiers.
|
|
714
|
+
*
|
|
715
|
+
* A downstream firewall must translate for every network behind it, and the
|
|
716
|
+
* tiers are only the ones that happen to form the dmz→app→secure chain.
|
|
717
|
+
* `secure-mgmt` and the control-plane VPN are equally behind it and equally
|
|
718
|
+
* unroutable untranslated. Reading the canonical zone list rather than the tier
|
|
719
|
+
* list means declaring a zone's subnet is sufficient to get it translated — no
|
|
720
|
+
* module registration required.
|
|
721
|
+
*
|
|
722
|
+
* The egress network is NOT excluded here: the firewall derives that from its
|
|
723
|
+
* own routing table, so nothing on this side has to guess which zone it is.
|
|
724
|
+
*/
|
|
725
|
+
frontedSubnets: string[];
|
|
702
726
|
}
|
|
703
727
|
|
|
704
728
|
/** The module that IS celilo's control plane; its network is what we trust. */
|
|
@@ -802,6 +826,10 @@ function loadFirewallZones(db: DbClient): FirewallZones {
|
|
|
802
826
|
zoneTiers,
|
|
803
827
|
trustedSubnets: loadTrustedSubnets(db).map((e) => e.subnet),
|
|
804
828
|
controlPlaneSubnet: loadControlPlaneSubnet(db) ?? readZoneSubnet(db, 'internal'),
|
|
829
|
+
// Every zone celilo knows about, not the tier list — see the field doc.
|
|
830
|
+
frontedSubnets: NETWORK_ZONES.map((zone) => readZoneSubnet(db, zone)).filter(
|
|
831
|
+
(subnet): subnet is string => !!subnet,
|
|
832
|
+
),
|
|
805
833
|
};
|
|
806
834
|
}
|
|
807
835
|
|
|
@@ -953,6 +981,7 @@ async function buildFirewallChain(
|
|
|
953
981
|
zoneTiers: zones.zoneTiers,
|
|
954
982
|
trustedSubnets: zones.trustedSubnets,
|
|
955
983
|
controlPlaneSubnet: zones.controlPlaneSubnet,
|
|
984
|
+
frontedSubnets: zones.frontedSubnets,
|
|
956
985
|
},
|
|
957
986
|
store,
|
|
958
987
|
currentUpstream,
|
package/src/ipam/allocator.ts
CHANGED
|
@@ -9,16 +9,26 @@ import type { BunSQLiteDatabase } from 'drizzle-orm/bun-sqlite';
|
|
|
9
9
|
import type { DbClient } from '../db/client';
|
|
10
10
|
import { ipAllocations, ipReservations, systemConfig, vmidReservations } from '../db/schema';
|
|
11
11
|
import type { NewIpAllocation, NewIpReservation, NewVmidReservation } from '../db/schema';
|
|
12
|
+
import { ALLOCATABLE_ZONES, type AllocatableZone } from '../db/schema';
|
|
12
13
|
import type * as schema from '../db/schema';
|
|
13
14
|
import { generateIPsInSubnet, isIPInRange, isInSubnet, stripCIDR } from './subnet-parser';
|
|
14
15
|
|
|
15
16
|
// Type that accepts both database client and transaction
|
|
16
17
|
type DbOrTransaction = BunSQLiteDatabase<typeof schema> | DbClient;
|
|
17
18
|
|
|
18
|
-
/**
|
|
19
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Zones that support IPAM auto-allocation of VMID and container IP.
|
|
21
|
+
*
|
|
22
|
+
* ALIASED to `AllocatableZone`, not hand-written. This was its own union
|
|
23
|
+
* (`'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal'`) that happened to
|
|
24
|
+
* agree with `AllocatableZone` — a fourth hand-maintained copy of the zone list,
|
|
25
|
+
* in a repo where three earlier copies had already drifted and left comments
|
|
26
|
+
* saying so. Deriving means a zone celilo stops allocating for (as `vpn` is,
|
|
27
|
+
* since the tunnel module assigns client addresses) cannot be missed here.
|
|
28
|
+
*/
|
|
29
|
+
export type IpamZone = AllocatableZone;
|
|
20
30
|
|
|
21
|
-
const IPAM_ZONES: IpamZone[] =
|
|
31
|
+
const IPAM_ZONES: IpamZone[] = ALLOCATABLE_ZONES;
|
|
22
32
|
|
|
23
33
|
/**
|
|
24
34
|
* Infer which zone an IP address belongs to by checking configured zone subnets.
|
|
@@ -27,6 +27,12 @@ describe('targetKindForCategory', () => {
|
|
|
27
27
|
expect(targetKindForCategory('backups')).toBe('module');
|
|
28
28
|
});
|
|
29
29
|
|
|
30
|
+
// Machine-scoped so suppression inherits the machine topology: an unreachable
|
|
31
|
+
// host suppresses its own disk alert instead of paging twice for one dead box.
|
|
32
|
+
test('disk_space is machine-scoped', () => {
|
|
33
|
+
expect(targetKindForCategory('disk_space')).toBe('machine');
|
|
34
|
+
});
|
|
35
|
+
|
|
30
36
|
// Whole-system categories have no narrower subject to suppress against.
|
|
31
37
|
test('unmapped category falls back to system', () => {
|
|
32
38
|
expect(targetKindForCategory('cli_version')).toBe('system');
|
|
@@ -46,6 +52,42 @@ describe('severityForDriftSeverity', () => {
|
|
|
46
52
|
});
|
|
47
53
|
|
|
48
54
|
describe('failingKeysFromFindings', () => {
|
|
55
|
+
// Asserts the WHOLE key, not a prefix. A prefix-only assertion is exactly why
|
|
56
|
+
// #596 survived — machines_reachable emits a UUID where suppression expects a
|
|
57
|
+
// hostname, and `toContain('builtin:machines_reachable/machine:')` passes for
|
|
58
|
+
// both. This check must be pinned to the identifier itself.
|
|
59
|
+
test('disk_space keys on the hostname, in full', () => {
|
|
60
|
+
const keys = failingKeysFromFindings(
|
|
61
|
+
'disk_space',
|
|
62
|
+
[
|
|
63
|
+
finding({
|
|
64
|
+
category: 'disk_space',
|
|
65
|
+
code: 'disk_critical',
|
|
66
|
+
severity: 'blocked',
|
|
67
|
+
subject: 'celilo-mgr',
|
|
68
|
+
message: 'celilo-mgr: root filesystem 96% full',
|
|
69
|
+
}),
|
|
70
|
+
],
|
|
71
|
+
'critical',
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
expect(keys).toHaveLength(1);
|
|
75
|
+
expect(keys[0]?.key).toBe('builtin:disk_space/machine:celilo-mgr');
|
|
76
|
+
expect(keys[0]?.severity).toBe('critical');
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
// An unmeasurable host records without paging.
|
|
80
|
+
test('an unmeasured disk downgrades to warning', () => {
|
|
81
|
+
const keys = failingKeysFromFindings(
|
|
82
|
+
'disk_space',
|
|
83
|
+
[finding({ category: 'disk_space', severity: 'todo', subject: 'iot' })],
|
|
84
|
+
'critical',
|
|
85
|
+
);
|
|
86
|
+
|
|
87
|
+
expect(keys[0]?.key).toBe('builtin:disk_space/machine:iot');
|
|
88
|
+
expect(keys[0]?.severity).toBe('warning');
|
|
89
|
+
});
|
|
90
|
+
|
|
49
91
|
test('projects findings into builtin keys', () => {
|
|
50
92
|
const keys = failingKeysFromFindings('machines_reachable', [finding()], 'critical');
|
|
51
93
|
expect(keys).toEqual([
|
|
@@ -24,9 +24,12 @@ import { type FailingKey, builtinAlertKey } from './keys';
|
|
|
24
24
|
*/
|
|
25
25
|
const TARGET_KIND_BY_CATEGORY: Partial<Record<DriftCategory, string>> = {
|
|
26
26
|
machines_reachable: 'machine',
|
|
27
|
+
disk_space: 'machine',
|
|
28
|
+
transport_reads: 'module',
|
|
27
29
|
services_reachable: 'service',
|
|
28
30
|
services_credentials: 'service',
|
|
29
31
|
backups: 'module',
|
|
32
|
+
abandoned_operations: 'module',
|
|
30
33
|
health: 'module',
|
|
31
34
|
module_versions: 'module',
|
|
32
35
|
module_configs: 'module',
|
|
@@ -13,16 +13,21 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import type { DbClient } from '../../db/client';
|
|
16
|
+
import { auditAbandonedOperations, loadAbandonedOperations } from '../audit/abandoned-operations';
|
|
16
17
|
import { loadBackupAuditInfo } from '../audit/backup-source';
|
|
17
18
|
import { auditBackups } from '../audit/backups';
|
|
19
|
+
import { auditDiskSpace } from '../audit/disk-space';
|
|
18
20
|
import { auditMachinesReachable } from '../audit/machines-reachable';
|
|
19
21
|
import type { DriftCategory, DriftFinding } from '../audit/types';
|
|
22
|
+
import { probeDiskUsage } from '../disk-probe';
|
|
20
23
|
import { probeMachines } from '../machine-probe';
|
|
21
24
|
|
|
22
25
|
/** Categories a monitor can currently schedule. */
|
|
23
26
|
export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
|
|
24
27
|
'machines_reachable',
|
|
25
28
|
'backups',
|
|
29
|
+
'disk_space',
|
|
30
|
+
'abandoned_operations',
|
|
26
31
|
];
|
|
27
32
|
|
|
28
33
|
export function isSchedulableBuiltin(category: string): category is DriftCategory {
|
|
@@ -37,12 +42,22 @@ export async function runBuiltinCheckForMonitor(
|
|
|
37
42
|
return auditMachinesReachable({ results: await probeMachines() });
|
|
38
43
|
}
|
|
39
44
|
|
|
45
|
+
// Same shape as machines_reachable — one SSH round trip per machine, bounded
|
|
46
|
+
// — but it MEASURES the local box rather than exempting it. See disk-probe.ts.
|
|
47
|
+
if (category === 'disk_space') {
|
|
48
|
+
return auditDiskSpace({ results: await probeDiskUsage() });
|
|
49
|
+
}
|
|
50
|
+
|
|
40
51
|
// Local DB reads only — cheap enough to run on every sweep, which is
|
|
41
52
|
// the whole reason this category is schedulable and most are not.
|
|
42
53
|
if (category === 'backups') {
|
|
43
54
|
return auditBackups({ modules: loadBackupAuditInfo(db) });
|
|
44
55
|
}
|
|
45
56
|
|
|
57
|
+
if (category === 'abandoned_operations') {
|
|
58
|
+
return auditAbandonedOperations({ records: loadAbandonedOperations(db) });
|
|
59
|
+
}
|
|
60
|
+
|
|
46
61
|
throw new Error(
|
|
47
62
|
`Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
|
|
48
63
|
);
|
|
@@ -7,7 +7,12 @@ import { eq } from 'drizzle-orm';
|
|
|
7
7
|
import type { DbClient } from '../../db/client';
|
|
8
8
|
import { type Route, alerts, modules, monitors, notificationDeliveries } from '../../db/schema';
|
|
9
9
|
import { setupTestDatabase } from '../../test-utils/setup-test-db';
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
type InboundPollDeps,
|
|
12
|
+
type TransportReadRecord,
|
|
13
|
+
pollInbound,
|
|
14
|
+
transportsWithRoutes,
|
|
15
|
+
} from './inbound-poller';
|
|
11
16
|
import { moduleCheckAlertKey } from './keys';
|
|
12
17
|
import { createPerson, createRoute } from './people';
|
|
13
18
|
import { mintDelivery } from './tokens';
|
|
@@ -386,6 +391,63 @@ describe('pollInbound', () => {
|
|
|
386
391
|
expect(ackedBy()).toBe(peterRoute.personId);
|
|
387
392
|
});
|
|
388
393
|
|
|
394
|
+
// The gap this closes. The ONLY per-transport state celilo persisted was the
|
|
395
|
+
// cursor, and writeCursor early-returns when there is no cursor — which a
|
|
396
|
+
// failed read never produces. So the store could not REPRESENT a failure, and
|
|
397
|
+
// "unreadable for a week" and "nobody replied for a week" left identical
|
|
398
|
+
// traces (#501).
|
|
399
|
+
describe('recording what each read attempt produced', () => {
|
|
400
|
+
const records: Array<[string, TransportReadRecord]> = [];
|
|
401
|
+
const recording = (over: Partial<InboundPollDeps> = {}) =>
|
|
402
|
+
deps([], { recordRead: (t, r) => records.push([t, r]), ...over });
|
|
403
|
+
|
|
404
|
+
beforeEach(() => {
|
|
405
|
+
records.length = 0;
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
test('a FAILED read is recorded — the case the cursor could never express', async () => {
|
|
409
|
+
await pollInbound(
|
|
410
|
+
db,
|
|
411
|
+
recording({ receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
|
|
412
|
+
);
|
|
413
|
+
expect(records).toHaveLength(1);
|
|
414
|
+
expect(records[0][0]).toBe('signal');
|
|
415
|
+
expect(records[0][1]).toMatchObject({ outcome: 'failed', error: 'connection refused' });
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
test('a successful read is recorded with how many messages it returned', async () => {
|
|
419
|
+
await pollInbound(
|
|
420
|
+
db,
|
|
421
|
+
deps([inbound(PETER, 'hello')], { recordRead: (t, r) => records.push([t, r]) }),
|
|
422
|
+
);
|
|
423
|
+
expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 1 });
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
// Zero messages is not failure. Conflating them is the original bug.
|
|
427
|
+
test('an empty but successful read records received, not failed', async () => {
|
|
428
|
+
await pollInbound(db, recording());
|
|
429
|
+
expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 0 });
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
test('a unidirectional transport is recorded as such', async () => {
|
|
433
|
+
await pollInbound(db, recording({ receiveFrom: async () => ({ status: 'unidirectional' }) }));
|
|
434
|
+
expect(records[0][1].outcome).toBe('unidirectional');
|
|
435
|
+
});
|
|
436
|
+
|
|
437
|
+
// Every attempt, not just interesting ones — a gap in the record would be
|
|
438
|
+
// read as "nothing happened".
|
|
439
|
+
test('every attempt is recorded, and carries when it happened', async () => {
|
|
440
|
+
await pollInbound(db, recording());
|
|
441
|
+
expect(records[0][1].at).toBe(NOW.toISOString());
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
// Optional dep: a caller that does not persist must still poll.
|
|
445
|
+
test('polling works with no recorder attached', async () => {
|
|
446
|
+
const report = await pollInbound(db, deps([]));
|
|
447
|
+
expect(report.transportsPolled).toBe(1);
|
|
448
|
+
});
|
|
449
|
+
});
|
|
450
|
+
|
|
389
451
|
test('a route pointing at a transport nobody uses is not polled', () => {
|
|
390
452
|
db.delete(alerts).run();
|
|
391
453
|
expect(transportsWithRoutes(db)).toEqual(['signal']);
|
|
@@ -44,6 +44,35 @@ export interface TransportFailure {
|
|
|
44
44
|
error: string;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* The outcome of one attempt to read a transport, as recorded for later.
|
|
49
|
+
*
|
|
50
|
+
* Written on EVERY attempt, including failures — which is the whole point. The
|
|
51
|
+
* only per-transport state celilo persisted before this was the cursor, and a
|
|
52
|
+
* failed read produces no cursor, so `writeCursor` returned early and nothing
|
|
53
|
+
* was written. The store could not REPRESENT a failure, so an absence of
|
|
54
|
+
* recorded failures was never evidence there had been none: a transport that
|
|
55
|
+
* had not been readable for a week looked identical to one nobody had replied
|
|
56
|
+
* on (#501).
|
|
57
|
+
*/
|
|
58
|
+
export interface TransportReadRecord {
|
|
59
|
+
/** When the attempt happened, ISO-8601. */
|
|
60
|
+
at: string;
|
|
61
|
+
outcome: 'received' | 'unidirectional' | 'failed';
|
|
62
|
+
/** Present only when `failed`. */
|
|
63
|
+
error?: string;
|
|
64
|
+
/** How many messages the read returned. Zero is not the same as failure. */
|
|
65
|
+
messages: number;
|
|
66
|
+
/**
|
|
67
|
+
* When a read last SUCCEEDED, carried forward across failures.
|
|
68
|
+
*
|
|
69
|
+
* This is the field that answers the question the old state could not: a
|
|
70
|
+
* transport reporting "0 messages" for a week and one that has not been
|
|
71
|
+
* readable for a week are the same picture until you can see this.
|
|
72
|
+
*/
|
|
73
|
+
lastSuccessAt?: string;
|
|
74
|
+
}
|
|
75
|
+
|
|
47
76
|
/** A message that was read but not acted on, and why. */
|
|
48
77
|
export interface UnheardMessage {
|
|
49
78
|
senderAddress: string;
|
|
@@ -65,6 +94,13 @@ export interface InboundPollDeps {
|
|
|
65
94
|
/** Persisted receive cursor per transport. */
|
|
66
95
|
readCursor(transportModuleId: string): string | null;
|
|
67
96
|
writeCursor(transportModuleId: string, cursor: string | null): void;
|
|
97
|
+
/**
|
|
98
|
+
* Record what one read attempt produced. Called for EVERY attempt — a failed
|
|
99
|
+
* read must leave a trace, or "we could not read this transport" stays
|
|
100
|
+
* indistinguishable from "nobody replied". Optional so a caller that does not
|
|
101
|
+
* care (tests, one-off invocations) need not supply it.
|
|
102
|
+
*/
|
|
103
|
+
recordRead?(transportModuleId: string, record: TransportReadRecord): void;
|
|
68
104
|
now(): Date;
|
|
69
105
|
/**
|
|
70
106
|
* Transport for a route, so an ack can be broadcast to everyone else paged.
|
|
@@ -131,6 +167,12 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
|
|
|
131
167
|
for (const transportId of transportsWithRoutes(db)) {
|
|
132
168
|
const received = await deps.receiveFrom(transportId, deps.readCursor(transportId));
|
|
133
169
|
report.transportsPolled++;
|
|
170
|
+
deps.recordRead?.(transportId, {
|
|
171
|
+
at: deps.now().toISOString(),
|
|
172
|
+
outcome: received.status,
|
|
173
|
+
...(received.status === 'failed' ? { error: received.error } : {}),
|
|
174
|
+
messages: received.status === 'received' ? received.messages.length : 0,
|
|
175
|
+
});
|
|
134
176
|
// One dead transport must not stop the others being read — but it is
|
|
135
177
|
// RECORDED rather than skipped in silence, because "cannot read" and
|
|
136
178
|
// "nothing to read" are the two things an operator most needs to tell
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The per-transport record of whether celilo can still READ replies.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from the poll command because it now has two readers: the poller
|
|
5
|
+
* writes it, and the audit asserts on it. Keeping the key format private to the
|
|
6
|
+
* writer would have meant the audit re-deriving a string literal — the sort of
|
|
7
|
+
* duplication that silently stops matching.
|
|
8
|
+
*
|
|
9
|
+
* The shape it stores is the point. celilo's only per-transport state used to
|
|
10
|
+
* be the receive cursor, and `writeCursor` returns early when there is no
|
|
11
|
+
* cursor — which a failed read never produces. So the store could not
|
|
12
|
+
* REPRESENT a failure, and "unreadable since Tuesday" left exactly the same
|
|
13
|
+
* trace as "nobody replied since Tuesday" (#501).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { eq } from 'drizzle-orm';
|
|
17
|
+
import type { DbClient } from '../../db/client';
|
|
18
|
+
import { systemConfig } from '../../db/schema';
|
|
19
|
+
import { type TransportReadRecord, transportsWithRoutes } from './inbound-poller';
|
|
20
|
+
|
|
21
|
+
const READ_PREFIX = 'alerting.last_read.';
|
|
22
|
+
|
|
23
|
+
export function readLastRead(db: DbClient, transportModuleId: string): TransportReadRecord | null {
|
|
24
|
+
const row = db
|
|
25
|
+
.select()
|
|
26
|
+
.from(systemConfig)
|
|
27
|
+
.where(eq(systemConfig.key, `${READ_PREFIX}${transportModuleId}`))
|
|
28
|
+
.get();
|
|
29
|
+
if (!row?.value) return null;
|
|
30
|
+
try {
|
|
31
|
+
return JSON.parse(row.value) as TransportReadRecord;
|
|
32
|
+
} catch {
|
|
33
|
+
// A malformed row must not read as "never succeeded" — that would invent a
|
|
34
|
+
// fact and page someone about it.
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Record one attempt, carrying the last SUCCESS forward.
|
|
41
|
+
*
|
|
42
|
+
* Deliberately has no counterpart to `writeCursor`'s early return: a failure
|
|
43
|
+
* that writes nothing is why this record did not exist before. Carrying the
|
|
44
|
+
* success forward is what turns a log into an answer — a run of failures must
|
|
45
|
+
* not erase when the transport last actually worked.
|
|
46
|
+
*/
|
|
47
|
+
export function writeLastRead(
|
|
48
|
+
db: DbClient,
|
|
49
|
+
transportModuleId: string,
|
|
50
|
+
record: TransportReadRecord,
|
|
51
|
+
): void {
|
|
52
|
+
const key = `${READ_PREFIX}${transportModuleId}`;
|
|
53
|
+
const previous = readLastRead(db, transportModuleId);
|
|
54
|
+
const lastSuccessAt =
|
|
55
|
+
record.outcome === 'received' ? record.at : (previous?.lastSuccessAt ?? undefined);
|
|
56
|
+
const value = JSON.stringify({ ...record, ...(lastSuccessAt ? { lastSuccessAt } : {}) });
|
|
57
|
+
|
|
58
|
+
const existing = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
|
|
59
|
+
if (existing) {
|
|
60
|
+
db.update(systemConfig).set({ value }).where(eq(systemConfig.key, key)).run();
|
|
61
|
+
} else {
|
|
62
|
+
db.insert(systemConfig)
|
|
63
|
+
.values({ key, value, description: `Last inbound read attempt for ${transportModuleId}` })
|
|
64
|
+
.run();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface TransportReadStatus {
|
|
69
|
+
transportModuleId: string;
|
|
70
|
+
last: TransportReadRecord | null;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Read state for every transport that has a route pointing at it.
|
|
75
|
+
*
|
|
76
|
+
* Scoped to transports with routes on purpose: a transport nobody is routed to
|
|
77
|
+
* cannot fail to deliver anyone's acknowledgement, and paging about it would be
|
|
78
|
+
* noise that trains an operator to ignore this check.
|
|
79
|
+
*/
|
|
80
|
+
export function readAllTransportStatuses(db: DbClient): TransportReadStatus[] {
|
|
81
|
+
return transportsWithRoutes(db).map((transportModuleId) => ({
|
|
82
|
+
transportModuleId,
|
|
83
|
+
last: readLastRead(db, transportModuleId),
|
|
84
|
+
}));
|
|
85
|
+
}
|