@celilo/cli 0.16.2 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/CELILO_CORE_MODULES.md +1 -1
  2. package/CELILO_SUBSYSTEMS.md +39 -9
  3. package/drizzle/0019_backup_pid.sql +18 -0
  4. package/drizzle/meta/_journal.json +7 -0
  5. package/package.json +5 -5
  6. package/schemas/system_config.json +1 -1
  7. package/src/cli/command-tree-parser.ts +0 -1
  8. package/src/cli/commands/alerts-poll.ts +26 -1
  9. package/src/cli/commands/backup-sweep.ts +62 -0
  10. package/src/cli/commands/module-operations.test.ts +45 -1
  11. package/src/cli/commands/module-operations.ts +35 -12
  12. package/src/cli/commands/module-show.ts +1 -0
  13. package/src/cli/commands/storage-set-path.test.ts +281 -0
  14. package/src/cli/commands/storage-set-path.ts +190 -0
  15. package/src/cli/commands/system-audit.ts +14 -0
  16. package/src/cli/commands/system-migrate.ts +40 -0
  17. package/src/cli/commands/system-update.ts +6 -0
  18. package/src/cli/completion.ts +24 -3
  19. package/src/cli/fuel-gauge.ts +0 -1
  20. package/src/cli/generate-zsh-completion.ts +1 -1
  21. package/src/cli/index.ts +12 -0
  22. package/src/cli/tui/audit-state.test.ts +15 -1
  23. package/src/cli/tui/audit-state.ts +6 -0
  24. package/src/cli/tui/audit-tui.test.tsx +0 -1
  25. package/src/db/schema.ts +53 -9
  26. package/src/hooks/capability-loader.ts +30 -1
  27. package/src/ipam/allocator.ts +13 -3
  28. package/src/services/alerting/builtin-monitors.test.ts +42 -0
  29. package/src/services/alerting/builtin-monitors.ts +3 -0
  30. package/src/services/alerting/builtin-source.ts +15 -0
  31. package/src/services/alerting/inbound-poller.test.ts +63 -1
  32. package/src/services/alerting/inbound-poller.ts +42 -0
  33. package/src/services/alerting/read-records.ts +85 -0
  34. package/src/services/audit/abandoned-operations.test.ts +73 -0
  35. package/src/services/audit/abandoned-operations.ts +0 -0
  36. package/src/services/audit/disk-space.test.ts +111 -0
  37. package/src/services/audit/disk-space.ts +114 -0
  38. package/src/services/audit/index.test.ts +2 -0
  39. package/src/services/audit/index.ts +12 -0
  40. package/src/services/audit/transport-reads.test.ts +113 -0
  41. package/src/services/audit/transport-reads.ts +120 -0
  42. package/src/services/audit/types.ts +3 -0
  43. package/src/services/backup-create.ts +4 -4
  44. package/src/services/backup-in-flight-refusal.test.ts +2 -0
  45. package/src/services/backup-metadata.ts +4 -0
  46. package/src/services/backup-staging.test.ts +134 -0
  47. package/src/services/backup-staging.ts +192 -0
  48. package/src/services/backup-storage.ts +29 -0
  49. package/src/services/backup-sweep.test.ts +68 -0
  50. package/src/services/backup-sweep.ts +62 -0
  51. package/src/services/config-interview.ts +1 -1
  52. package/src/services/deploy-ansible.ts +0 -1
  53. package/src/services/disk-probe.test.ts +74 -0
  54. package/src/services/disk-probe.ts +145 -0
  55. package/src/services/fleet-checks.ts +15 -0
  56. package/src/services/module-operations.test.ts +22 -0
  57. package/src/services/module-operations.ts +48 -1
  58. package/src/services/module-subscriptions.test.ts +39 -6
  59. package/src/services/module-subscriptions.ts +6 -4
  60. package/src/services/module-types-generator.test.ts +6 -3
  61. package/src/services/module-types-generator.ts +12 -7
  62. package/src/services/storage-providers/local.ts +2 -1
  63. package/src/services/update/orchestrator.test.ts +2 -0
  64. package/src/variables/context.ts +6 -1
package/src/cli/index.ts CHANGED
@@ -839,6 +839,9 @@ Subcommands:
839
839
  list List all configured storage destinations
840
840
  verify <storage-id> Test storage connectivity and permissions
841
841
  set-default <id> Set the default backup storage destination
842
+ set-path <id> <path> Relocate a local destination to a new directory
843
+ Options:
844
+ --no-migrate Change the path without moving existing archives
842
845
  remove <storage-id> Remove a storage destination
843
846
  Options:
844
847
  --force Skip confirmation prompts
@@ -860,6 +863,10 @@ Examples:
860
863
  # Set default destination
861
864
  celilo storage set-default local-backups
862
865
 
866
+ # Move a local destination (archives at the old path are moved too;
867
+ # if that path is gone, the change proceeds with nothing to migrate)
868
+ celilo storage set-path local-backups /var/lib/celilo/backups
869
+
863
870
  # Remove storage
864
871
  celilo storage remove local-backups --force
865
872
 
@@ -1836,6 +1843,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
1836
1843
  return handleStorageSetDefault(parsed.args, parsed.flags);
1837
1844
  }
1838
1845
 
1846
+ if (parsed.subcommand === 'set-path') {
1847
+ const { handleStorageSetPath } = await import('./commands/storage-set-path');
1848
+ return handleStorageSetPath(parsed.args, parsed.flags);
1849
+ }
1850
+
1839
1851
  return {
1840
1852
  success: false,
1841
1853
  error: `Unknown storage subcommand: ${parsed.subcommand}\n\nRun "celilo storage --help" for usage`,
@@ -1,7 +1,9 @@
1
1
  import { describe, expect, test } from 'bun:test';
2
- import type { DriftFinding, SystemAuditReport } from '../../services/audit/types';
2
+ import type { DriftCategory, DriftFinding, SystemAuditReport } from '../../services/audit/types';
3
3
  import {
4
+ ALL_CATEGORIES,
4
5
  type AuditTuiState,
6
+ CATEGORY_LABELS,
5
7
  groupFindings,
6
8
  initState,
7
9
  reducer,
@@ -244,3 +246,15 @@ describe('selectedFinding', () => {
244
246
  expect(selectedFinding(initState(report([])))).toBeNull();
245
247
  });
246
248
  });
249
+
250
+ // ALL_CATEGORIES is a plain array, so the type system cannot require every
251
+ // DriftCategory to appear in it — a new category compiles fine and then never
252
+ // shows up in the TUI. CATEGORY_LABELS is a Record and IS exhaustive, so it is
253
+ // the honest source of truth to compare against.
254
+ describe('ALL_CATEGORIES covers every category', () => {
255
+ test('every labelled category is listed, and vice versa', () => {
256
+ expect([...ALL_CATEGORIES].sort()).toEqual(
257
+ (Object.keys(CATEGORY_LABELS) as DriftCategory[]).sort(),
258
+ );
259
+ });
260
+ });
@@ -77,12 +77,15 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
77
77
  'module_configs',
78
78
  'health',
79
79
  'backups',
80
+ 'abandoned_operations',
80
81
  'undeployed_modules',
81
82
  'unconfigured_modules',
82
83
  'services_credentials',
83
84
  'secrets_decryptable',
84
85
  'services_reachable',
85
86
  'machines_reachable',
87
+ 'disk_space',
88
+ 'transport_reads',
86
89
  'trusted_sources',
87
90
  ];
88
91
 
@@ -95,12 +98,15 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
95
98
  module_configs: 'Module configs',
96
99
  health: 'Module health',
97
100
  backups: 'Backups',
101
+ abandoned_operations: 'Abandoned operations',
98
102
  undeployed_modules: 'Undeployed modules',
99
103
  unconfigured_modules: 'Unconfigured modules',
100
104
  services_credentials: 'Service credentials',
101
105
  secrets_decryptable: 'Secrets',
102
106
  services_reachable: 'Service reachability',
103
107
  machines_reachable: 'Machine reachability',
108
+ disk_space: 'Disk space',
109
+ transport_reads: 'Transport readability',
104
110
  trusted_sources: 'Trusted networks',
105
111
  };
106
112
 
@@ -35,7 +35,6 @@ const FIXTURE: SystemAuditReport = {
35
35
  };
36
36
 
37
37
  function strip(s: string): string {
38
- // biome-ignore lint/suspicious/noControlCharactersInRegex: stripping CSI escape sequences
39
38
  return s.replace(/\[[0-9;]*m/g, '');
40
39
  }
41
40
 
package/src/db/schema.ts CHANGED
@@ -249,14 +249,31 @@ export const moduleBuilds = sqliteTable('module_builds', {
249
249
  });
250
250
 
251
251
  /**
252
- * Network zones type
253
- * - internal: Home network (192.168.0.0/24)
254
- * - dmz: Public-facing services in home lab (10.0.10.0/24)
255
- * - app: Internal application services (10.0.20.0/24)
256
- * - secure: Authentication and database services (10.0.30.0/24)
252
+ * Network zones — the DEFINED vocabulary, not the active set.
253
+ *
254
+ * A zone being listed here means celilo knows the name and will accept
255
+ * `network.<zone>.*` config for it. It does NOT mean the zone exists on a given
256
+ * fleet: per openspec/specs/progressive-zone-disclosure/spec.md, "deployable
257
+ * zones are those with a configured subnet", so a zone becomes REAL only when
258
+ * something declares `network.<zone>.subnet` — normally the module that supplies
259
+ * the network. Defining a zone here costs nothing and activates nothing.
260
+ *
261
+ * - internal: the semi-trusted LAN the management server sits on
262
+ * - dmz: public-facing services
263
+ * - app: internal application services
264
+ * - secure: authentication and database services
257
265
  * - secure-mgmt: celilo's own control plane (management server + management-plane
258
266
  * modules). Outside the data-plane tier chain; reaches every tier by trust.
259
- * - external: Internet-hosted services (outside home network)
267
+ * - external: internet-hosted services a cloud/VPS provider's network, NOT a
268
+ * VPN. Addressed by the provider, which is why it is excluded from IPAM below.
269
+ * - control-plane-vpn: the ADMINISTRATIVE remote-access client subnet, activated
270
+ * by the module that terminates the tunnel (`wireguard` writes
271
+ * `network.control-plane-vpn.subnet`). Named for its purpose rather than its
272
+ * technology because a fleet may run more than one VPN — a site-to-site link
273
+ * or a user VPN is a different network with different trust, and `vpn` would
274
+ * have been the wrong name to have to share. Distinct from `external` too:
275
+ * that is someone else's cloud, this is a network the fleet's own firewall
276
+ * holds a leg on and must translate for.
260
277
  */
261
278
  export const NETWORK_ZONES = [
262
279
  'internal',
@@ -265,6 +282,7 @@ export const NETWORK_ZONES = [
265
282
  'secure',
266
283
  'secure-mgmt',
267
284
  'external',
285
+ 'control-plane-vpn',
268
286
  ] as const;
269
287
 
270
288
  /**
@@ -276,15 +294,36 @@ export const NETWORK_ZONES = [
276
294
  export type NetworkZone = (typeof NETWORK_ZONES)[number];
277
295
 
278
296
  /**
279
- * Zones an IP allocation or reservation can name: every NetworkZone except
280
- * `external`, whose systems are addressed by the provider, not by our IPAM.
297
+ * Zones an IP allocation or reservation can name: every NetworkZone whose
298
+ * addresses celilo hands out.
299
+ *
300
+ * Two are excluded, for the same underlying reason — somebody else is the
301
+ * address authority:
302
+ * - `external`, whose systems are addressed by the cloud/VPS provider;
303
+ * - `vpn`, whose client addresses are assigned by the module terminating the
304
+ * tunnel. celilo allocating into that subnet would collide with the VPN
305
+ * server's own assignments.
281
306
  *
282
307
  * Derived rather than hand-written for the same reason as NetworkZone above —
283
308
  * the previous hand-written union was copied into two column definitions and a
284
309
  * cast in machine-pool.ts, and the cast had already drifted (it was missing
285
310
  * `secure-mgmt`, and its comment claimed the only difference was `external`).
286
311
  */
287
- export type AllocatableZone = Exclude<NetworkZone, 'external'>;
312
+ export type AllocatableZone = Exclude<NetworkZone, 'external' | 'control-plane-vpn'>;
313
+
314
+ /** The zones whose addresses celilo hands out, as a runtime list. */
315
+ export const ALLOCATABLE_ZONES: AllocatableZone[] = NETWORK_ZONES.filter(
316
+ (zone): zone is AllocatableZone => zone !== 'external' && zone !== 'control-plane-vpn',
317
+ );
318
+
319
+ /**
320
+ * Is this a zone celilo allocates addresses in? Use this rather than testing
321
+ * `zone !== 'external'` by hand — that check predates `vpn` and read as "the one
322
+ * externally-addressed zone" when there are now two.
323
+ */
324
+ export function isAllocatableZone(zone: NetworkZone): zone is AllocatableZone {
325
+ return (ALLOCATABLE_ZONES as string[]).includes(zone);
326
+ }
288
327
 
289
328
  /**
290
329
  * Container services table
@@ -679,6 +718,11 @@ export const backups = sqliteTable('backups', {
679
718
  status: text('status').$type<BackupStatus>().notNull().default('in_progress'),
680
719
  errorMessage: text('error_message'),
681
720
  name: text('name'), // optional human-readable name/annotation
721
+ // The process assembling this backup's staging directory. Lets the staging
722
+ // reaper (services/backup-staging.ts) tell a live backup from one whose
723
+ // process was killed before its `finally` could clean up. Nullable: rows
724
+ // written before this column existed have no pid and age out via the TTL.
725
+ pid: integer('pid'),
682
726
  startedAt: integer('started_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
683
727
  completedAt: integer('completed_at', { mode: 'timestamp' }),
684
728
  });
@@ -28,7 +28,14 @@ import type {
28
28
  } from '@celilo/capabilities';
29
29
  import { and, eq } from 'drizzle-orm';
30
30
  import type { DbClient } from '../db/client';
31
- import { capabilities, modules, secrets, systemConfig, webRoutes } from '../db/schema';
31
+ import {
32
+ NETWORK_ZONES,
33
+ capabilities,
34
+ modules,
35
+ secrets,
36
+ systemConfig,
37
+ webRoutes,
38
+ } from '../db/schema';
32
39
  import { decryptSecret } from '../secrets/encryption';
33
40
  import { getOrCreateMasterKey } from '../secrets/master-key';
34
41
  import { emitWebRoutesChangedAndWait } from '../services/celilo-events';
@@ -593,6 +600,7 @@ function buildCapabilityInterface(
593
600
  zoneTiers: zones?.zoneTiers ?? [],
594
601
  trustedSubnets: zones?.trustedSubnets ?? [],
595
602
  controlPlaneSubnet: zones?.controlPlaneSubnet,
603
+ frontedSubnets: zones?.frontedSubnets ?? [],
596
604
  },
597
605
  store,
598
606
  undefined, // no upstream — the chain path handles that
@@ -699,6 +707,22 @@ interface FirewallZones {
699
707
  trustedSubnets: string[];
700
708
  /** celilo's control-plane network, as a DESTINATION for trusted sources. */
701
709
  controlPlaneSubnet?: string;
710
+ /**
711
+ * Every declared zone subnet — read from
712
+ * `network.<zone>.subnet` across all of `NETWORK_ZONES`, not just the three
713
+ * data-plane tiers.
714
+ *
715
+ * A downstream firewall must translate for every network behind it, and the
716
+ * tiers are only the ones that happen to form the dmz→app→secure chain.
717
+ * `secure-mgmt` and the control-plane VPN are equally behind it and equally
718
+ * unroutable untranslated. Reading the canonical zone list rather than the tier
719
+ * list means declaring a zone's subnet is sufficient to get it translated — no
720
+ * module registration required.
721
+ *
722
+ * The egress network is NOT excluded here: the firewall derives that from its
723
+ * own routing table, so nothing on this side has to guess which zone it is.
724
+ */
725
+ frontedSubnets: string[];
702
726
  }
703
727
 
704
728
  /** The module that IS celilo's control plane; its network is what we trust. */
@@ -802,6 +826,10 @@ function loadFirewallZones(db: DbClient): FirewallZones {
802
826
  zoneTiers,
803
827
  trustedSubnets: loadTrustedSubnets(db).map((e) => e.subnet),
804
828
  controlPlaneSubnet: loadControlPlaneSubnet(db) ?? readZoneSubnet(db, 'internal'),
829
+ // Every zone celilo knows about, not the tier list — see the field doc.
830
+ frontedSubnets: NETWORK_ZONES.map((zone) => readZoneSubnet(db, zone)).filter(
831
+ (subnet): subnet is string => !!subnet,
832
+ ),
805
833
  };
806
834
  }
807
835
 
@@ -953,6 +981,7 @@ async function buildFirewallChain(
953
981
  zoneTiers: zones.zoneTiers,
954
982
  trustedSubnets: zones.trustedSubnets,
955
983
  controlPlaneSubnet: zones.controlPlaneSubnet,
984
+ frontedSubnets: zones.frontedSubnets,
956
985
  },
957
986
  store,
958
987
  currentUpstream,
@@ -9,16 +9,26 @@ import type { BunSQLiteDatabase } from 'drizzle-orm/bun-sqlite';
9
9
  import type { DbClient } from '../db/client';
10
10
  import { ipAllocations, ipReservations, systemConfig, vmidReservations } from '../db/schema';
11
11
  import type { NewIpAllocation, NewIpReservation, NewVmidReservation } from '../db/schema';
12
+ import { ALLOCATABLE_ZONES, type AllocatableZone } from '../db/schema';
12
13
  import type * as schema from '../db/schema';
13
14
  import { generateIPsInSubnet, isIPInRange, isInSubnet, stripCIDR } from './subnet-parser';
14
15
 
15
16
  // Type that accepts both database client and transaction
16
17
  type DbOrTransaction = BunSQLiteDatabase<typeof schema> | DbClient;
17
18
 
18
- /** Zones that support IPAM auto-allocation of VMID and container IP */
19
- export type IpamZone = 'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal';
19
+ /**
20
+ * Zones that support IPAM auto-allocation of VMID and container IP.
21
+ *
22
+ * ALIASED to `AllocatableZone`, not hand-written. This was its own union
23
+ * (`'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal'`) that happened to
24
+ * agree with `AllocatableZone` — a fourth hand-maintained copy of the zone list,
25
+ * in a repo where three earlier copies had already drifted and left comments
26
+ * saying so. Deriving means a zone celilo stops allocating for (as `vpn` is,
27
+ * since the tunnel module assigns client addresses) cannot be missed here.
28
+ */
29
+ export type IpamZone = AllocatableZone;
20
30
 
21
- const IPAM_ZONES: IpamZone[] = ['internal', 'dmz', 'app', 'secure', 'secure-mgmt'];
31
+ const IPAM_ZONES: IpamZone[] = ALLOCATABLE_ZONES;
22
32
 
23
33
  /**
24
34
  * Infer which zone an IP address belongs to by checking configured zone subnets.
@@ -27,6 +27,12 @@ describe('targetKindForCategory', () => {
27
27
  expect(targetKindForCategory('backups')).toBe('module');
28
28
  });
29
29
 
30
+ // Machine-scoped so suppression inherits the machine topology: an unreachable
31
+ // host suppresses its own disk alert instead of paging twice for one dead box.
32
+ test('disk_space is machine-scoped', () => {
33
+ expect(targetKindForCategory('disk_space')).toBe('machine');
34
+ });
35
+
30
36
  // Whole-system categories have no narrower subject to suppress against.
31
37
  test('unmapped category falls back to system', () => {
32
38
  expect(targetKindForCategory('cli_version')).toBe('system');
@@ -46,6 +52,42 @@ describe('severityForDriftSeverity', () => {
46
52
  });
47
53
 
48
54
  describe('failingKeysFromFindings', () => {
55
+ // Asserts the WHOLE key, not a prefix. A prefix-only assertion is exactly why
56
+ // #596 survived — machines_reachable emits a UUID where suppression expects a
57
+ // hostname, and `toContain('builtin:machines_reachable/machine:')` passes for
58
+ // both. This check must be pinned to the identifier itself.
59
+ test('disk_space keys on the hostname, in full', () => {
60
+ const keys = failingKeysFromFindings(
61
+ 'disk_space',
62
+ [
63
+ finding({
64
+ category: 'disk_space',
65
+ code: 'disk_critical',
66
+ severity: 'blocked',
67
+ subject: 'celilo-mgr',
68
+ message: 'celilo-mgr: root filesystem 96% full',
69
+ }),
70
+ ],
71
+ 'critical',
72
+ );
73
+
74
+ expect(keys).toHaveLength(1);
75
+ expect(keys[0]?.key).toBe('builtin:disk_space/machine:celilo-mgr');
76
+ expect(keys[0]?.severity).toBe('critical');
77
+ });
78
+
79
+ // An unmeasurable host records without paging.
80
+ test('an unmeasured disk downgrades to warning', () => {
81
+ const keys = failingKeysFromFindings(
82
+ 'disk_space',
83
+ [finding({ category: 'disk_space', severity: 'todo', subject: 'iot' })],
84
+ 'critical',
85
+ );
86
+
87
+ expect(keys[0]?.key).toBe('builtin:disk_space/machine:iot');
88
+ expect(keys[0]?.severity).toBe('warning');
89
+ });
90
+
49
91
  test('projects findings into builtin keys', () => {
50
92
  const keys = failingKeysFromFindings('machines_reachable', [finding()], 'critical');
51
93
  expect(keys).toEqual([
@@ -24,9 +24,12 @@ import { type FailingKey, builtinAlertKey } from './keys';
24
24
  */
25
25
  const TARGET_KIND_BY_CATEGORY: Partial<Record<DriftCategory, string>> = {
26
26
  machines_reachable: 'machine',
27
+ disk_space: 'machine',
28
+ transport_reads: 'module',
27
29
  services_reachable: 'service',
28
30
  services_credentials: 'service',
29
31
  backups: 'module',
32
+ abandoned_operations: 'module',
30
33
  health: 'module',
31
34
  module_versions: 'module',
32
35
  module_configs: 'module',
@@ -13,16 +13,21 @@
13
13
  */
14
14
 
15
15
  import type { DbClient } from '../../db/client';
16
+ import { auditAbandonedOperations, loadAbandonedOperations } from '../audit/abandoned-operations';
16
17
  import { loadBackupAuditInfo } from '../audit/backup-source';
17
18
  import { auditBackups } from '../audit/backups';
19
+ import { auditDiskSpace } from '../audit/disk-space';
18
20
  import { auditMachinesReachable } from '../audit/machines-reachable';
19
21
  import type { DriftCategory, DriftFinding } from '../audit/types';
22
+ import { probeDiskUsage } from '../disk-probe';
20
23
  import { probeMachines } from '../machine-probe';
21
24
 
22
25
  /** Categories a monitor can currently schedule. */
23
26
  export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
24
27
  'machines_reachable',
25
28
  'backups',
29
+ 'disk_space',
30
+ 'abandoned_operations',
26
31
  ];
27
32
 
28
33
  export function isSchedulableBuiltin(category: string): category is DriftCategory {
@@ -37,12 +42,22 @@ export async function runBuiltinCheckForMonitor(
37
42
  return auditMachinesReachable({ results: await probeMachines() });
38
43
  }
39
44
 
45
+ // Same shape as machines_reachable — one SSH round trip per machine, bounded
46
+ // — but it MEASURES the local box rather than exempting it. See disk-probe.ts.
47
+ if (category === 'disk_space') {
48
+ return auditDiskSpace({ results: await probeDiskUsage() });
49
+ }
50
+
40
51
  // Local DB reads only — cheap enough to run on every sweep, which is
41
52
  // the whole reason this category is schedulable and most are not.
42
53
  if (category === 'backups') {
43
54
  return auditBackups({ modules: loadBackupAuditInfo(db) });
44
55
  }
45
56
 
57
+ if (category === 'abandoned_operations') {
58
+ return auditAbandonedOperations({ records: loadAbandonedOperations(db) });
59
+ }
60
+
46
61
  throw new Error(
47
62
  `Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
48
63
  );
@@ -7,7 +7,12 @@ import { eq } from 'drizzle-orm';
7
7
  import type { DbClient } from '../../db/client';
8
8
  import { type Route, alerts, modules, monitors, notificationDeliveries } from '../../db/schema';
9
9
  import { setupTestDatabase } from '../../test-utils/setup-test-db';
10
- import { type InboundPollDeps, pollInbound, transportsWithRoutes } from './inbound-poller';
10
+ import {
11
+ type InboundPollDeps,
12
+ type TransportReadRecord,
13
+ pollInbound,
14
+ transportsWithRoutes,
15
+ } from './inbound-poller';
11
16
  import { moduleCheckAlertKey } from './keys';
12
17
  import { createPerson, createRoute } from './people';
13
18
  import { mintDelivery } from './tokens';
@@ -386,6 +391,63 @@ describe('pollInbound', () => {
386
391
  expect(ackedBy()).toBe(peterRoute.personId);
387
392
  });
388
393
 
394
+ // The gap this closes. The ONLY per-transport state celilo persisted was the
395
+ // cursor, and writeCursor early-returns when there is no cursor — which a
396
+ // failed read never produces. So the store could not REPRESENT a failure, and
397
+ // "unreadable for a week" and "nobody replied for a week" left identical
398
+ // traces (#501).
399
+ describe('recording what each read attempt produced', () => {
400
+ const records: Array<[string, TransportReadRecord]> = [];
401
+ const recording = (over: Partial<InboundPollDeps> = {}) =>
402
+ deps([], { recordRead: (t, r) => records.push([t, r]), ...over });
403
+
404
+ beforeEach(() => {
405
+ records.length = 0;
406
+ });
407
+
408
+ test('a FAILED read is recorded — the case the cursor could never express', async () => {
409
+ await pollInbound(
410
+ db,
411
+ recording({ receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
412
+ );
413
+ expect(records).toHaveLength(1);
414
+ expect(records[0][0]).toBe('signal');
415
+ expect(records[0][1]).toMatchObject({ outcome: 'failed', error: 'connection refused' });
416
+ });
417
+
418
+ test('a successful read is recorded with how many messages it returned', async () => {
419
+ await pollInbound(
420
+ db,
421
+ deps([inbound(PETER, 'hello')], { recordRead: (t, r) => records.push([t, r]) }),
422
+ );
423
+ expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 1 });
424
+ });
425
+
426
+ // Zero messages is not failure. Conflating them is the original bug.
427
+ test('an empty but successful read records received, not failed', async () => {
428
+ await pollInbound(db, recording());
429
+ expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 0 });
430
+ });
431
+
432
+ test('a unidirectional transport is recorded as such', async () => {
433
+ await pollInbound(db, recording({ receiveFrom: async () => ({ status: 'unidirectional' }) }));
434
+ expect(records[0][1].outcome).toBe('unidirectional');
435
+ });
436
+
437
+ // Every attempt, not just interesting ones — a gap in the record would be
438
+ // read as "nothing happened".
439
+ test('every attempt is recorded, and carries when it happened', async () => {
440
+ await pollInbound(db, recording());
441
+ expect(records[0][1].at).toBe(NOW.toISOString());
442
+ });
443
+
444
+ // Optional dep: a caller that does not persist must still poll.
445
+ test('polling works with no recorder attached', async () => {
446
+ const report = await pollInbound(db, deps([]));
447
+ expect(report.transportsPolled).toBe(1);
448
+ });
449
+ });
450
+
389
451
  test('a route pointing at a transport nobody uses is not polled', () => {
390
452
  db.delete(alerts).run();
391
453
  expect(transportsWithRoutes(db)).toEqual(['signal']);
@@ -44,6 +44,35 @@ export interface TransportFailure {
44
44
  error: string;
45
45
  }
46
46
 
47
+ /**
48
+ * The outcome of one attempt to read a transport, as recorded for later.
49
+ *
50
+ * Written on EVERY attempt, including failures — which is the whole point. The
51
+ * only per-transport state celilo persisted before this was the cursor, and a
52
+ * failed read produces no cursor, so `writeCursor` returned early and nothing
53
+ * was written. The store could not REPRESENT a failure, so an absence of
54
+ * recorded failures was never evidence there had been none: a transport that
55
+ * had not been readable for a week looked identical to one nobody had replied
56
+ * on (#501).
57
+ */
58
+ export interface TransportReadRecord {
59
+ /** When the attempt happened, ISO-8601. */
60
+ at: string;
61
+ outcome: 'received' | 'unidirectional' | 'failed';
62
+ /** Present only when `failed`. */
63
+ error?: string;
64
+ /** How many messages the read returned. Zero is not the same as failure. */
65
+ messages: number;
66
+ /**
67
+ * When a read last SUCCEEDED, carried forward across failures.
68
+ *
69
+ * This is the field that answers the question the old state could not: a
70
+ * transport reporting "0 messages" for a week and one that has not been
71
+ * readable for a week are the same picture until you can see this.
72
+ */
73
+ lastSuccessAt?: string;
74
+ }
75
+
47
76
  /** A message that was read but not acted on, and why. */
48
77
  export interface UnheardMessage {
49
78
  senderAddress: string;
@@ -65,6 +94,13 @@ export interface InboundPollDeps {
65
94
  /** Persisted receive cursor per transport. */
66
95
  readCursor(transportModuleId: string): string | null;
67
96
  writeCursor(transportModuleId: string, cursor: string | null): void;
97
+ /**
98
+ * Record what one read attempt produced. Called for EVERY attempt — a failed
99
+ * read must leave a trace, or "we could not read this transport" stays
100
+ * indistinguishable from "nobody replied". Optional so a caller that does not
101
+ * care (tests, one-off invocations) need not supply it.
102
+ */
103
+ recordRead?(transportModuleId: string, record: TransportReadRecord): void;
68
104
  now(): Date;
69
105
  /**
70
106
  * Transport for a route, so an ack can be broadcast to everyone else paged.
@@ -131,6 +167,12 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
131
167
  for (const transportId of transportsWithRoutes(db)) {
132
168
  const received = await deps.receiveFrom(transportId, deps.readCursor(transportId));
133
169
  report.transportsPolled++;
170
+ deps.recordRead?.(transportId, {
171
+ at: deps.now().toISOString(),
172
+ outcome: received.status,
173
+ ...(received.status === 'failed' ? { error: received.error } : {}),
174
+ messages: received.status === 'received' ? received.messages.length : 0,
175
+ });
134
176
  // One dead transport must not stop the others being read — but it is
135
177
  // RECORDED rather than skipped in silence, because "cannot read" and
136
178
  // "nothing to read" are the two things an operator most needs to tell
@@ -0,0 +1,85 @@
1
+ /**
2
+ * The per-transport record of whether celilo can still READ replies.
3
+ *
4
+ * Extracted from the poll command because it now has two readers: the poller
5
+ * writes it, and the audit asserts on it. Keeping the key format private to the
6
+ * writer would have meant the audit re-deriving a string literal — the sort of
7
+ * duplication that silently stops matching.
8
+ *
9
+ * The shape it stores is the point. celilo's only per-transport state used to
10
+ * be the receive cursor, and `writeCursor` returns early when there is no
11
+ * cursor — which a failed read never produces. So the store could not
12
+ * REPRESENT a failure, and "unreadable since Tuesday" left exactly the same
13
+ * trace as "nobody replied since Tuesday" (#501).
14
+ */
15
+
16
+ import { eq } from 'drizzle-orm';
17
+ import type { DbClient } from '../../db/client';
18
+ import { systemConfig } from '../../db/schema';
19
+ import { type TransportReadRecord, transportsWithRoutes } from './inbound-poller';
20
+
21
+ const READ_PREFIX = 'alerting.last_read.';
22
+
23
+ export function readLastRead(db: DbClient, transportModuleId: string): TransportReadRecord | null {
24
+ const row = db
25
+ .select()
26
+ .from(systemConfig)
27
+ .where(eq(systemConfig.key, `${READ_PREFIX}${transportModuleId}`))
28
+ .get();
29
+ if (!row?.value) return null;
30
+ try {
31
+ return JSON.parse(row.value) as TransportReadRecord;
32
+ } catch {
33
+ // A malformed row must not read as "never succeeded" — that would invent a
34
+ // fact and page someone about it.
35
+ return null;
36
+ }
37
+ }
38
+
39
+ /**
40
+ * Record one attempt, carrying the last SUCCESS forward.
41
+ *
42
+ * Deliberately has no counterpart to `writeCursor`'s early return: a failure
43
+ * that writes nothing is why this record did not exist before. Carrying the
44
+ * success forward is what turns a log into an answer — a run of failures must
45
+ * not erase when the transport last actually worked.
46
+ */
47
+ export function writeLastRead(
48
+ db: DbClient,
49
+ transportModuleId: string,
50
+ record: TransportReadRecord,
51
+ ): void {
52
+ const key = `${READ_PREFIX}${transportModuleId}`;
53
+ const previous = readLastRead(db, transportModuleId);
54
+ const lastSuccessAt =
55
+ record.outcome === 'received' ? record.at : (previous?.lastSuccessAt ?? undefined);
56
+ const value = JSON.stringify({ ...record, ...(lastSuccessAt ? { lastSuccessAt } : {}) });
57
+
58
+ const existing = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
59
+ if (existing) {
60
+ db.update(systemConfig).set({ value }).where(eq(systemConfig.key, key)).run();
61
+ } else {
62
+ db.insert(systemConfig)
63
+ .values({ key, value, description: `Last inbound read attempt for ${transportModuleId}` })
64
+ .run();
65
+ }
66
+ }
67
+
68
+ export interface TransportReadStatus {
69
+ transportModuleId: string;
70
+ last: TransportReadRecord | null;
71
+ }
72
+
73
+ /**
74
+ * Read state for every transport that has a route pointing at it.
75
+ *
76
+ * Scoped to transports with routes on purpose: a transport nobody is routed to
77
+ * cannot fail to deliver anyone's acknowledgement, and paging about it would be
78
+ * noise that trains an operator to ignore this check.
79
+ */
80
+ export function readAllTransportStatuses(db: DbClient): TransportReadStatus[] {
81
+ return transportsWithRoutes(db).map((transportModuleId) => ({
82
+ transportModuleId,
83
+ last: readLastRead(db, transportModuleId),
84
+ }));
85
+ }