@celilo/cli 0.16.2 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -197,6 +197,7 @@ Creation, scheduling and freshness. A module declares an `on_backup` hook and a
197
197
  - **The sweep** — `apps/celilo/src/services/backup-sweep.ts` — `runBackupSweep` (the pass that makes backups run by themselves: for each eligible module, is its declared cadence due → back it up → apply declared retention) + `ensureBackupSweepSubscriber`. Driven by `celilo backup sweep` on `timer.tick.1h` — the coarsest tick that can still serve an `hourly` cadence. Armed automatically: `registerModuleSubscriptions` registers the subscriber for any module declaring an `on_backup` hook, so it appears on install or `module update`. A run refused by the in-flight operation lock is a skip retried next tick, never a failure.
198
198
  - **Freshness audit** — `apps/celilo/src/services/audit/backups.ts` (`auditBackups`) — `backup_missing` / `backup_stale` drift findings against the same declared cadence.
199
199
  - **CLI** — `apps/celilo/src/cli/commands/` — `backup-sweep.ts`, `backup-create.ts` (also `celilo module backup`), `backup-list.ts`, `backup-restore.ts`, `backup-prune.ts`, `backup-delete.ts`, `backup-import.ts`, `backup-pull.ts`, `backup-name.ts`.
200
+ - **Storage destinations CLI** — `storage-add-local.ts`, `storage-add-s3.ts`, `storage-list.ts`, `storage-verify.ts`, `storage-set-default.ts`, `storage-set-path.ts` (relocate a local destination, migrating existing archives unless `--no-migrate`), `storage-remove.ts`. Any credential change goes through `updateStorageCredentials` in `backup-storage.ts`, which clears the verification stamp — a `✓ Verified` must never describe a destination it was not measured against (#566).
200
201
 
201
202
  ## Persistence
202
203
 
@@ -218,7 +219,8 @@ Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <h
218
219
  - **Access control** — `apps/celilo/src/services/api-access.ts` — `grantPrincipal`, `isAuthorized` (deny-by-default, `command:subcommand` grants), `renderAuthorizedKeys`. Table: `api_principals` (`apps/celilo/src/db/schema.ts`). CLI: `apps/celilo/src/cli/commands/api.ts` (`api grant|list|revoke|authorized-keys|key new`).
219
220
  - **Mid-run interview bridge (`kind:daemon` responder)** — `apps/celilo/src/services/remote-responder.ts` — `startRemoteResponder` bridges bus `interview.required.*` ↔ wire.
220
221
  - **Server provisioning** — the `celilo-bootstrap` deb (`packaging/celilo-bootstrap/scripts/postinst`) creates the non-root `celilo-api` landing account + sshd; membership in the `celilo` group + `/etc/sudoers.d/celilo` (`!use_pty`) gives api-serve DB access via the wrapper's sudo-drop.
221
- - **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100). It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`.
222
+ - **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100). It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`. **This upgrades celilo ITSELF — not the modules it manages. For those, see Module auto-upgrade below; the two are routinely confused.**
223
+ - **Module auto-upgrade (registry-poll CD)** — the *pull* half of continuous deployment: celilo-mgr polls the registry and upgrades opted-in modules unattended. Spec: `openspec/specs/module-auto-upgrade/spec.md`. Entry points: `apps/celilo/src/cli/commands/module-upgrade.ts` — `runRegistryPoll` (the `--poll` path), `selectPollTargets` (pure: `autoUpgrade && latest && change ∉ {up-to-date, ahead}`), `upgradeOneModule` (update → backup → deploy → verify), `needsPreUpgradeBackup`, `pickAutoUpgrade`/`pickUpgradePolicy` (both fail closed/safe); `classifyVersionChange` in `module-update.ts` (treats a registry `+N` revision as a patch); `resolveDeployPosture` in `apps/celilo/src/services/deploy-posture.ts`. Trigger: celilo-mgmt's `registry-poll` subscription (`modules/celilo-mgmt/manifest.yml`) on `timer.tick.15m` with handler **`celilo module upgrade --poll`** — the flag is REQUIRED, since the dispatcher appends the event id positionally and a bare handler would consume it as the optional module name (silent: 3108 deliveries, 0 successes). Operator controls are framework config keys settable on ANY module (`FRAMEWORK_CONFIG_KEYS` in `module-config.ts`): `auto_upgrade` (opt-in, default false) and `upgrade_policy` (`by-semver`|`always-safe`|`always-fast`), validated at set time because both readers fail open. ⚠️ `always-safe` guarantees safe *posture*, NOT a backup — `needsPreUpgradeBackup` also requires the TARGET manifest to declare an `on_backup` hook, else it warns and proceeds. Confirm a data-bearing module declares `on_backup` before enabling `auto_upgrade` on it. The *build* half (app CI publishing a `.netapp` on merge) is not yet shipped — `openspec/changes/build-bus-poll-cd`.
222
224
  - **MCP service (`@celilo/mcp`)** — `packages/mcp/src/` — an operator-facing stdio MCP server (official `@modelcontextprotocol/sdk`, bin `celilo-mcp`) that drives a remote celilo server over the Remote API for an AI client. Two-item config (`config.ts`: `server` + `defaultUser`, env or `~/.config/celilo-mcp/config.json`). Dual-principal auth (`auth.ts`: `celilo-mcp auth setup` enrolls read-only `celilo-mcp-ro` + full `celilo-mcp-rw` ed25519 keypairs, prints the exact `celilo api grant` lines the operator runs server-side). Transport (`transport.ts`): reuses `@celilo/core` `runRemoteClient`, selecting the principal by `ssh -i <key>` and capturing structured output. Tool surface is generated LIVE from the server's command registry — `registry-fetch.ts` fetches `celilo commands --json` (+ `service list --json` for configured providers) over the RO principal on connect; `tools-from-registry.ts` (pure) projects that into one tool per runnable leaf, grouped by top-level command (`celilo_module_*`, `celilo_proxmox_*`, …), each with a Zod input schema from the leaf's args/flags and a read/write tag → RO/RW routing, plus a generic `celilo_run` escape hatch. Auto-detect hides provider-gated groups (e.g. `celilo_proxmox_*` until a Proxmox service is configured) and re-detects on a timer, emitting `notifications/tools/list_changed` when the surface changes. Coverage gate (`tests/coverage.test.ts`) asserts every registry leaf maps to a tool. Composite RO troubleshooting tools (`troubleshoot.ts` pure correlation + `troubleshoot-tools.ts` thin adapters): `celilo_assess_module <id>` and `celilo_fleet_status` correlate `celilo audit --json` (the drift backbone) with the `module list --json` roster into a per-module / fleet-wide verdict. Design: `openspec/changes/celilo-mcp-service/proposal.md`. (Distinct from the dev/ops `@celilo/mcp-server` below.)
223
225
 
224
226
  ## E2E simulation
@@ -227,4 +229,5 @@ Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <h
227
229
  - **Run wrapper** — `infra/scripts/cele2e-run.sh` lives in the separate `infra/` clone, **not** in this repo. See the cele2e section of the repo-root `CLAUDE.md` for the operator workflow.
228
230
  - **Signal simulators** — three containers, three jobs. `docker/Dockerfile.signal-cli` runs the REAL unlinked daemon (`network().withSignalCli()`, reachable at `signal-cli.lab`) so `e2e/tests/signal-contract.test.ts` can re-check celilo's understanding of the JSON-RPC surface against the actual binary — the pebble pattern. `docker/Dockerfile.signal-sim` runs `simulators/signal-cli/server.ts` (`withSignalSim()`, `signal-sim.lab`), the drivable stand-in with a control surface (`/_control/inbound`, `/_control/sent`, `/_control/unlink`). `/_control/inbound` from the LINKED account's own number emits a `syncMessage.sentMessage` transcript rather than a `dataMessage`, because that is the only shape the real daemon delivers a note-to-self in — and a note-to-self is what every reply is in the default single-operator setup (#460). `docker/Dockerfile.signal-release` (`withSignalRelease()`, `signal-release.lab`) serves the signal-cli release tarball so the module's deploy-time download resolves inside the sealed network — the download is served, never skipped. The libsignal aarch64 native is compiled at `build-infra` time from the module's own recipe (`packages/e2e/scripts/stage-libsignal.ts` → `modules/signal/build/`) and staged into the apt-repo pool, so the recipe is exercised for real on every rebuild while the deploy stays fast and the network stays sealed.
229
231
  - **Public-boundary NAT model** — `packages/e2e/config/routing/` — exactly ONE NAT sits between the fleet and the simulated internet: the customer firewall (`fw-main` in `direct-internet`, `fw-isp` in the two-layer default), which MASQUERADEs to `100.100.0.100`. The ISP edge `fw-ext` ROUTES the customer's `100.100.0.0/24` and must never re-NAT it (`-s 100.100.0.0/24 -j RETURN` ahead of its MASQUERADE) — an ISP does not NAT a subscriber that already holds a public address. This is load-bearing, not cosmetic: Namecheap-style DDNS registers the SOURCE address when the caller omits `ip=`, which is how celilo registers public names since #464/#466, so a second NAT here publishes the simulator's own address for every public hostname and quietly breaks ACME, inbound reach, and seeded apex records. Constants: `externalWanIp()` / `externalWanSubnet()` in `src/types.ts`. The corollary: every simulator on `internet-external` must default-route via fw-ext (`100.64.0.1`) so it can reply to the customer's public address — Docker's bridge gateway has no path across networks. `config/routing/public-sim-entrypoint.sh` is the shared two-liner; the DNS hierarchy, pebble, isitup and celilo-website-sim already carried it, and npm-registry / registry / apt-repo / minio / cpanel-host only appeared to work because the double NAT put fw-ext's on-link address in the source field.
232
+ - **Simulated address plan** — `packages/e2e/src/types.ts` — `SIM_PRIVATE_SUPERNET` (`10.226.0.0/16`), `zoneIp(zone, host)`, `ZONE_SUBNETS`, `ZONE_GATEWAYS`. Every simulated PRIVATE zone is derived from this one table; the compose generator, `zone-classifier.ts` and the harness's `system init` all read it, so renumbering the whole sim is a one-line change. The sim deliberately does NOT reuse a real fleet's zone /24s (#539): the previous plan was byte-identical to production's, so a stack leaked on celilo's own forgejo-builder — which lives in the real dmz — claimed the builder's own subnet and blackholed every containerized CI job's route to the forge for ~135s. Teardown cannot prevent that (a SIGKILL runs no handler), so the addresses moved instead; the second, better reason is that a suite which only passes on production's exact octets is asserting one site's address plan rather than celilo's behaviour. `src/address-plan.test.ts` is the recurrence gate — it fails if a zone leaves the supernet, or if any retired fleet prefix reappears anywhere in `packages/e2e`, `e2e/tests` or `modules/*/e2e`.
230
233
  - **MCP server (agent-driven e2e)** — `packages/mcp-server/src/index.ts` — stdio MCP server exposing `start_run`/`run_status`/`run_result`/`stop_run` (detached cele2e runs read off the event bus, no ANSI scraping) + `env_check` (docker VM, run-lock, shared-infra, mgmt image CLI version, netapps). Dev/ops tool, `private`, not shipped to consumers.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@celilo/cli",
3
- "version": "0.16.2",
3
+ "version": "0.17.0",
4
4
  "description": "Celilo — home lab orchestration CLI",
5
5
  "type": "module",
6
6
  "bin": {
@@ -59,7 +59,7 @@
59
59
  "@aws-sdk/client-s3": "^3.1024.0",
60
60
  "@celilo/capabilities": "^0.9.1",
61
61
  "@celilo/cli-display": "^0.1.9",
62
- "@celilo/core": "^0.3.0",
62
+ "@celilo/core": "^0.3.1",
63
63
  "@celilo/event-bus": "^0.1.10",
64
64
  "@clack/prompts": "^1.1.0",
65
65
  "ajv": "^8.18.0",
@@ -17,6 +17,7 @@ import { systemConfig } from '../../db/schema';
17
17
  import { makeReceiver, pollInbound } from '../../services/alerting/inbound-poller';
18
18
  import { startNotificationResponder } from '../../services/alerting/notification-responder';
19
19
  import { listRoutes } from '../../services/alerting/people';
20
+ import { readLastRead, writeLastRead } from '../../services/alerting/read-records';
20
21
  import { loadNotificationTransport } from '../../services/alerting/transport-loader';
21
22
  import type { CommandResult } from '../types';
22
23
 
@@ -52,6 +53,26 @@ function writeCursor(
52
53
  }
53
54
  }
54
55
 
56
+ /**
57
+ * " (last read OK 3h ago)" or " (never read successfully)".
58
+ *
59
+ * Appended to a read failure because the failure alone does not say how bad it
60
+ * is. One failed poll is a blip; a transport that has not been readable since
61
+ * Tuesday is an outage nobody was told about, and those two produced identical
62
+ * output until this record existed.
63
+ */
64
+ function sinceLastSuccess(db: ReturnType<typeof getDb>, transportModuleId: string): string {
65
+ const last = readLastRead(db, transportModuleId)?.lastSuccessAt;
66
+ if (!last) return ' (never read successfully)';
67
+ const ms = Date.now() - new Date(last).getTime();
68
+ const mins = Math.floor(ms / 60_000);
69
+ if (mins < 60) return ` (last read OK ${mins}m ago)`;
70
+ const hours = Math.floor(mins / 60);
71
+ return hours < 48
72
+ ? ` (last read OK ${hours}h ago)`
73
+ : ` (last read OK ${Math.floor(hours / 24)}d ago)`;
74
+ }
75
+
55
76
  export async function handleAlertsPoll(
56
77
  flags: Record<string, string | boolean> = {},
57
78
  ): Promise<CommandResult> {
@@ -97,6 +118,7 @@ export async function handleAlertsPoll(
97
118
  receiveFrom: makeReceiver(db),
98
119
  readCursor: (t) => readCursor(db, t),
99
120
  writeCursor: (t, c) => writeCursor(db, t, c),
121
+ recordRead: (t, r) => writeLastRead(db, t, r),
100
122
  now: () => new Date(),
101
123
  transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
102
124
  answerInterview: (eventId, value) => responder.answer(eventId, value),
@@ -122,7 +144,10 @@ export async function handleAlertsPoll(
122
144
  // an operator has no way to tell which — the ambiguity that made the Signal
123
145
  // ack path take a week to diagnose.
124
146
  const detail = [
125
- ...report.failures.map((f) => `${f.transportModuleId} COULD NOT BE READ: ${f.error}`),
147
+ ...report.failures.map(
148
+ (f) =>
149
+ `${f.transportModuleId} COULD NOT BE READ: ${f.error}${sinceLastSuccess(db, f.transportModuleId)}`,
150
+ ),
126
151
  ...report.unheard.map((u) => `not heard from ${u.senderAddress}: ${u.reason}`),
127
152
  ...askErrors,
128
153
  ];
@@ -0,0 +1,281 @@
1
+ /**
2
+ * Tests for `storage set-path`.
3
+ *
4
+ * The motivating case (#566) is celilo-mgr: `local-backups` points at
5
+ * `/Users/pbanka/hobby/backups/celilo-backups/`, a macOS path that came
6
+ * across when the database was restored onto a Linux host. The directory
7
+ * does not exist there. Relocating must therefore succeed with nothing
8
+ * migrated — not error, and not claim files were moved — and must not
9
+ * carry the four-month-old `✓ Verified` stamp onto the new path.
10
+ *
11
+ * Isolation: CELILO_DB_PATH / CELILO_DATA_DIR are set before the SUT is
12
+ * imported, so nothing touches the production database.
13
+ */
14
+
15
+ import { afterAll, describe, expect, test } from 'bun:test';
16
+ import {
17
+ chmodSync,
18
+ existsSync,
19
+ mkdirSync,
20
+ mkdtempSync,
21
+ rmSync,
22
+ statSync,
23
+ utimesSync,
24
+ writeFileSync,
25
+ } from 'node:fs';
26
+ import { homedir, tmpdir } from 'node:os';
27
+ import { join } from 'node:path';
28
+
29
+ const testRoot = mkdtempSync(join(tmpdir(), 'celilo-setpath-'));
30
+ process.env.CELILO_DB_PATH = join(testRoot, 'celilo.db');
31
+ process.env.CELILO_DATA_DIR = join(testRoot, 'data');
32
+
33
+ const {
34
+ addBackupStorage,
35
+ getBackupStorageByStorageId,
36
+ getStorageCredentials,
37
+ updateStorageCredentials,
38
+ } = await import('../../services/backup-storage');
39
+ const { handleStorageSetPath, inspectSourceDir, planRelocation, resolveTargetPath } = await import(
40
+ './storage-set-path'
41
+ );
42
+
43
+ afterAll(() => {
44
+ rmSync(testRoot, { recursive: true, force: true });
45
+ });
46
+
47
+ describe('planRelocation (pure)', () => {
48
+ const base = { sourceDir: '/old/celilo-backups', targetDir: '/new/celilo-backups' };
49
+
50
+ test('migrates when the source has files', () => {
51
+ const plan = planRelocation({ ...base, sourceState: 'populated', migrateRequested: true });
52
+ expect(plan.migrate).toBe(true);
53
+ expect(plan.note).toContain('Migrating archives');
54
+ });
55
+
56
+ test('PRODUCTION CASE: source missing — skips migration, says so, does not error', () => {
57
+ const plan = planRelocation({ ...base, sourceState: 'missing', migrateRequested: true });
58
+ expect(plan.migrate).toBe(false);
59
+ expect(plan.note).toContain('does not exist');
60
+ expect(plan.note).toContain('nothing to migrate');
61
+ });
62
+
63
+ test('source unreadable — skips migration and says nothing was moved', () => {
64
+ const plan = planRelocation({ ...base, sourceState: 'unreadable', migrateRequested: true });
65
+ expect(plan.migrate).toBe(false);
66
+ expect(plan.note).toContain('not readable');
67
+ });
68
+
69
+ test('source empty — skips migration', () => {
70
+ const plan = planRelocation({ ...base, sourceState: 'empty', migrateRequested: true });
71
+ expect(plan.migrate).toBe(false);
72
+ expect(plan.note).toContain('empty');
73
+ });
74
+
75
+ test('--no-migrate wins even over a populated source', () => {
76
+ const plan = planRelocation({ ...base, sourceState: 'populated', migrateRequested: false });
77
+ expect(plan.migrate).toBe(false);
78
+ expect(plan.note).toContain('--no-migrate');
79
+ });
80
+ });
81
+
82
+ describe('resolveTargetPath (pure)', () => {
83
+ test('rejects a path identical to the current one', () => {
84
+ const result = resolveTargetPath('/var/backups', '/var/backups/');
85
+ expect(result).toMatchObject({ success: false });
86
+ });
87
+
88
+ test('preserves a path containing a space', () => {
89
+ const result = resolveTargetPath('/tmp/back ups/celilo', '/somewhere/else');
90
+ expect(result).toEqual({ path: '/tmp/back ups/celilo' });
91
+ });
92
+
93
+ test('expands a leading tilde', () => {
94
+ const result = resolveTargetPath('~/backups', '/somewhere/else');
95
+ expect(result).toMatchObject({ path: join(homedir(), 'backups') });
96
+ });
97
+ });
98
+
99
+ describe('inspectSourceDir', () => {
100
+ test('missing directory reports missing, not unreadable', () => {
101
+ expect(inspectSourceDir(join(testRoot, 'no-such-dir'))).toBe('missing');
102
+ });
103
+
104
+ test('empty directory reports empty', () => {
105
+ const dir = join(testRoot, 'empty-dir');
106
+ mkdirSync(dir, { recursive: true });
107
+ expect(inspectSourceDir(dir)).toBe('empty');
108
+ });
109
+
110
+ test('directory with files reports populated', () => {
111
+ const dir = join(testRoot, 'full-dir');
112
+ mkdirSync(dir, { recursive: true });
113
+ writeFileSync(join(dir, 'a.backup'), 'x');
114
+ expect(inspectSourceDir(dir)).toBe('populated');
115
+ });
116
+
117
+ test('unreadable directory reports unreadable, not missing', () => {
118
+ const parent = join(testRoot, 'locked');
119
+ const dir = join(parent, 'celilo-backups');
120
+ mkdirSync(dir, { recursive: true });
121
+ chmodSync(parent, 0o000);
122
+ try {
123
+ expect(inspectSourceDir(dir)).toBe('unreadable');
124
+ } finally {
125
+ chmodSync(parent, 0o700);
126
+ }
127
+ });
128
+ });
129
+
130
+ describe('updateStorageCredentials', () => {
131
+ test('clears the verification stamp, so a stale ✓ never survives a credential change (#566)', async () => {
132
+ const storage = await addBackupStorage({
133
+ name: 'Stamp Backups',
134
+ providerName: 'local',
135
+ credentials: { path: '/Users/nobody/old' },
136
+ });
137
+
138
+ const { getDb } = await import('../../db/client');
139
+ const { backupStorages } = await import('../../db/schema');
140
+ const { eq } = await import('drizzle-orm');
141
+ getDb()
142
+ .update(backupStorages)
143
+ .set({ verified: true, verifiedAt: new Date('2026-04-08T02:24:43.000Z') })
144
+ .where(eq(backupStorages.id, storage.id))
145
+ .run();
146
+
147
+ await updateStorageCredentials(storage.id, { path: '/var/lib/celilo/backups' });
148
+
149
+ // Asserted WITHOUT a follow-up verify: even if celilo dies between
150
+ // the path change and re-verification, the row must not still claim
151
+ // the destination was verified.
152
+ const after = getBackupStorageByStorageId(storage.storageId);
153
+ expect(after?.verified).toBe(false);
154
+ expect(after?.verifiedAt).toBeNull();
155
+ });
156
+ });
157
+
158
+ describe('handleStorageSetPath (end to end, isolated DB)', () => {
159
+ test('PRODUCTION CASE: current path does not exist — path changes, nothing migrated, re-verified against the new path', async () => {
160
+ const storage = await addBackupStorage({
161
+ name: 'Ghost Backups',
162
+ providerName: 'local',
163
+ credentials: { path: '/Users/nobody/hobby/backups' },
164
+ });
165
+
166
+ // Simulate the stale stamp: the row says Verified from a host that
167
+ // no longer exists.
168
+ const { getDb } = await import('../../db/client');
169
+ const { backupStorages } = await import('../../db/schema');
170
+ const { eq } = await import('drizzle-orm');
171
+ getDb()
172
+ .update(backupStorages)
173
+ .set({ verified: true, verifiedAt: new Date('2026-04-08T02:24:43.000Z') })
174
+ .where(eq(backupStorages.id, storage.id))
175
+ .run();
176
+
177
+ const newPath = join(testRoot, 'relocated');
178
+ const result = await handleStorageSetPath([storage.storageId, newPath]);
179
+
180
+ expect(result.success).toBe(true);
181
+
182
+ const creds = await getStorageCredentials(storage.id);
183
+ expect(creds).toMatchObject({ path: newPath });
184
+
185
+ // The stamp must describe the NEW path, not the dead one.
186
+ const after = getBackupStorageByStorageId(storage.storageId);
187
+ expect(after?.verified).toBe(true);
188
+ expect(after?.verifiedAt?.getTime()).toBeGreaterThan(
189
+ new Date('2026-04-08T02:24:43.000Z').getTime(),
190
+ );
191
+ });
192
+
193
+ test('moves existing archives to the new location', async () => {
194
+ const oldPath = join(testRoot, 'movable-old');
195
+ const newPath = join(testRoot, 'movable-new');
196
+ mkdirSync(join(oldPath, 'celilo-backups', '2026-08-01'), { recursive: true });
197
+ writeFileSync(join(oldPath, 'celilo-backups', '2026-08-01', 'x.backup'), 'payload');
198
+
199
+ const storage = await addBackupStorage({
200
+ name: 'Movable Backups',
201
+ providerName: 'local',
202
+ credentials: { path: oldPath },
203
+ });
204
+
205
+ const result = await handleStorageSetPath([storage.storageId, newPath]);
206
+
207
+ expect(result.success).toBe(true);
208
+ expect(existsSync(join(newPath, 'celilo-backups', '2026-08-01', 'x.backup'))).toBe(true);
209
+ expect(existsSync(join(oldPath, 'celilo-backups'))).toBe(false);
210
+ });
211
+
212
+ test('preserves archive mtimes — a move does not restamp backups', async () => {
213
+ const oldPath = join(testRoot, 'mtime-old');
214
+ const newPath = join(testRoot, 'mtime-new');
215
+ const archive = join(oldPath, 'celilo-backups', 'a.backup');
216
+ mkdirSync(join(oldPath, 'celilo-backups'), { recursive: true });
217
+ writeFileSync(archive, 'payload');
218
+ const stamp = new Date('2026-04-26T01:37:53.740Z');
219
+ utimesSync(archive, stamp, stamp);
220
+
221
+ const storage = await addBackupStorage({
222
+ name: 'Mtime Backups',
223
+ providerName: 'local',
224
+ credentials: { path: oldPath },
225
+ });
226
+
227
+ await handleStorageSetPath([storage.storageId, newPath]);
228
+
229
+ const moved = statSync(join(newPath, 'celilo-backups', 'a.backup'));
230
+ expect(Math.round(moved.mtimeMs)).toBe(stamp.getTime());
231
+ });
232
+
233
+ test('--no-migrate leaves the old archives where they are', async () => {
234
+ const oldPath = join(testRoot, 'kept-old');
235
+ const newPath = join(testRoot, 'kept-new');
236
+ mkdirSync(join(oldPath, 'celilo-backups'), { recursive: true });
237
+ writeFileSync(join(oldPath, 'celilo-backups', 'y.backup'), 'payload');
238
+
239
+ const storage = await addBackupStorage({
240
+ name: 'Kept Backups',
241
+ providerName: 'local',
242
+ credentials: { path: oldPath },
243
+ });
244
+
245
+ const result = await handleStorageSetPath([storage.storageId, newPath], {
246
+ 'no-migrate': true,
247
+ });
248
+
249
+ expect(result.success).toBe(true);
250
+ expect(existsSync(join(oldPath, 'celilo-backups', 'y.backup'))).toBe(true);
251
+ expect(existsSync(join(newPath, 'celilo-backups', 'y.backup'))).toBe(false);
252
+ });
253
+
254
+ test('handles a new path containing a space', async () => {
255
+ const oldPath = join(testRoot, 'spacey-old');
256
+ const newPath = join(testRoot, 'back ups', "Bob's celilo");
257
+ mkdirSync(join(oldPath, 'celilo-backups'), { recursive: true });
258
+ writeFileSync(join(oldPath, 'celilo-backups', 'z.backup'), 'payload');
259
+
260
+ const storage = await addBackupStorage({
261
+ name: 'Spacey Backups',
262
+ providerName: 'local',
263
+ credentials: { path: oldPath },
264
+ });
265
+
266
+ const result = await handleStorageSetPath([storage.storageId, newPath]);
267
+
268
+ expect(result.success).toBe(true);
269
+ expect(existsSync(join(newPath, 'celilo-backups', 'z.backup'))).toBe(true);
270
+ });
271
+
272
+ test('rejects an unknown storage id', async () => {
273
+ const result = await handleStorageSetPath(['no-such-storage', join(testRoot, 'x')]);
274
+ expect(result).toMatchObject({ success: false });
275
+ });
276
+
277
+ test('requires both a storage id and a path', async () => {
278
+ const result = await handleStorageSetPath(['only-one-arg']);
279
+ expect(result).toMatchObject({ success: false });
280
+ });
281
+ });
@@ -0,0 +1,190 @@
1
+ /**
2
+ * Storage Set Path Command
3
+ * Relocate a local storage destination to a new directory, migrating
4
+ * any archives that are actually there.
5
+ *
6
+ * Motivating case (#566): celilo-mgr's `local-backups` points at
7
+ * `/Users/pbanka/...`, a macOS path carried across when the database was
8
+ * restored onto a Linux host. The directory does not exist there, so the
9
+ * relocation must complete cleanly with nothing to migrate rather than
10
+ * erroring — and must not carry the old `✓ Verified` stamp onto the new
11
+ * path.
12
+ */
13
+
14
+ import { cpSync, readdirSync, rmSync } from 'node:fs';
15
+ import { homedir } from 'node:os';
16
+ import { join, resolve } from 'node:path';
17
+ import {
18
+ getBackupStorageByStorageId,
19
+ getStorageCredentials,
20
+ updateStorageCredentials,
21
+ verifyBackupStorage,
22
+ } from '../../services/backup-storage';
23
+ import { BACKUP_PREFIX } from '../../services/storage-providers/local';
24
+ import { celiloIntro, celiloOutro } from '../prompts';
25
+ import type { CommandResult } from '../types';
26
+ import { probePathWriteable } from './storage-add-local';
27
+
28
+ /** What the current archive directory turned out to be, on disk. */
29
+ export type SourceState = 'missing' | 'unreadable' | 'empty' | 'populated';
30
+
31
+ export interface RelocationPlan {
32
+ /** Whether to copy the source tree to the target. */
33
+ migrate: boolean;
34
+ /** One line for the operator explaining what will (not) happen. */
35
+ note: string;
36
+ }
37
+
38
+ /**
39
+ * Decide whether to migrate. Pure — takes an already-observed source
40
+ * state so it is testable without a filesystem (Rule 10.4).
41
+ */
42
+ export function planRelocation(input: {
43
+ sourceState: SourceState;
44
+ sourceDir: string;
45
+ targetDir: string;
46
+ migrateRequested: boolean;
47
+ }): RelocationPlan {
48
+ const { sourceState, sourceDir, targetDir, migrateRequested } = input;
49
+
50
+ if (!migrateRequested) {
51
+ return { migrate: false, note: `Migration skipped (--no-migrate). ${sourceDir} left as-is.` };
52
+ }
53
+
54
+ switch (sourceState) {
55
+ case 'missing':
56
+ return { migrate: false, note: `${sourceDir} does not exist — nothing to migrate.` };
57
+ case 'unreadable':
58
+ return { migrate: false, note: `${sourceDir} is not readable — nothing migrated.` };
59
+ case 'empty':
60
+ return { migrate: false, note: `${sourceDir} is empty — nothing to migrate.` };
61
+ case 'populated':
62
+ return { migrate: true, note: `Migrating archives from ${sourceDir} to ${targetDir}.` };
63
+ }
64
+ }
65
+
66
+ /**
67
+ * Validate and normalise the requested path. Pure apart from `~`
68
+ * expansion, which reads the environment but touches no filesystem.
69
+ */
70
+ export function resolveTargetPath(
71
+ raw: string,
72
+ currentPath: string,
73
+ ): CommandResult | { path: string } {
74
+ const expanded = raw.startsWith('~/') || raw === '~' ? raw.replace('~', homedir()) : raw;
75
+ const resolved = resolve(expanded);
76
+
77
+ if (resolved === resolve(currentPath)) {
78
+ return { success: false, error: `Storage path is already '${resolved}' — nothing to do.` };
79
+ }
80
+
81
+ return { path: resolved };
82
+ }
83
+
84
+ /** Observe the archive directory. Distinguishes absent from unreadable. */
85
+ export function inspectSourceDir(dir: string): SourceState {
86
+ try {
87
+ return readdirSync(dir).length === 0 ? 'empty' : 'populated';
88
+ } catch (error) {
89
+ return (error as NodeJS.ErrnoException).code === 'ENOENT' ? 'missing' : 'unreadable';
90
+ }
91
+ }
92
+
93
+ export async function handleStorageSetPath(
94
+ args: string[],
95
+ flags: Record<string, boolean | string> = {},
96
+ ): Promise<CommandResult> {
97
+ try {
98
+ celiloIntro('Relocate Backup Storage');
99
+
100
+ const storageId = args[0];
101
+ const newPathArg = args[1];
102
+ if (!storageId || !newPathArg) {
103
+ return {
104
+ success: false,
105
+ error:
106
+ 'Storage ID and new path are required\n\nUsage: celilo storage set-path <storage-id> <new-path> [--no-migrate]',
107
+ };
108
+ }
109
+
110
+ const storage = getBackupStorageByStorageId(storageId);
111
+ if (!storage) {
112
+ return { success: false, error: `Storage not found: ${storageId}` };
113
+ }
114
+ if (storage.providerName !== 'local') {
115
+ return {
116
+ success: false,
117
+ error: `'${storage.storageId}' is a ${storage.providerName} destination — set-path only applies to local storage.`,
118
+ };
119
+ }
120
+
121
+ const credentials = await getStorageCredentials(storage.id);
122
+ if (!('path' in credentials)) {
123
+ return { success: false, error: `Storage '${storage.storageId}' has no path configured.` };
124
+ }
125
+ const currentPath = credentials.path;
126
+
127
+ const resolved = resolveTargetPath(newPathArg, currentPath);
128
+ if ('success' in resolved) return resolved;
129
+ const newPath = resolved.path;
130
+
131
+ const writeError = probePathWriteable(newPath);
132
+ if (writeError !== null) {
133
+ return { success: false, error: `Path '${newPath}' is not writeable: ${writeError}` };
134
+ }
135
+
136
+ const sourceDir = join(currentPath, BACKUP_PREFIX);
137
+ const targetDir = join(newPath, BACKUP_PREFIX);
138
+ const plan = planRelocation({
139
+ sourceState: inspectSourceDir(sourceDir),
140
+ sourceDir,
141
+ targetDir,
142
+ migrateRequested: flags['no-migrate'] !== true,
143
+ });
144
+
145
+ console.log(`\n${plan.note}`);
146
+
147
+ if (plan.migrate) {
148
+ // A backup archive's mtime is part of what it is; a move must not
149
+ // restamp it. Bun's cpSync already preserves timestamps, so the
150
+ // flag is a no-op today — it is here for Node semantics, where the
151
+ // default is the other way. The mtime test guards the behavior,
152
+ // not this flag.
153
+ cpSync(sourceDir, targetDir, { recursive: true, force: true, preserveTimestamps: true });
154
+ }
155
+
156
+ await updateStorageCredentials(storage.id, { ...credentials, path: newPath });
157
+ console.log(`✓ Path updated: ${currentPath} → ${newPath}`);
158
+
159
+ if (plan.migrate) {
160
+ // Only after the DB points at the copy, so a failure here leaves
161
+ // archives duplicated rather than orphaned.
162
+ try {
163
+ rmSync(sourceDir, { recursive: true, force: true });
164
+ } catch (error) {
165
+ console.log(
166
+ `⚠ Copied, but could not remove ${sourceDir}: ${error instanceof Error ? error.message : String(error)}`,
167
+ );
168
+ }
169
+ }
170
+
171
+ const { result } = await verifyBackupStorage(storage.id);
172
+ if (!result.success) {
173
+ console.log(`✗ ${result.message}`);
174
+ celiloOutro(
175
+ `Path changed but verification failed.\n\nFix the path and re-verify: celilo storage verify ${storage.storageId}`,
176
+ );
177
+ return { success: false, error: result.message };
178
+ }
179
+
180
+ console.log(`✓ ${result.message}`);
181
+ celiloOutro(`'${storage.storageId}' now stores backups at ${targetDir}/`);
182
+
183
+ return { success: true, message: `Relocated ${storage.storageId} to ${newPath}` };
184
+ } catch (error) {
185
+ return {
186
+ success: false,
187
+ error: `Failed to set storage path: ${error instanceof Error ? error.message : String(error)}`,
188
+ };
189
+ }
190
+ }
@@ -35,6 +35,7 @@ import type { ModuleManifest } from '../../manifest/schema';
35
35
  import { RegistryClient } from '../../registry/client';
36
36
  import { decryptSecret } from '../../secrets/encryption';
37
37
  import { getOrCreateMasterKey } from '../../secrets/master-key';
38
+ import { readAllTransportStatuses } from '../../services/alerting/read-records';
38
39
  import { runAudit } from '../../services/audit';
39
40
  import type { DriftFinding, SystemAuditReport } from '../../services/audit';
40
41
  import { loadBackupAuditInfo } from '../../services/audit/backup-source';
@@ -378,6 +379,17 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
378
379
  secretsDecryptable: { results: secretResults },
379
380
  servicesReachable: { results: serviceReachableResults },
380
381
  machinesReachable: { results: machineReachableResults },
382
+ // Reads the record the poller already writes — this check never performs a
383
+ // read of its own. One that did would drain the queue and eat the
384
+ // acknowledgement it exists to protect (#541).
385
+ transportReads: {
386
+ statuses: readAllTransportStatuses(db),
387
+ now: new Date(),
388
+ // Six missed polls. The poller runs every five minutes, so this absorbs a
389
+ // slow sweep, a restart, and a missed tick without crying wolf — while
390
+ // still catching a transport that has genuinely stopped being readable.
391
+ staleAfterMs: 30 * 60_000,
392
+ },
381
393
  trustedSources: { firewalls: collectFirewallReach(db) },
382
394
  };
383
395
  }
@@ -585,6 +585,7 @@ export async function handleSystemUpdate(
585
585
  secretsDecryptable: { results: [] },
586
586
  servicesReachable: { results: [] },
587
587
  machinesReachable: { results: [] },
588
+ transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
588
589
  trustedSources: { firewalls: [] },
589
590
  };
590
591
 
@@ -486,7 +486,7 @@ export async function getCompletions(words: string[], current: number): Promise<
486
486
 
487
487
  // Storage subcommands
488
488
  if (command === 'storage' && currentIndex === 1) {
489
- const subcommands = ['add', 'list', 'remove', 'verify', 'set-default'];
489
+ const subcommands = ['add', 'list', 'remove', 'verify', 'set-default', 'set-path'];
490
490
  return filterSuggestions(subcommands, args[1] || '');
491
491
  }
492
492
 
@@ -496,10 +496,13 @@ export async function getCompletions(words: string[], current: number): Promise<
496
496
  return filterSuggestions(providers, args[2] || '');
497
497
  }
498
498
 
499
- // Storage remove/verify/set-default - complete with storage IDs
499
+ // Storage remove/verify/set-default/set-path - complete with storage IDs
500
500
  if (
501
501
  command === 'storage' &&
502
- (args[1] === 'remove' || args[1] === 'verify' || args[1] === 'set-default') &&
502
+ (args[1] === 'remove' ||
503
+ args[1] === 'verify' ||
504
+ args[1] === 'set-default' ||
505
+ args[1] === 'set-path') &&
503
506
  currentIndex === 2
504
507
  ) {
505
508
  const storages = listBackupStorages();
package/src/cli/index.ts CHANGED
@@ -839,6 +839,9 @@ Subcommands:
839
839
  list List all configured storage destinations
840
840
  verify <storage-id> Test storage connectivity and permissions
841
841
  set-default <id> Set the default backup storage destination
842
+ set-path <id> <path> Relocate a local destination to a new directory
843
+ Options:
844
+ --no-migrate Change the path without moving existing archives
842
845
  remove <storage-id> Remove a storage destination
843
846
  Options:
844
847
  --force Skip confirmation prompts
@@ -860,6 +863,10 @@ Examples:
860
863
  # Set default destination
861
864
  celilo storage set-default local-backups
862
865
 
866
+ # Move a local destination (archives at the old path are moved too;
867
+ # if that path is gone, the change proceeds with nothing to migrate)
868
+ celilo storage set-path local-backups /var/lib/celilo/backups
869
+
863
870
  # Remove storage
864
871
  celilo storage remove local-backups --force
865
872
 
@@ -1836,6 +1843,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
1836
1843
  return handleStorageSetDefault(parsed.args, parsed.flags);
1837
1844
  }
1838
1845
 
1846
+ if (parsed.subcommand === 'set-path') {
1847
+ const { handleStorageSetPath } = await import('./commands/storage-set-path');
1848
+ return handleStorageSetPath(parsed.args, parsed.flags);
1849
+ }
1850
+
1839
1851
  return {
1840
1852
  success: false,
1841
1853
  error: `Unknown storage subcommand: ${parsed.subcommand}\n\nRun "celilo storage --help" for usage`,
@@ -1,7 +1,9 @@
1
1
  import { describe, expect, test } from 'bun:test';
2
- import type { DriftFinding, SystemAuditReport } from '../../services/audit/types';
2
+ import type { DriftCategory, DriftFinding, SystemAuditReport } from '../../services/audit/types';
3
3
  import {
4
+ ALL_CATEGORIES,
4
5
  type AuditTuiState,
6
+ CATEGORY_LABELS,
5
7
  groupFindings,
6
8
  initState,
7
9
  reducer,
@@ -244,3 +246,15 @@ describe('selectedFinding', () => {
244
246
  expect(selectedFinding(initState(report([])))).toBeNull();
245
247
  });
246
248
  });
249
+
250
+ // ALL_CATEGORIES is a plain array, so the type system cannot require every
251
+ // DriftCategory to appear in it — a new category compiles fine and then never
252
+ // shows up in the TUI. CATEGORY_LABELS is a Record and IS exhaustive, so it is
253
+ // the honest source of truth to compare against.
254
+ describe('ALL_CATEGORIES covers every category', () => {
255
+ test('every labelled category is listed, and vice versa', () => {
256
+ expect([...ALL_CATEGORIES].sort()).toEqual(
257
+ (Object.keys(CATEGORY_LABELS) as DriftCategory[]).sort(),
258
+ );
259
+ });
260
+ });
@@ -83,6 +83,7 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
83
83
  'secrets_decryptable',
84
84
  'services_reachable',
85
85
  'machines_reachable',
86
+ 'transport_reads',
86
87
  'trusted_sources',
87
88
  ];
88
89
 
@@ -101,6 +102,7 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
101
102
  secrets_decryptable: 'Secrets',
102
103
  services_reachable: 'Service reachability',
103
104
  machines_reachable: 'Machine reachability',
105
+ transport_reads: 'Transport readability',
104
106
  trusted_sources: 'Trusted networks',
105
107
  };
106
108
 
@@ -24,6 +24,7 @@ import { type FailingKey, builtinAlertKey } from './keys';
24
24
  */
25
25
  const TARGET_KIND_BY_CATEGORY: Partial<Record<DriftCategory, string>> = {
26
26
  machines_reachable: 'machine',
27
+ transport_reads: 'module',
27
28
  services_reachable: 'service',
28
29
  services_credentials: 'service',
29
30
  backups: 'module',
@@ -7,7 +7,12 @@ import { eq } from 'drizzle-orm';
7
7
  import type { DbClient } from '../../db/client';
8
8
  import { type Route, alerts, modules, monitors, notificationDeliveries } from '../../db/schema';
9
9
  import { setupTestDatabase } from '../../test-utils/setup-test-db';
10
- import { type InboundPollDeps, pollInbound, transportsWithRoutes } from './inbound-poller';
10
+ import {
11
+ type InboundPollDeps,
12
+ type TransportReadRecord,
13
+ pollInbound,
14
+ transportsWithRoutes,
15
+ } from './inbound-poller';
11
16
  import { moduleCheckAlertKey } from './keys';
12
17
  import { createPerson, createRoute } from './people';
13
18
  import { mintDelivery } from './tokens';
@@ -386,6 +391,63 @@ describe('pollInbound', () => {
386
391
  expect(ackedBy()).toBe(peterRoute.personId);
387
392
  });
388
393
 
394
+ // The gap this closes. The ONLY per-transport state celilo persisted was the
395
+ // cursor, and writeCursor early-returns when there is no cursor — which a
396
+ // failed read never produces. So the store could not REPRESENT a failure, and
397
+ // "unreadable for a week" and "nobody replied for a week" left identical
398
+ // traces (#501).
399
+ describe('recording what each read attempt produced', () => {
400
+ const records: Array<[string, TransportReadRecord]> = [];
401
+ const recording = (over: Partial<InboundPollDeps> = {}) =>
402
+ deps([], { recordRead: (t, r) => records.push([t, r]), ...over });
403
+
404
+ beforeEach(() => {
405
+ records.length = 0;
406
+ });
407
+
408
+ test('a FAILED read is recorded — the case the cursor could never express', async () => {
409
+ await pollInbound(
410
+ db,
411
+ recording({ receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
412
+ );
413
+ expect(records).toHaveLength(1);
414
+ expect(records[0][0]).toBe('signal');
415
+ expect(records[0][1]).toMatchObject({ outcome: 'failed', error: 'connection refused' });
416
+ });
417
+
418
+ test('a successful read is recorded with how many messages it returned', async () => {
419
+ await pollInbound(
420
+ db,
421
+ deps([inbound(PETER, 'hello')], { recordRead: (t, r) => records.push([t, r]) }),
422
+ );
423
+ expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 1 });
424
+ });
425
+
426
+ // Zero messages is not failure. Conflating them is the original bug.
427
+ test('an empty but successful read records received, not failed', async () => {
428
+ await pollInbound(db, recording());
429
+ expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 0 });
430
+ });
431
+
432
+ test('a unidirectional transport is recorded as such', async () => {
433
+ await pollInbound(db, recording({ receiveFrom: async () => ({ status: 'unidirectional' }) }));
434
+ expect(records[0][1].outcome).toBe('unidirectional');
435
+ });
436
+
437
+ // Every attempt, not just interesting ones — a gap in the record would be
438
+ // read as "nothing happened".
439
+ test('every attempt is recorded, and carries when it happened', async () => {
440
+ await pollInbound(db, recording());
441
+ expect(records[0][1].at).toBe(NOW.toISOString());
442
+ });
443
+
444
+ // Optional dep: a caller that does not persist must still poll.
445
+ test('polling works with no recorder attached', async () => {
446
+ const report = await pollInbound(db, deps([]));
447
+ expect(report.transportsPolled).toBe(1);
448
+ });
449
+ });
450
+
389
451
  test('a route pointing at a transport nobody uses is not polled', () => {
390
452
  db.delete(alerts).run();
391
453
  expect(transportsWithRoutes(db)).toEqual(['signal']);
@@ -44,6 +44,35 @@ export interface TransportFailure {
44
44
  error: string;
45
45
  }
46
46
 
47
+ /**
48
+ * The outcome of one attempt to read a transport, as recorded for later.
49
+ *
50
+ * Written on EVERY attempt, including failures — which is the whole point. The
51
+ * only per-transport state celilo persisted before this was the cursor, and a
52
+ * failed read produces no cursor, so `writeCursor` returned early and nothing
53
+ * was written. The store could not REPRESENT a failure, so an absence of
54
+ * recorded failures was never evidence there had been none: a transport that
55
+ * had not been readable for a week looked identical to one nobody had replied
56
+ * on (#501).
57
+ */
58
+ export interface TransportReadRecord {
59
+ /** When the attempt happened, ISO-8601. */
60
+ at: string;
61
+ outcome: 'received' | 'unidirectional' | 'failed';
62
+ /** Present only when `failed`. */
63
+ error?: string;
64
+ /** How many messages the read returned. Zero is not the same as failure. */
65
+ messages: number;
66
+ /**
67
+ * When a read last SUCCEEDED, carried forward across failures.
68
+ *
69
+ * This is the field that answers the question the old state could not: a
70
+ * transport reporting "0 messages" for a week and one that has not been
71
+ * readable for a week are the same picture until you can see this.
72
+ */
73
+ lastSuccessAt?: string;
74
+ }
75
+
47
76
  /** A message that was read but not acted on, and why. */
48
77
  export interface UnheardMessage {
49
78
  senderAddress: string;
@@ -65,6 +94,13 @@ export interface InboundPollDeps {
65
94
  /** Persisted receive cursor per transport. */
66
95
  readCursor(transportModuleId: string): string | null;
67
96
  writeCursor(transportModuleId: string, cursor: string | null): void;
97
+ /**
98
+ * Record what one read attempt produced. Called for EVERY attempt — a failed
99
+ * read must leave a trace, or "we could not read this transport" stays
100
+ * indistinguishable from "nobody replied". Optional so a caller that does not
101
+ * care (tests, one-off invocations) need not supply it.
102
+ */
103
+ recordRead?(transportModuleId: string, record: TransportReadRecord): void;
68
104
  now(): Date;
69
105
  /**
70
106
  * Transport for a route, so an ack can be broadcast to everyone else paged.
@@ -131,6 +167,12 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
131
167
  for (const transportId of transportsWithRoutes(db)) {
132
168
  const received = await deps.receiveFrom(transportId, deps.readCursor(transportId));
133
169
  report.transportsPolled++;
170
+ deps.recordRead?.(transportId, {
171
+ at: deps.now().toISOString(),
172
+ outcome: received.status,
173
+ ...(received.status === 'failed' ? { error: received.error } : {}),
174
+ messages: received.status === 'received' ? received.messages.length : 0,
175
+ });
134
176
  // One dead transport must not stop the others being read — but it is
135
177
  // RECORDED rather than skipped in silence, because "cannot read" and
136
178
  // "nothing to read" are the two things an operator most needs to tell
@@ -0,0 +1,85 @@
1
+ /**
2
+ * The per-transport record of whether celilo can still READ replies.
3
+ *
4
+ * Extracted from the poll command because it now has two readers: the poller
5
+ * writes it, and the audit asserts on it. Keeping the key format private to the
6
+ * writer would have meant the audit re-deriving a string literal — the sort of
7
+ * duplication that silently stops matching.
8
+ *
9
+ * The shape it stores is the point. celilo's only per-transport state used to
10
+ * be the receive cursor, and `writeCursor` returns early when there is no
11
+ * cursor — which a failed read never produces. So the store could not
12
+ * REPRESENT a failure, and "unreadable since Tuesday" left exactly the same
13
+ * trace as "nobody replied since Tuesday" (#501).
14
+ */
15
+
16
+ import { eq } from 'drizzle-orm';
17
+ import type { DbClient } from '../../db/client';
18
+ import { systemConfig } from '../../db/schema';
19
+ import { type TransportReadRecord, transportsWithRoutes } from './inbound-poller';
20
+
21
+ const READ_PREFIX = 'alerting.last_read.';
22
+
23
+ export function readLastRead(db: DbClient, transportModuleId: string): TransportReadRecord | null {
24
+ const row = db
25
+ .select()
26
+ .from(systemConfig)
27
+ .where(eq(systemConfig.key, `${READ_PREFIX}${transportModuleId}`))
28
+ .get();
29
+ if (!row?.value) return null;
30
+ try {
31
+ return JSON.parse(row.value) as TransportReadRecord;
32
+ } catch {
33
+ // A malformed row must not read as "never succeeded" — that would invent a
34
+ // fact and page someone about it.
35
+ return null;
36
+ }
37
+ }
38
+
39
+ /**
40
+ * Record one attempt, carrying the last SUCCESS forward.
41
+ *
42
+ * Deliberately has no counterpart to `writeCursor`'s early return: a failure
43
+ * that writes nothing is why this record did not exist before. Carrying the
44
+ * success forward is what turns a log into an answer — a run of failures must
45
+ * not erase when the transport last actually worked.
46
+ */
47
+ export function writeLastRead(
48
+ db: DbClient,
49
+ transportModuleId: string,
50
+ record: TransportReadRecord,
51
+ ): void {
52
+ const key = `${READ_PREFIX}${transportModuleId}`;
53
+ const previous = readLastRead(db, transportModuleId);
54
+ const lastSuccessAt =
55
+ record.outcome === 'received' ? record.at : (previous?.lastSuccessAt ?? undefined);
56
+ const value = JSON.stringify({ ...record, ...(lastSuccessAt ? { lastSuccessAt } : {}) });
57
+
58
+ const existing = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
59
+ if (existing) {
60
+ db.update(systemConfig).set({ value }).where(eq(systemConfig.key, key)).run();
61
+ } else {
62
+ db.insert(systemConfig)
63
+ .values({ key, value, description: `Last inbound read attempt for ${transportModuleId}` })
64
+ .run();
65
+ }
66
+ }
67
+
68
+ export interface TransportReadStatus {
69
+ transportModuleId: string;
70
+ last: TransportReadRecord | null;
71
+ }
72
+
73
+ /**
74
+ * Read state for every transport that has a route pointing at it.
75
+ *
76
+ * Scoped to transports with routes on purpose: a transport nobody is routed to
77
+ * cannot fail to deliver anyone's acknowledgement, and paging about it would be
78
+ * noise that trains an operator to ignore this check.
79
+ */
80
+ export function readAllTransportStatuses(db: DbClient): TransportReadStatus[] {
81
+ return transportsWithRoutes(db).map((transportModuleId) => ({
82
+ transportModuleId,
83
+ last: readLastRead(db, transportModuleId),
84
+ }));
85
+ }
@@ -32,6 +32,7 @@ const emptyDeps = {
32
32
  secretsDecryptable: { results: [] },
33
33
  servicesReachable: { results: [] },
34
34
  machinesReachable: { results: [] },
35
+ transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
35
36
  trustedSources: { firewalls: [] },
36
37
  };
37
38
 
@@ -25,6 +25,7 @@ import {
25
25
  } from './services-credentials';
26
26
  import { type ServicesReachableAuditDeps, auditServicesReachable } from './services-reachable';
27
27
  import { type TerraformPlanAuditDeps, auditTerraformPlan } from './terraform-plan';
28
+ import { type TransportReadsAuditDeps, auditTransportReads } from './transport-reads';
28
29
  import { type TrustedSourcesAuditDeps, auditTrustedSources } from './trusted-sources';
29
30
  import {
30
31
  type DriftCategory,
@@ -53,6 +54,7 @@ export interface AuditDeps {
53
54
  secretsDecryptable: SecretsDecryptableAuditDeps;
54
55
  servicesReachable: ServicesReachableAuditDeps;
55
56
  machinesReachable: MachinesReachableAuditDeps;
57
+ transportReads: TransportReadsAuditDeps;
56
58
  trustedSources: TrustedSourcesAuditDeps;
57
59
  /** Defaults to `Date.now()`-based ISO string. */
58
60
  now?: () => Date;
@@ -103,6 +105,7 @@ export async function runAudit(
103
105
  wrap('secrets_decryptable', auditSecretsDecryptable(deps.secretsDecryptable)),
104
106
  wrap('services_reachable', auditServicesReachable(deps.servicesReachable)),
105
107
  wrap('machines_reachable', auditMachinesReachable(deps.machinesReachable)),
108
+ wrap('transport_reads', auditTransportReads(deps.transportReads)),
106
109
  wrap('trusted_sources', auditTrustedSources(deps.trustedSources)),
107
110
  ]);
108
111
 
@@ -0,0 +1,113 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import type { TransportReadStatus } from '../alerting/read-records';
3
+ import { auditTransportReads } from './transport-reads';
4
+
5
+ const NOW = new Date('2026-08-01T12:00:00Z');
6
+ const STALE_AFTER = 30 * 60_000;
7
+
8
+ const ago = (ms: number) => new Date(NOW.getTime() - ms).toISOString();
9
+
10
+ function status(over: Partial<TransportReadStatus['last']> | null): TransportReadStatus {
11
+ return {
12
+ transportModuleId: 'signal',
13
+ last:
14
+ over === null
15
+ ? null
16
+ : { at: ago(0), outcome: 'received', messages: 0, lastSuccessAt: ago(0), ...over },
17
+ };
18
+ }
19
+
20
+ const run = (statuses: TransportReadStatus[]) =>
21
+ auditTransportReads({ statuses, now: NOW, staleAfterMs: STALE_AFTER });
22
+
23
+ describe('auditTransportReads', () => {
24
+ test('a transport read successfully just now is not a finding', async () => {
25
+ expect(await run([status({})])).toEqual([]);
26
+ });
27
+
28
+ // THE case this check exists for. Six tokens were issued and zero consumed
29
+ // over a week, and nothing anywhere was red (#501).
30
+ test('a transport not read successfully for hours is drift', async () => {
31
+ const findings = await run([
32
+ status({
33
+ at: ago(60_000),
34
+ outcome: 'failed',
35
+ error: 'refused',
36
+ lastSuccessAt: ago(3 * 3600_000),
37
+ }),
38
+ ]);
39
+ expect(findings).toHaveLength(1);
40
+ expect(findings[0]).toMatchObject({
41
+ category: 'transport_reads',
42
+ code: 'transport_reads_stale',
43
+ severity: 'drift',
44
+ subject: 'signal',
45
+ });
46
+ expect(findings[0].message).toContain('3h ago');
47
+ });
48
+
49
+ // The decision that makes staleness safe to page on: an EMPTY read is a
50
+ // success, so a quiet transport keeps refreshing lastSuccessAt. If empty
51
+ // reads counted as failure this check would page on every quiet afternoon
52
+ // and be switched off within a week.
53
+ test('a quiet transport — successful reads, zero messages — is NOT stale', async () => {
54
+ const findings = await run([
55
+ status({ at: ago(0), outcome: 'received', messages: 0, lastSuccessAt: ago(0) }),
56
+ ]);
57
+ expect(findings).toEqual([]);
58
+ });
59
+
60
+ test('a transport that has never succeeded is drift, even if attempts are recent', async () => {
61
+ const findings = await run([
62
+ status({
63
+ at: ago(0),
64
+ outcome: 'failed',
65
+ error: 'connection refused',
66
+ lastSuccessAt: undefined,
67
+ }),
68
+ ]);
69
+ expect(findings[0]).toMatchObject({ code: 'transport_never_read', severity: 'drift' });
70
+ expect(findings[0].details).toContain('connection refused');
71
+ });
72
+
73
+ test('a transport with no record at all is drift, not silently fine', async () => {
74
+ const findings = await run([status(null)]);
75
+ expect(findings[0]).toMatchObject({ code: 'transport_never_polled', severity: 'drift' });
76
+ });
77
+
78
+ // A transport with no `receive` is working as designed. Paging about it
79
+ // would be paging about a healthy system, which teaches operators to ignore
80
+ // the check.
81
+ test('a unidirectional transport is never a finding', async () => {
82
+ const findings = await run([
83
+ status({ at: ago(10 * 3600_000), outcome: 'unidirectional', lastSuccessAt: undefined }),
84
+ ]);
85
+ expect(findings).toEqual([]);
86
+ });
87
+
88
+ test('just inside the threshold is not yet drift', async () => {
89
+ expect(await run([status({ lastSuccessAt: ago(STALE_AFTER - 1_000) })])).toEqual([]);
90
+ });
91
+
92
+ test('just outside the threshold is drift', async () => {
93
+ const findings = await run([status({ lastSuccessAt: ago(STALE_AFTER + 1_000) })]);
94
+ expect(findings).toHaveLength(1);
95
+ });
96
+
97
+ // The remediation must point at something that does not consume the queue.
98
+ // `celilo alerts poll` would READ, and a suggestion that eats the operator's
99
+ // acknowledgement is worse than no suggestion (#541).
100
+ test('a stale finding sends you to the journal, not to a read', async () => {
101
+ const findings = await run([status({ lastSuccessAt: ago(3 * 3600_000) })]);
102
+ expect(findings[0].remediation).toBe('celilo module journal signal');
103
+ });
104
+
105
+ test('each transport is judged on its own record', async () => {
106
+ const findings = await run([
107
+ { transportModuleId: 'signal', last: status({}).last },
108
+ { transportModuleId: 'sms', last: status({ lastSuccessAt: ago(9 * 3600_000) }).last },
109
+ ]);
110
+ expect(findings).toHaveLength(1);
111
+ expect(findings[0].subject).toBe('sms');
112
+ });
113
+ });
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Transport-reads check — is celilo still able to READ replies?
3
+ *
4
+ * Every other signal that a notification transport is working proves the wrong
5
+ * half. `send` succeeding proves outbound. A daemon answering its API proves a
6
+ * socket. Neither can see the return leg fail, and for a week none of them did:
7
+ * six alert tokens were issued, zero were consumed, and nothing anywhere was
8
+ * red (#501).
9
+ *
10
+ * This is the check that would have caught it. It asserts the contract —
11
+ * celilo can read this transport — using first-hand evidence rather than a
12
+ * proxy: the recorded outcome of the reads the poller already performs every
13
+ * few minutes.
14
+ *
15
+ * WHY STALENESS IS A SOUND SIGNAL HERE, and it rests on one decision:
16
+ * an empty read is recorded as a SUCCESS. A transport nobody has replied on
17
+ * still records a successful read on every poll. So "no successful read in a
18
+ * while" cannot mean "quiet" — it can only mean the reads stopped happening or
19
+ * stopped working. Had zero-messages been recorded as failure, this check would
20
+ * page every time an operator had a peaceful afternoon, and would be turned off
21
+ * within a week.
22
+ *
23
+ * Deliberately consumes pre-computed state rather than reading the store
24
+ * itself, so it stays unit-testable and cannot become a second thing that
25
+ * performs reads. A check that drained the queue to find out whether the queue
26
+ * could be drained would eat the acknowledgement it was protecting (#541).
27
+ */
28
+
29
+ import type { TransportReadStatus } from '../alerting/read-records';
30
+ import type { DriftFinding } from './types';
31
+
32
+ export type { TransportReadStatus };
33
+
34
+ export interface TransportReadsAuditDeps {
35
+ /** One entry per transport that has at least one route pointing at it. */
36
+ statuses: TransportReadStatus[];
37
+ now: Date;
38
+ /**
39
+ * How long without a successful read before it counts as drift.
40
+ *
41
+ * The poller runs every five minutes, so this is a multiple of that rather
42
+ * than a guess: it has to absorb a missed tick, a slow sweep, and a restart
43
+ * without crying wolf, while still catching a transport that has genuinely
44
+ * stopped being readable.
45
+ */
46
+ staleAfterMs: number;
47
+ }
48
+
49
+ const CATEGORY = 'transport_reads' as const;
50
+
51
+ function describeAge(ms: number): string {
52
+ const mins = Math.floor(ms / 60_000);
53
+ if (mins < 60) return `${mins}m`;
54
+ const hours = Math.floor(mins / 60);
55
+ return hours < 48 ? `${hours}h` : `${Math.floor(hours / 24)}d`;
56
+ }
57
+
58
+ export async function auditTransportReads(deps: TransportReadsAuditDeps): Promise<DriftFinding[]> {
59
+ const findings: DriftFinding[] = [];
60
+
61
+ for (const status of deps.statuses) {
62
+ const id = status.transportModuleId;
63
+
64
+ // Nothing recorded at all. Either the poller has never run, or this
65
+ // transport was added and never polled. Both mean no reply from here has
66
+ // ever been collectable, which is worth saying out loud rather than
67
+ // treating an empty store as "fine so far".
68
+ if (!status.last) {
69
+ findings.push({
70
+ category: CATEGORY,
71
+ severity: 'drift',
72
+ code: 'transport_never_polled',
73
+ message: `${id}: celilo has never recorded a read attempt`,
74
+ details:
75
+ 'No inbound read has been attempted for this transport, so a\n' +
76
+ 'reply sent to it would not be collected. This is the state a\n' +
77
+ 'newly-added transport is in until the first poll runs.',
78
+ remediation: 'celilo alerts poll',
79
+ actionable: true,
80
+ subject: id,
81
+ });
82
+ continue;
83
+ }
84
+
85
+ // A transport with no `receive` is unidirectional BY DESIGN — pages go out,
86
+ // replies were never possible, and the route's can_ack already records
87
+ // that. Flagging it would be flagging a working system.
88
+ if (status.last.outcome === 'unidirectional') continue;
89
+
90
+ if (!status.last.lastSuccessAt) {
91
+ findings.push({
92
+ category: CATEGORY,
93
+ severity: 'drift',
94
+ code: 'transport_never_read',
95
+ message: `${id}: no read has ever SUCCEEDED`,
96
+ details: `Reads have been attempted — the most recent was ${status.last.outcome}${status.last.error ? ` (${status.last.error})` : ''} — but none has\never succeeded. Acknowledgements sent to this transport cannot be\ncollected, and outbound paging will keep working, so nothing else\nwill report this.`,
97
+ remediation: `celilo module journal ${id}`,
98
+ actionable: true,
99
+ subject: id,
100
+ });
101
+ continue;
102
+ }
103
+
104
+ const age = deps.now.getTime() - new Date(status.last.lastSuccessAt).getTime();
105
+ if (age > deps.staleAfterMs) {
106
+ findings.push({
107
+ category: CATEGORY,
108
+ severity: 'drift',
109
+ code: 'transport_reads_stale',
110
+ message: `${id}: last successful read ${describeAge(age)} ago`,
111
+ details: `The most recent attempt was ${status.last.outcome}${status.last.error ? `: ${status.last.error}` : ''}.\nAn empty read still counts as a success, so this is not "nobody\nreplied" — reads are either not happening or not working, and a\nreply sent now would not be collected.`,
112
+ remediation: `celilo module journal ${id}`,
113
+ actionable: true,
114
+ subject: id,
115
+ });
116
+ }
117
+ }
118
+
119
+ return findings;
120
+ }
@@ -36,6 +36,7 @@ export type DriftCategory =
36
36
  | 'secrets_decryptable'
37
37
  | 'services_reachable'
38
38
  | 'machines_reachable'
39
+ | 'transport_reads'
39
40
  | 'trusted_sources';
40
41
 
41
42
  export type DriftSeverity = 'todo' | 'drift' | 'blocked';
@@ -104,6 +104,35 @@ export async function addBackupStorage(params: {
104
104
  };
105
105
  }
106
106
 
107
+ /**
108
+ * Replace a storage destination's credentials.
109
+ *
110
+ * Clears the verification stamp in the same statement. A `verified`
111
+ * flag describes the destination the credentials pointed at; once they
112
+ * change it describes somewhere else, and carrying it forward is how
113
+ * celilo-mgr ended up reporting `✓ Verified` for a macOS path on a
114
+ * Linux host (#566). Callers re-verify against the new destination.
115
+ */
116
+ export async function updateStorageCredentials(
117
+ id: string,
118
+ credentials: Record<string, unknown>,
119
+ ): Promise<void> {
120
+ const masterKey = await getOrCreateMasterKey();
121
+ const encrypted = encryptSecret(JSON.stringify(credentials), masterKey);
122
+
123
+ getDb()
124
+ .update(backupStorages)
125
+ .set({
126
+ credentialsEncrypted: JSON.stringify(encrypted),
127
+ verified: false,
128
+ verifiedAt: null,
129
+ verificationError: null,
130
+ updatedAt: new Date(),
131
+ })
132
+ .where(eq(backupStorages.id, id))
133
+ .run();
134
+ }
135
+
107
136
  /**
108
137
  * Get backup storage by storage ID (user-facing identifier)
109
138
  */
@@ -14,7 +14,8 @@ import {
14
14
  import { dirname, join, relative } from 'node:path';
15
15
  import type { StorageProvider, StorageVerifyResult } from './types';
16
16
 
17
- const BACKUP_PREFIX = 'celilo-backups';
17
+ /** Subdirectory of the configured path that actually holds archives. */
18
+ export const BACKUP_PREFIX = 'celilo-backups';
18
19
 
19
20
  export interface LocalStorageConfig {
20
21
  path: string;
@@ -82,6 +82,7 @@ const cleanAudit: AuditDeps = {
82
82
  secretsDecryptable: { results: [] },
83
83
  servicesReachable: { results: [] },
84
84
  machinesReachable: { results: [] },
85
+ transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
85
86
  trustedSources: { firewalls: [] },
86
87
  };
87
88