@celilo/cli 0.16.2 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_SUBSYSTEMS.md +4 -1
- package/package.json +2 -2
- package/src/cli/commands/alerts-poll.ts +26 -1
- package/src/cli/commands/storage-set-path.test.ts +281 -0
- package/src/cli/commands/storage-set-path.ts +190 -0
- package/src/cli/commands/system-audit.ts +12 -0
- package/src/cli/commands/system-update.ts +1 -0
- package/src/cli/completion.ts +6 -3
- package/src/cli/index.ts +12 -0
- package/src/cli/tui/audit-state.test.ts +15 -1
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/services/alerting/builtin-monitors.ts +1 -0
- package/src/services/alerting/inbound-poller.test.ts +63 -1
- package/src/services/alerting/inbound-poller.ts +42 -0
- package/src/services/alerting/read-records.ts +85 -0
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/transport-reads.test.ts +113 -0
- package/src/services/audit/transport-reads.ts +120 -0
- package/src/services/audit/types.ts +1 -0
- package/src/services/backup-storage.ts +29 -0
- package/src/services/storage-providers/local.ts +2 -1
- package/src/services/update/orchestrator.test.ts +1 -0
package/CELILO_SUBSYSTEMS.md
CHANGED
|
@@ -197,6 +197,7 @@ Creation, scheduling and freshness. A module declares an `on_backup` hook and a
|
|
|
197
197
|
- **The sweep** — `apps/celilo/src/services/backup-sweep.ts` — `runBackupSweep` (the pass that makes backups run by themselves: for each eligible module, is its declared cadence due → back it up → apply declared retention) + `ensureBackupSweepSubscriber`. Driven by `celilo backup sweep` on `timer.tick.1h` — the coarsest tick that can still serve an `hourly` cadence. Armed automatically: `registerModuleSubscriptions` registers the subscriber for any module declaring an `on_backup` hook, so it appears on install or `module update`. A run refused by the in-flight operation lock is a skip retried next tick, never a failure.
|
|
198
198
|
- **Freshness audit** — `apps/celilo/src/services/audit/backups.ts` (`auditBackups`) — `backup_missing` / `backup_stale` drift findings against the same declared cadence.
|
|
199
199
|
- **CLI** — `apps/celilo/src/cli/commands/` — `backup-sweep.ts`, `backup-create.ts` (also `celilo module backup`), `backup-list.ts`, `backup-restore.ts`, `backup-prune.ts`, `backup-delete.ts`, `backup-import.ts`, `backup-pull.ts`, `backup-name.ts`.
|
|
200
|
+
- **Storage destinations CLI** — `storage-add-local.ts`, `storage-add-s3.ts`, `storage-list.ts`, `storage-verify.ts`, `storage-set-default.ts`, `storage-set-path.ts` (relocate a local destination, migrating existing archives unless `--no-migrate`), `storage-remove.ts`. Any credential change goes through `updateStorageCredentials` in `backup-storage.ts`, which clears the verification stamp — a `✓ Verified` must never describe a destination it was not measured against (#566).
|
|
200
201
|
|
|
201
202
|
## Persistence
|
|
202
203
|
|
|
@@ -218,7 +219,8 @@ Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <h
|
|
|
218
219
|
- **Access control** — `apps/celilo/src/services/api-access.ts` — `grantPrincipal`, `isAuthorized` (deny-by-default, `command:subcommand` grants), `renderAuthorizedKeys`. Table: `api_principals` (`apps/celilo/src/db/schema.ts`). CLI: `apps/celilo/src/cli/commands/api.ts` (`api grant|list|revoke|authorized-keys|key new`).
|
|
219
220
|
- **Mid-run interview bridge (`kind:daemon` responder)** — `apps/celilo/src/services/remote-responder.ts` — `startRemoteResponder` bridges bus `interview.required.*` ↔ wire.
|
|
220
221
|
- **Server provisioning** — the `celilo-bootstrap` deb (`packaging/celilo-bootstrap/scripts/postinst`) creates the non-root `celilo-api` landing account + sshd; membership in the `celilo` group + `/etc/sudoers.d/celilo` (`!use_pty`) gives api-serve DB access via the wrapper's sudo-drop.
|
|
221
|
-
- **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100). It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`.
|
|
222
|
+
- **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100). It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`. **This upgrades celilo ITSELF — not the modules it manages. For those, see Module auto-upgrade below; the two are routinely confused.**
|
|
223
|
+
- **Module auto-upgrade (registry-poll CD)** — the *pull* half of continuous deployment: celilo-mgr polls the registry and upgrades opted-in modules unattended. Spec: `openspec/specs/module-auto-upgrade/spec.md`. Entry points: `apps/celilo/src/cli/commands/module-upgrade.ts` — `runRegistryPoll` (the `--poll` path), `selectPollTargets` (pure: `autoUpgrade && latest && change ∉ {up-to-date, ahead}`), `upgradeOneModule` (update → backup → deploy → verify), `needsPreUpgradeBackup`, `pickAutoUpgrade`/`pickUpgradePolicy` (both fail closed/safe); `classifyVersionChange` in `module-update.ts` (treats a registry `+N` revision as a patch); `resolveDeployPosture` in `apps/celilo/src/services/deploy-posture.ts`. Trigger: celilo-mgmt's `registry-poll` subscription (`modules/celilo-mgmt/manifest.yml`) on `timer.tick.15m` with handler **`celilo module upgrade --poll`** — the flag is REQUIRED, since the dispatcher appends the event id positionally and a bare handler would consume it as the optional module name (silent: 3108 deliveries, 0 successes). Operator controls are framework config keys settable on ANY module (`FRAMEWORK_CONFIG_KEYS` in `module-config.ts`): `auto_upgrade` (opt-in, default false) and `upgrade_policy` (`by-semver`|`always-safe`|`always-fast`), validated at set time because both readers fail open. ⚠️ `always-safe` guarantees safe *posture*, NOT a backup — `needsPreUpgradeBackup` also requires the TARGET manifest to declare an `on_backup` hook, else it warns and proceeds. Confirm a data-bearing module declares `on_backup` before enabling `auto_upgrade` on it. The *build* half (app CI publishing a `.netapp` on merge) is not yet shipped — `openspec/changes/build-bus-poll-cd`.
|
|
222
224
|
- **MCP service (`@celilo/mcp`)** — `packages/mcp/src/` — an operator-facing stdio MCP server (official `@modelcontextprotocol/sdk`, bin `celilo-mcp`) that drives a remote celilo server over the Remote API for an AI client. Two-item config (`config.ts`: `server` + `defaultUser`, env or `~/.config/celilo-mcp/config.json`). Dual-principal auth (`auth.ts`: `celilo-mcp auth setup` enrolls read-only `celilo-mcp-ro` + full `celilo-mcp-rw` ed25519 keypairs, prints the exact `celilo api grant` lines the operator runs server-side). Transport (`transport.ts`): reuses `@celilo/core` `runRemoteClient`, selecting the principal by `ssh -i <key>` and capturing structured output. Tool surface is generated LIVE from the server's command registry — `registry-fetch.ts` fetches `celilo commands --json` (+ `service list --json` for configured providers) over the RO principal on connect; `tools-from-registry.ts` (pure) projects that into one tool per runnable leaf, grouped by top-level command (`celilo_module_*`, `celilo_proxmox_*`, …), each with a Zod input schema from the leaf's args/flags and a read/write tag → RO/RW routing, plus a generic `celilo_run` escape hatch. Auto-detect hides provider-gated groups (e.g. `celilo_proxmox_*` until a Proxmox service is configured) and re-detects on a timer, emitting `notifications/tools/list_changed` when the surface changes. Coverage gate (`tests/coverage.test.ts`) asserts every registry leaf maps to a tool. Composite RO troubleshooting tools (`troubleshoot.ts` pure correlation + `troubleshoot-tools.ts` thin adapters): `celilo_assess_module <id>` and `celilo_fleet_status` correlate `celilo audit --json` (the drift backbone) with the `module list --json` roster into a per-module / fleet-wide verdict. Design: `openspec/changes/celilo-mcp-service/proposal.md`. (Distinct from the dev/ops `@celilo/mcp-server` below.)
|
|
223
225
|
|
|
224
226
|
## E2E simulation
|
|
@@ -227,4 +229,5 @@ Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <h
|
|
|
227
229
|
- **Run wrapper** — `infra/scripts/cele2e-run.sh` lives in the separate `infra/` clone, **not** in this repo. See the cele2e section of the repo-root `CLAUDE.md` for the operator workflow.
|
|
228
230
|
- **Signal simulators** — three containers, three jobs. `docker/Dockerfile.signal-cli` runs the REAL unlinked daemon (`network().withSignalCli()`, reachable at `signal-cli.lab`) so `e2e/tests/signal-contract.test.ts` can re-check celilo's understanding of the JSON-RPC surface against the actual binary — the pebble pattern. `docker/Dockerfile.signal-sim` runs `simulators/signal-cli/server.ts` (`withSignalSim()`, `signal-sim.lab`), the drivable stand-in with a control surface (`/_control/inbound`, `/_control/sent`, `/_control/unlink`). `/_control/inbound` from the LINKED account's own number emits a `syncMessage.sentMessage` transcript rather than a `dataMessage`, because that is the only shape the real daemon delivers a note-to-self in — and a note-to-self is what every reply is in the default single-operator setup (#460). `docker/Dockerfile.signal-release` (`withSignalRelease()`, `signal-release.lab`) serves the signal-cli release tarball so the module's deploy-time download resolves inside the sealed network — the download is served, never skipped. The libsignal aarch64 native is compiled at `build-infra` time from the module's own recipe (`packages/e2e/scripts/stage-libsignal.ts` → `modules/signal/build/`) and staged into the apt-repo pool, so the recipe is exercised for real on every rebuild while the deploy stays fast and the network stays sealed.
|
|
229
231
|
- **Public-boundary NAT model** — `packages/e2e/config/routing/` — exactly ONE NAT sits between the fleet and the simulated internet: the customer firewall (`fw-main` in `direct-internet`, `fw-isp` in the two-layer default), which MASQUERADEs to `100.100.0.100`. The ISP edge `fw-ext` ROUTES the customer's `100.100.0.0/24` and must never re-NAT it (`-s 100.100.0.0/24 -j RETURN` ahead of its MASQUERADE) — an ISP does not NAT a subscriber that already holds a public address. This is load-bearing, not cosmetic: Namecheap-style DDNS registers the SOURCE address when the caller omits `ip=`, which is how celilo registers public names since #464/#466, so a second NAT here publishes the simulator's own address for every public hostname and quietly breaks ACME, inbound reach, and seeded apex records. Constants: `externalWanIp()` / `externalWanSubnet()` in `src/types.ts`. The corollary: every simulator on `internet-external` must default-route via fw-ext (`100.64.0.1`) so it can reply to the customer's public address — Docker's bridge gateway has no path across networks. `config/routing/public-sim-entrypoint.sh` is the shared two-liner; the DNS hierarchy, pebble, isitup and celilo-website-sim already carried it, and npm-registry / registry / apt-repo / minio / cpanel-host only appeared to work because the double NAT put fw-ext's on-link address in the source field.
|
|
232
|
+
- **Simulated address plan** — `packages/e2e/src/types.ts` — `SIM_PRIVATE_SUPERNET` (`10.226.0.0/16`), `zoneIp(zone, host)`, `ZONE_SUBNETS`, `ZONE_GATEWAYS`. Every simulated PRIVATE zone is derived from this one table; the compose generator, `zone-classifier.ts` and the harness's `system init` all read it, so renumbering the whole sim is a one-line change. The sim deliberately does NOT reuse a real fleet's zone /24s (#539): the previous plan was byte-identical to production's, so a stack leaked on celilo's own forgejo-builder — which lives in the real dmz — claimed the builder's own subnet and blackholed every containerized CI job's route to the forge for ~135s. Teardown cannot prevent that (a SIGKILL runs no handler), so the addresses moved instead; the second, better reason is that a suite which only passes on production's exact octets is asserting one site's address plan rather than celilo's behaviour. `src/address-plan.test.ts` is the recurrence gate — it fails if a zone leaves the supernet, or if any retired fleet prefix reappears anywhere in `packages/e2e`, `e2e/tests` or `modules/*/e2e`.
|
|
230
233
|
- **MCP server (agent-driven e2e)** — `packages/mcp-server/src/index.ts` — stdio MCP server exposing `start_run`/`run_status`/`run_result`/`stop_run` (detached cele2e runs read off the event bus, no ANSI scraping) + `env_check` (docker VM, run-lock, shared-infra, mgmt image CLI version, netapps). Dev/ops tool, `private`, not shipped to consumers.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "Celilo — home lab orchestration CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -59,7 +59,7 @@
|
|
|
59
59
|
"@aws-sdk/client-s3": "^3.1024.0",
|
|
60
60
|
"@celilo/capabilities": "^0.9.1",
|
|
61
61
|
"@celilo/cli-display": "^0.1.9",
|
|
62
|
-
"@celilo/core": "^0.3.
|
|
62
|
+
"@celilo/core": "^0.3.1",
|
|
63
63
|
"@celilo/event-bus": "^0.1.10",
|
|
64
64
|
"@clack/prompts": "^1.1.0",
|
|
65
65
|
"ajv": "^8.18.0",
|
|
@@ -17,6 +17,7 @@ import { systemConfig } from '../../db/schema';
|
|
|
17
17
|
import { makeReceiver, pollInbound } from '../../services/alerting/inbound-poller';
|
|
18
18
|
import { startNotificationResponder } from '../../services/alerting/notification-responder';
|
|
19
19
|
import { listRoutes } from '../../services/alerting/people';
|
|
20
|
+
import { readLastRead, writeLastRead } from '../../services/alerting/read-records';
|
|
20
21
|
import { loadNotificationTransport } from '../../services/alerting/transport-loader';
|
|
21
22
|
import type { CommandResult } from '../types';
|
|
22
23
|
|
|
@@ -52,6 +53,26 @@ function writeCursor(
|
|
|
52
53
|
}
|
|
53
54
|
}
|
|
54
55
|
|
|
56
|
+
/**
|
|
57
|
+
* " (last read OK 3h ago)" or " (never read successfully)".
|
|
58
|
+
*
|
|
59
|
+
* Appended to a read failure because the failure alone does not say how bad it
|
|
60
|
+
* is. One failed poll is a blip; a transport that has not been readable since
|
|
61
|
+
* Tuesday is an outage nobody was told about, and those two produced identical
|
|
62
|
+
* output until this record existed.
|
|
63
|
+
*/
|
|
64
|
+
function sinceLastSuccess(db: ReturnType<typeof getDb>, transportModuleId: string): string {
|
|
65
|
+
const last = readLastRead(db, transportModuleId)?.lastSuccessAt;
|
|
66
|
+
if (!last) return ' (never read successfully)';
|
|
67
|
+
const ms = Date.now() - new Date(last).getTime();
|
|
68
|
+
const mins = Math.floor(ms / 60_000);
|
|
69
|
+
if (mins < 60) return ` (last read OK ${mins}m ago)`;
|
|
70
|
+
const hours = Math.floor(mins / 60);
|
|
71
|
+
return hours < 48
|
|
72
|
+
? ` (last read OK ${hours}h ago)`
|
|
73
|
+
: ` (last read OK ${Math.floor(hours / 24)}d ago)`;
|
|
74
|
+
}
|
|
75
|
+
|
|
55
76
|
export async function handleAlertsPoll(
|
|
56
77
|
flags: Record<string, string | boolean> = {},
|
|
57
78
|
): Promise<CommandResult> {
|
|
@@ -97,6 +118,7 @@ export async function handleAlertsPoll(
|
|
|
97
118
|
receiveFrom: makeReceiver(db),
|
|
98
119
|
readCursor: (t) => readCursor(db, t),
|
|
99
120
|
writeCursor: (t, c) => writeCursor(db, t, c),
|
|
121
|
+
recordRead: (t, r) => writeLastRead(db, t, r),
|
|
100
122
|
now: () => new Date(),
|
|
101
123
|
transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
|
|
102
124
|
answerInterview: (eventId, value) => responder.answer(eventId, value),
|
|
@@ -122,7 +144,10 @@ export async function handleAlertsPoll(
|
|
|
122
144
|
// an operator has no way to tell which — the ambiguity that made the Signal
|
|
123
145
|
// ack path take a week to diagnose.
|
|
124
146
|
const detail = [
|
|
125
|
-
...report.failures.map(
|
|
147
|
+
...report.failures.map(
|
|
148
|
+
(f) =>
|
|
149
|
+
`${f.transportModuleId} COULD NOT BE READ: ${f.error}${sinceLastSuccess(db, f.transportModuleId)}`,
|
|
150
|
+
),
|
|
126
151
|
...report.unheard.map((u) => `not heard from ${u.senderAddress}: ${u.reason}`),
|
|
127
152
|
...askErrors,
|
|
128
153
|
];
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for `storage set-path`.
|
|
3
|
+
*
|
|
4
|
+
* The motivating case (#566) is celilo-mgr: `local-backups` points at
|
|
5
|
+
* `/Users/pbanka/hobby/backups/celilo-backups/`, a macOS path that came
|
|
6
|
+
* across when the database was restored onto a Linux host. The directory
|
|
7
|
+
* does not exist there. Relocating must therefore succeed with nothing
|
|
8
|
+
* migrated — not error, and not claim files were moved — and must not
|
|
9
|
+
* carry the four-month-old `✓ Verified` stamp onto the new path.
|
|
10
|
+
*
|
|
11
|
+
* Isolation: CELILO_DB_PATH / CELILO_DATA_DIR are set before the SUT is
|
|
12
|
+
* imported, so nothing touches the production database.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { afterAll, describe, expect, test } from 'bun:test';
|
|
16
|
+
import {
|
|
17
|
+
chmodSync,
|
|
18
|
+
existsSync,
|
|
19
|
+
mkdirSync,
|
|
20
|
+
mkdtempSync,
|
|
21
|
+
rmSync,
|
|
22
|
+
statSync,
|
|
23
|
+
utimesSync,
|
|
24
|
+
writeFileSync,
|
|
25
|
+
} from 'node:fs';
|
|
26
|
+
import { homedir, tmpdir } from 'node:os';
|
|
27
|
+
import { join } from 'node:path';
|
|
28
|
+
|
|
29
|
+
const testRoot = mkdtempSync(join(tmpdir(), 'celilo-setpath-'));
|
|
30
|
+
process.env.CELILO_DB_PATH = join(testRoot, 'celilo.db');
|
|
31
|
+
process.env.CELILO_DATA_DIR = join(testRoot, 'data');
|
|
32
|
+
|
|
33
|
+
const {
|
|
34
|
+
addBackupStorage,
|
|
35
|
+
getBackupStorageByStorageId,
|
|
36
|
+
getStorageCredentials,
|
|
37
|
+
updateStorageCredentials,
|
|
38
|
+
} = await import('../../services/backup-storage');
|
|
39
|
+
const { handleStorageSetPath, inspectSourceDir, planRelocation, resolveTargetPath } = await import(
|
|
40
|
+
'./storage-set-path'
|
|
41
|
+
);
|
|
42
|
+
|
|
43
|
+
afterAll(() => {
|
|
44
|
+
rmSync(testRoot, { recursive: true, force: true });
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
describe('planRelocation (pure)', () => {
|
|
48
|
+
const base = { sourceDir: '/old/celilo-backups', targetDir: '/new/celilo-backups' };
|
|
49
|
+
|
|
50
|
+
test('migrates when the source has files', () => {
|
|
51
|
+
const plan = planRelocation({ ...base, sourceState: 'populated', migrateRequested: true });
|
|
52
|
+
expect(plan.migrate).toBe(true);
|
|
53
|
+
expect(plan.note).toContain('Migrating archives');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test('PRODUCTION CASE: source missing — skips migration, says so, does not error', () => {
|
|
57
|
+
const plan = planRelocation({ ...base, sourceState: 'missing', migrateRequested: true });
|
|
58
|
+
expect(plan.migrate).toBe(false);
|
|
59
|
+
expect(plan.note).toContain('does not exist');
|
|
60
|
+
expect(plan.note).toContain('nothing to migrate');
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test('source unreadable — skips migration and says nothing was moved', () => {
|
|
64
|
+
const plan = planRelocation({ ...base, sourceState: 'unreadable', migrateRequested: true });
|
|
65
|
+
expect(plan.migrate).toBe(false);
|
|
66
|
+
expect(plan.note).toContain('not readable');
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test('source empty — skips migration', () => {
|
|
70
|
+
const plan = planRelocation({ ...base, sourceState: 'empty', migrateRequested: true });
|
|
71
|
+
expect(plan.migrate).toBe(false);
|
|
72
|
+
expect(plan.note).toContain('empty');
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test('--no-migrate wins even over a populated source', () => {
|
|
76
|
+
const plan = planRelocation({ ...base, sourceState: 'populated', migrateRequested: false });
|
|
77
|
+
expect(plan.migrate).toBe(false);
|
|
78
|
+
expect(plan.note).toContain('--no-migrate');
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe('resolveTargetPath (pure)', () => {
|
|
83
|
+
test('rejects a path identical to the current one', () => {
|
|
84
|
+
const result = resolveTargetPath('/var/backups', '/var/backups/');
|
|
85
|
+
expect(result).toMatchObject({ success: false });
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test('preserves a path containing a space', () => {
|
|
89
|
+
const result = resolveTargetPath('/tmp/back ups/celilo', '/somewhere/else');
|
|
90
|
+
expect(result).toEqual({ path: '/tmp/back ups/celilo' });
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test('expands a leading tilde', () => {
|
|
94
|
+
const result = resolveTargetPath('~/backups', '/somewhere/else');
|
|
95
|
+
expect(result).toMatchObject({ path: join(homedir(), 'backups') });
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
describe('inspectSourceDir', () => {
|
|
100
|
+
test('missing directory reports missing, not unreadable', () => {
|
|
101
|
+
expect(inspectSourceDir(join(testRoot, 'no-such-dir'))).toBe('missing');
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('empty directory reports empty', () => {
|
|
105
|
+
const dir = join(testRoot, 'empty-dir');
|
|
106
|
+
mkdirSync(dir, { recursive: true });
|
|
107
|
+
expect(inspectSourceDir(dir)).toBe('empty');
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test('directory with files reports populated', () => {
|
|
111
|
+
const dir = join(testRoot, 'full-dir');
|
|
112
|
+
mkdirSync(dir, { recursive: true });
|
|
113
|
+
writeFileSync(join(dir, 'a.backup'), 'x');
|
|
114
|
+
expect(inspectSourceDir(dir)).toBe('populated');
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test('unreadable directory reports unreadable, not missing', () => {
|
|
118
|
+
const parent = join(testRoot, 'locked');
|
|
119
|
+
const dir = join(parent, 'celilo-backups');
|
|
120
|
+
mkdirSync(dir, { recursive: true });
|
|
121
|
+
chmodSync(parent, 0o000);
|
|
122
|
+
try {
|
|
123
|
+
expect(inspectSourceDir(dir)).toBe('unreadable');
|
|
124
|
+
} finally {
|
|
125
|
+
chmodSync(parent, 0o700);
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
describe('updateStorageCredentials', () => {
|
|
131
|
+
test('clears the verification stamp, so a stale ✓ never survives a credential change (#566)', async () => {
|
|
132
|
+
const storage = await addBackupStorage({
|
|
133
|
+
name: 'Stamp Backups',
|
|
134
|
+
providerName: 'local',
|
|
135
|
+
credentials: { path: '/Users/nobody/old' },
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
const { getDb } = await import('../../db/client');
|
|
139
|
+
const { backupStorages } = await import('../../db/schema');
|
|
140
|
+
const { eq } = await import('drizzle-orm');
|
|
141
|
+
getDb()
|
|
142
|
+
.update(backupStorages)
|
|
143
|
+
.set({ verified: true, verifiedAt: new Date('2026-04-08T02:24:43.000Z') })
|
|
144
|
+
.where(eq(backupStorages.id, storage.id))
|
|
145
|
+
.run();
|
|
146
|
+
|
|
147
|
+
await updateStorageCredentials(storage.id, { path: '/var/lib/celilo/backups' });
|
|
148
|
+
|
|
149
|
+
// Asserted WITHOUT a follow-up verify: even if celilo dies between
|
|
150
|
+
// the path change and re-verification, the row must not still claim
|
|
151
|
+
// the destination was verified.
|
|
152
|
+
const after = getBackupStorageByStorageId(storage.storageId);
|
|
153
|
+
expect(after?.verified).toBe(false);
|
|
154
|
+
expect(after?.verifiedAt).toBeNull();
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
describe('handleStorageSetPath (end to end, isolated DB)', () => {
|
|
159
|
+
test('PRODUCTION CASE: current path does not exist — path changes, nothing migrated, re-verified against the new path', async () => {
|
|
160
|
+
const storage = await addBackupStorage({
|
|
161
|
+
name: 'Ghost Backups',
|
|
162
|
+
providerName: 'local',
|
|
163
|
+
credentials: { path: '/Users/nobody/hobby/backups' },
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
// Simulate the stale stamp: the row says Verified from a host that
|
|
167
|
+
// no longer exists.
|
|
168
|
+
const { getDb } = await import('../../db/client');
|
|
169
|
+
const { backupStorages } = await import('../../db/schema');
|
|
170
|
+
const { eq } = await import('drizzle-orm');
|
|
171
|
+
getDb()
|
|
172
|
+
.update(backupStorages)
|
|
173
|
+
.set({ verified: true, verifiedAt: new Date('2026-04-08T02:24:43.000Z') })
|
|
174
|
+
.where(eq(backupStorages.id, storage.id))
|
|
175
|
+
.run();
|
|
176
|
+
|
|
177
|
+
const newPath = join(testRoot, 'relocated');
|
|
178
|
+
const result = await handleStorageSetPath([storage.storageId, newPath]);
|
|
179
|
+
|
|
180
|
+
expect(result.success).toBe(true);
|
|
181
|
+
|
|
182
|
+
const creds = await getStorageCredentials(storage.id);
|
|
183
|
+
expect(creds).toMatchObject({ path: newPath });
|
|
184
|
+
|
|
185
|
+
// The stamp must describe the NEW path, not the dead one.
|
|
186
|
+
const after = getBackupStorageByStorageId(storage.storageId);
|
|
187
|
+
expect(after?.verified).toBe(true);
|
|
188
|
+
expect(after?.verifiedAt?.getTime()).toBeGreaterThan(
|
|
189
|
+
new Date('2026-04-08T02:24:43.000Z').getTime(),
|
|
190
|
+
);
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
test('moves existing archives to the new location', async () => {
|
|
194
|
+
const oldPath = join(testRoot, 'movable-old');
|
|
195
|
+
const newPath = join(testRoot, 'movable-new');
|
|
196
|
+
mkdirSync(join(oldPath, 'celilo-backups', '2026-08-01'), { recursive: true });
|
|
197
|
+
writeFileSync(join(oldPath, 'celilo-backups', '2026-08-01', 'x.backup'), 'payload');
|
|
198
|
+
|
|
199
|
+
const storage = await addBackupStorage({
|
|
200
|
+
name: 'Movable Backups',
|
|
201
|
+
providerName: 'local',
|
|
202
|
+
credentials: { path: oldPath },
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
const result = await handleStorageSetPath([storage.storageId, newPath]);
|
|
206
|
+
|
|
207
|
+
expect(result.success).toBe(true);
|
|
208
|
+
expect(existsSync(join(newPath, 'celilo-backups', '2026-08-01', 'x.backup'))).toBe(true);
|
|
209
|
+
expect(existsSync(join(oldPath, 'celilo-backups'))).toBe(false);
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
test('preserves archive mtimes — a move does not restamp backups', async () => {
|
|
213
|
+
const oldPath = join(testRoot, 'mtime-old');
|
|
214
|
+
const newPath = join(testRoot, 'mtime-new');
|
|
215
|
+
const archive = join(oldPath, 'celilo-backups', 'a.backup');
|
|
216
|
+
mkdirSync(join(oldPath, 'celilo-backups'), { recursive: true });
|
|
217
|
+
writeFileSync(archive, 'payload');
|
|
218
|
+
const stamp = new Date('2026-04-26T01:37:53.740Z');
|
|
219
|
+
utimesSync(archive, stamp, stamp);
|
|
220
|
+
|
|
221
|
+
const storage = await addBackupStorage({
|
|
222
|
+
name: 'Mtime Backups',
|
|
223
|
+
providerName: 'local',
|
|
224
|
+
credentials: { path: oldPath },
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
await handleStorageSetPath([storage.storageId, newPath]);
|
|
228
|
+
|
|
229
|
+
const moved = statSync(join(newPath, 'celilo-backups', 'a.backup'));
|
|
230
|
+
expect(Math.round(moved.mtimeMs)).toBe(stamp.getTime());
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
test('--no-migrate leaves the old archives where they are', async () => {
|
|
234
|
+
const oldPath = join(testRoot, 'kept-old');
|
|
235
|
+
const newPath = join(testRoot, 'kept-new');
|
|
236
|
+
mkdirSync(join(oldPath, 'celilo-backups'), { recursive: true });
|
|
237
|
+
writeFileSync(join(oldPath, 'celilo-backups', 'y.backup'), 'payload');
|
|
238
|
+
|
|
239
|
+
const storage = await addBackupStorage({
|
|
240
|
+
name: 'Kept Backups',
|
|
241
|
+
providerName: 'local',
|
|
242
|
+
credentials: { path: oldPath },
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
const result = await handleStorageSetPath([storage.storageId, newPath], {
|
|
246
|
+
'no-migrate': true,
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
expect(result.success).toBe(true);
|
|
250
|
+
expect(existsSync(join(oldPath, 'celilo-backups', 'y.backup'))).toBe(true);
|
|
251
|
+
expect(existsSync(join(newPath, 'celilo-backups', 'y.backup'))).toBe(false);
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
test('handles a new path containing a space', async () => {
|
|
255
|
+
const oldPath = join(testRoot, 'spacey-old');
|
|
256
|
+
const newPath = join(testRoot, 'back ups', "Bob's celilo");
|
|
257
|
+
mkdirSync(join(oldPath, 'celilo-backups'), { recursive: true });
|
|
258
|
+
writeFileSync(join(oldPath, 'celilo-backups', 'z.backup'), 'payload');
|
|
259
|
+
|
|
260
|
+
const storage = await addBackupStorage({
|
|
261
|
+
name: 'Spacey Backups',
|
|
262
|
+
providerName: 'local',
|
|
263
|
+
credentials: { path: oldPath },
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
const result = await handleStorageSetPath([storage.storageId, newPath]);
|
|
267
|
+
|
|
268
|
+
expect(result.success).toBe(true);
|
|
269
|
+
expect(existsSync(join(newPath, 'celilo-backups', 'z.backup'))).toBe(true);
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
test('rejects an unknown storage id', async () => {
|
|
273
|
+
const result = await handleStorageSetPath(['no-such-storage', join(testRoot, 'x')]);
|
|
274
|
+
expect(result).toMatchObject({ success: false });
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
test('requires both a storage id and a path', async () => {
|
|
278
|
+
const result = await handleStorageSetPath(['only-one-arg']);
|
|
279
|
+
expect(result).toMatchObject({ success: false });
|
|
280
|
+
});
|
|
281
|
+
});
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Storage Set Path Command
|
|
3
|
+
* Relocate a local storage destination to a new directory, migrating
|
|
4
|
+
* any archives that are actually there.
|
|
5
|
+
*
|
|
6
|
+
* Motivating case (#566): celilo-mgr's `local-backups` points at
|
|
7
|
+
* `/Users/pbanka/...`, a macOS path carried across when the database was
|
|
8
|
+
* restored onto a Linux host. The directory does not exist there, so the
|
|
9
|
+
* relocation must complete cleanly with nothing to migrate rather than
|
|
10
|
+
* erroring — and must not carry the old `✓ Verified` stamp onto the new
|
|
11
|
+
* path.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { cpSync, readdirSync, rmSync } from 'node:fs';
|
|
15
|
+
import { homedir } from 'node:os';
|
|
16
|
+
import { join, resolve } from 'node:path';
|
|
17
|
+
import {
|
|
18
|
+
getBackupStorageByStorageId,
|
|
19
|
+
getStorageCredentials,
|
|
20
|
+
updateStorageCredentials,
|
|
21
|
+
verifyBackupStorage,
|
|
22
|
+
} from '../../services/backup-storage';
|
|
23
|
+
import { BACKUP_PREFIX } from '../../services/storage-providers/local';
|
|
24
|
+
import { celiloIntro, celiloOutro } from '../prompts';
|
|
25
|
+
import type { CommandResult } from '../types';
|
|
26
|
+
import { probePathWriteable } from './storage-add-local';
|
|
27
|
+
|
|
28
|
+
/** What the current archive directory turned out to be, on disk. */
|
|
29
|
+
export type SourceState = 'missing' | 'unreadable' | 'empty' | 'populated';
|
|
30
|
+
|
|
31
|
+
export interface RelocationPlan {
|
|
32
|
+
/** Whether to copy the source tree to the target. */
|
|
33
|
+
migrate: boolean;
|
|
34
|
+
/** One line for the operator explaining what will (not) happen. */
|
|
35
|
+
note: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Decide whether to migrate. Pure — takes an already-observed source
|
|
40
|
+
* state so it is testable without a filesystem (Rule 10.4).
|
|
41
|
+
*/
|
|
42
|
+
export function planRelocation(input: {
|
|
43
|
+
sourceState: SourceState;
|
|
44
|
+
sourceDir: string;
|
|
45
|
+
targetDir: string;
|
|
46
|
+
migrateRequested: boolean;
|
|
47
|
+
}): RelocationPlan {
|
|
48
|
+
const { sourceState, sourceDir, targetDir, migrateRequested } = input;
|
|
49
|
+
|
|
50
|
+
if (!migrateRequested) {
|
|
51
|
+
return { migrate: false, note: `Migration skipped (--no-migrate). ${sourceDir} left as-is.` };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
switch (sourceState) {
|
|
55
|
+
case 'missing':
|
|
56
|
+
return { migrate: false, note: `${sourceDir} does not exist — nothing to migrate.` };
|
|
57
|
+
case 'unreadable':
|
|
58
|
+
return { migrate: false, note: `${sourceDir} is not readable — nothing migrated.` };
|
|
59
|
+
case 'empty':
|
|
60
|
+
return { migrate: false, note: `${sourceDir} is empty — nothing to migrate.` };
|
|
61
|
+
case 'populated':
|
|
62
|
+
return { migrate: true, note: `Migrating archives from ${sourceDir} to ${targetDir}.` };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Validate and normalise the requested path. Pure apart from `~`
|
|
68
|
+
* expansion, which reads the environment but touches no filesystem.
|
|
69
|
+
*/
|
|
70
|
+
export function resolveTargetPath(
|
|
71
|
+
raw: string,
|
|
72
|
+
currentPath: string,
|
|
73
|
+
): CommandResult | { path: string } {
|
|
74
|
+
const expanded = raw.startsWith('~/') || raw === '~' ? raw.replace('~', homedir()) : raw;
|
|
75
|
+
const resolved = resolve(expanded);
|
|
76
|
+
|
|
77
|
+
if (resolved === resolve(currentPath)) {
|
|
78
|
+
return { success: false, error: `Storage path is already '${resolved}' — nothing to do.` };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
return { path: resolved };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Observe the archive directory. Distinguishes absent from unreadable. */
|
|
85
|
+
export function inspectSourceDir(dir: string): SourceState {
|
|
86
|
+
try {
|
|
87
|
+
return readdirSync(dir).length === 0 ? 'empty' : 'populated';
|
|
88
|
+
} catch (error) {
|
|
89
|
+
return (error as NodeJS.ErrnoException).code === 'ENOENT' ? 'missing' : 'unreadable';
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export async function handleStorageSetPath(
|
|
94
|
+
args: string[],
|
|
95
|
+
flags: Record<string, boolean | string> = {},
|
|
96
|
+
): Promise<CommandResult> {
|
|
97
|
+
try {
|
|
98
|
+
celiloIntro('Relocate Backup Storage');
|
|
99
|
+
|
|
100
|
+
const storageId = args[0];
|
|
101
|
+
const newPathArg = args[1];
|
|
102
|
+
if (!storageId || !newPathArg) {
|
|
103
|
+
return {
|
|
104
|
+
success: false,
|
|
105
|
+
error:
|
|
106
|
+
'Storage ID and new path are required\n\nUsage: celilo storage set-path <storage-id> <new-path> [--no-migrate]',
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const storage = getBackupStorageByStorageId(storageId);
|
|
111
|
+
if (!storage) {
|
|
112
|
+
return { success: false, error: `Storage not found: ${storageId}` };
|
|
113
|
+
}
|
|
114
|
+
if (storage.providerName !== 'local') {
|
|
115
|
+
return {
|
|
116
|
+
success: false,
|
|
117
|
+
error: `'${storage.storageId}' is a ${storage.providerName} destination — set-path only applies to local storage.`,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const credentials = await getStorageCredentials(storage.id);
|
|
122
|
+
if (!('path' in credentials)) {
|
|
123
|
+
return { success: false, error: `Storage '${storage.storageId}' has no path configured.` };
|
|
124
|
+
}
|
|
125
|
+
const currentPath = credentials.path;
|
|
126
|
+
|
|
127
|
+
const resolved = resolveTargetPath(newPathArg, currentPath);
|
|
128
|
+
if ('success' in resolved) return resolved;
|
|
129
|
+
const newPath = resolved.path;
|
|
130
|
+
|
|
131
|
+
const writeError = probePathWriteable(newPath);
|
|
132
|
+
if (writeError !== null) {
|
|
133
|
+
return { success: false, error: `Path '${newPath}' is not writeable: ${writeError}` };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const sourceDir = join(currentPath, BACKUP_PREFIX);
|
|
137
|
+
const targetDir = join(newPath, BACKUP_PREFIX);
|
|
138
|
+
const plan = planRelocation({
|
|
139
|
+
sourceState: inspectSourceDir(sourceDir),
|
|
140
|
+
sourceDir,
|
|
141
|
+
targetDir,
|
|
142
|
+
migrateRequested: flags['no-migrate'] !== true,
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
console.log(`\n${plan.note}`);
|
|
146
|
+
|
|
147
|
+
if (plan.migrate) {
|
|
148
|
+
// A backup archive's mtime is part of what it is; a move must not
|
|
149
|
+
// restamp it. Bun's cpSync already preserves timestamps, so the
|
|
150
|
+
// flag is a no-op today — it is here for Node semantics, where the
|
|
151
|
+
// default is the other way. The mtime test guards the behavior,
|
|
152
|
+
// not this flag.
|
|
153
|
+
cpSync(sourceDir, targetDir, { recursive: true, force: true, preserveTimestamps: true });
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
await updateStorageCredentials(storage.id, { ...credentials, path: newPath });
|
|
157
|
+
console.log(`✓ Path updated: ${currentPath} → ${newPath}`);
|
|
158
|
+
|
|
159
|
+
if (plan.migrate) {
|
|
160
|
+
// Only after the DB points at the copy, so a failure here leaves
|
|
161
|
+
// archives duplicated rather than orphaned.
|
|
162
|
+
try {
|
|
163
|
+
rmSync(sourceDir, { recursive: true, force: true });
|
|
164
|
+
} catch (error) {
|
|
165
|
+
console.log(
|
|
166
|
+
`⚠ Copied, but could not remove ${sourceDir}: ${error instanceof Error ? error.message : String(error)}`,
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const { result } = await verifyBackupStorage(storage.id);
|
|
172
|
+
if (!result.success) {
|
|
173
|
+
console.log(`✗ ${result.message}`);
|
|
174
|
+
celiloOutro(
|
|
175
|
+
`Path changed but verification failed.\n\nFix the path and re-verify: celilo storage verify ${storage.storageId}`,
|
|
176
|
+
);
|
|
177
|
+
return { success: false, error: result.message };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
console.log(`✓ ${result.message}`);
|
|
181
|
+
celiloOutro(`'${storage.storageId}' now stores backups at ${targetDir}/`);
|
|
182
|
+
|
|
183
|
+
return { success: true, message: `Relocated ${storage.storageId} to ${newPath}` };
|
|
184
|
+
} catch (error) {
|
|
185
|
+
return {
|
|
186
|
+
success: false,
|
|
187
|
+
error: `Failed to set storage path: ${error instanceof Error ? error.message : String(error)}`,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
}
|
|
@@ -35,6 +35,7 @@ import type { ModuleManifest } from '../../manifest/schema';
|
|
|
35
35
|
import { RegistryClient } from '../../registry/client';
|
|
36
36
|
import { decryptSecret } from '../../secrets/encryption';
|
|
37
37
|
import { getOrCreateMasterKey } from '../../secrets/master-key';
|
|
38
|
+
import { readAllTransportStatuses } from '../../services/alerting/read-records';
|
|
38
39
|
import { runAudit } from '../../services/audit';
|
|
39
40
|
import type { DriftFinding, SystemAuditReport } from '../../services/audit';
|
|
40
41
|
import { loadBackupAuditInfo } from '../../services/audit/backup-source';
|
|
@@ -378,6 +379,17 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
|
|
|
378
379
|
secretsDecryptable: { results: secretResults },
|
|
379
380
|
servicesReachable: { results: serviceReachableResults },
|
|
380
381
|
machinesReachable: { results: machineReachableResults },
|
|
382
|
+
// Reads the record the poller already writes — this check never performs a
|
|
383
|
+
// read of its own. One that did would drain the queue and eat the
|
|
384
|
+
// acknowledgement it exists to protect (#541).
|
|
385
|
+
transportReads: {
|
|
386
|
+
statuses: readAllTransportStatuses(db),
|
|
387
|
+
now: new Date(),
|
|
388
|
+
// Six missed polls. The poller runs every five minutes, so this absorbs a
|
|
389
|
+
// slow sweep, a restart, and a missed tick without crying wolf — while
|
|
390
|
+
// still catching a transport that has genuinely stopped being readable.
|
|
391
|
+
staleAfterMs: 30 * 60_000,
|
|
392
|
+
},
|
|
381
393
|
trustedSources: { firewalls: collectFirewallReach(db) },
|
|
382
394
|
};
|
|
383
395
|
}
|
|
@@ -585,6 +585,7 @@ export async function handleSystemUpdate(
|
|
|
585
585
|
secretsDecryptable: { results: [] },
|
|
586
586
|
servicesReachable: { results: [] },
|
|
587
587
|
machinesReachable: { results: [] },
|
|
588
|
+
transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
|
|
588
589
|
trustedSources: { firewalls: [] },
|
|
589
590
|
};
|
|
590
591
|
|
package/src/cli/completion.ts
CHANGED
|
@@ -486,7 +486,7 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
486
486
|
|
|
487
487
|
// Storage subcommands
|
|
488
488
|
if (command === 'storage' && currentIndex === 1) {
|
|
489
|
-
const subcommands = ['add', 'list', 'remove', 'verify', 'set-default'];
|
|
489
|
+
const subcommands = ['add', 'list', 'remove', 'verify', 'set-default', 'set-path'];
|
|
490
490
|
return filterSuggestions(subcommands, args[1] || '');
|
|
491
491
|
}
|
|
492
492
|
|
|
@@ -496,10 +496,13 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
496
496
|
return filterSuggestions(providers, args[2] || '');
|
|
497
497
|
}
|
|
498
498
|
|
|
499
|
-
// Storage remove/verify/set-default - complete with storage IDs
|
|
499
|
+
// Storage remove/verify/set-default/set-path - complete with storage IDs
|
|
500
500
|
if (
|
|
501
501
|
command === 'storage' &&
|
|
502
|
-
(args[1] === 'remove' ||
|
|
502
|
+
(args[1] === 'remove' ||
|
|
503
|
+
args[1] === 'verify' ||
|
|
504
|
+
args[1] === 'set-default' ||
|
|
505
|
+
args[1] === 'set-path') &&
|
|
503
506
|
currentIndex === 2
|
|
504
507
|
) {
|
|
505
508
|
const storages = listBackupStorages();
|
package/src/cli/index.ts
CHANGED
|
@@ -839,6 +839,9 @@ Subcommands:
|
|
|
839
839
|
list List all configured storage destinations
|
|
840
840
|
verify <storage-id> Test storage connectivity and permissions
|
|
841
841
|
set-default <id> Set the default backup storage destination
|
|
842
|
+
set-path <id> <path> Relocate a local destination to a new directory
|
|
843
|
+
Options:
|
|
844
|
+
--no-migrate Change the path without moving existing archives
|
|
842
845
|
remove <storage-id> Remove a storage destination
|
|
843
846
|
Options:
|
|
844
847
|
--force Skip confirmation prompts
|
|
@@ -860,6 +863,10 @@ Examples:
|
|
|
860
863
|
# Set default destination
|
|
861
864
|
celilo storage set-default local-backups
|
|
862
865
|
|
|
866
|
+
# Move a local destination (archives at the old path are moved too;
|
|
867
|
+
# if that path is gone, the change proceeds with nothing to migrate)
|
|
868
|
+
celilo storage set-path local-backups /var/lib/celilo/backups
|
|
869
|
+
|
|
863
870
|
# Remove storage
|
|
864
871
|
celilo storage remove local-backups --force
|
|
865
872
|
|
|
@@ -1836,6 +1843,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1836
1843
|
return handleStorageSetDefault(parsed.args, parsed.flags);
|
|
1837
1844
|
}
|
|
1838
1845
|
|
|
1846
|
+
if (parsed.subcommand === 'set-path') {
|
|
1847
|
+
const { handleStorageSetPath } = await import('./commands/storage-set-path');
|
|
1848
|
+
return handleStorageSetPath(parsed.args, parsed.flags);
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1839
1851
|
return {
|
|
1840
1852
|
success: false,
|
|
1841
1853
|
error: `Unknown storage subcommand: ${parsed.subcommand}\n\nRun "celilo storage --help" for usage`,
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
|
-
import type { DriftFinding, SystemAuditReport } from '../../services/audit/types';
|
|
2
|
+
import type { DriftCategory, DriftFinding, SystemAuditReport } from '../../services/audit/types';
|
|
3
3
|
import {
|
|
4
|
+
ALL_CATEGORIES,
|
|
4
5
|
type AuditTuiState,
|
|
6
|
+
CATEGORY_LABELS,
|
|
5
7
|
groupFindings,
|
|
6
8
|
initState,
|
|
7
9
|
reducer,
|
|
@@ -244,3 +246,15 @@ describe('selectedFinding', () => {
|
|
|
244
246
|
expect(selectedFinding(initState(report([])))).toBeNull();
|
|
245
247
|
});
|
|
246
248
|
});
|
|
249
|
+
|
|
250
|
+
// ALL_CATEGORIES is a plain array, so the type system cannot require every
|
|
251
|
+
// DriftCategory to appear in it — a new category compiles fine and then never
|
|
252
|
+
// shows up in the TUI. CATEGORY_LABELS is a Record and IS exhaustive, so it is
|
|
253
|
+
// the honest source of truth to compare against.
|
|
254
|
+
describe('ALL_CATEGORIES covers every category', () => {
|
|
255
|
+
test('every labelled category is listed, and vice versa', () => {
|
|
256
|
+
expect([...ALL_CATEGORIES].sort()).toEqual(
|
|
257
|
+
(Object.keys(CATEGORY_LABELS) as DriftCategory[]).sort(),
|
|
258
|
+
);
|
|
259
|
+
});
|
|
260
|
+
});
|
|
@@ -83,6 +83,7 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
|
|
|
83
83
|
'secrets_decryptable',
|
|
84
84
|
'services_reachable',
|
|
85
85
|
'machines_reachable',
|
|
86
|
+
'transport_reads',
|
|
86
87
|
'trusted_sources',
|
|
87
88
|
];
|
|
88
89
|
|
|
@@ -101,6 +102,7 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
|
|
|
101
102
|
secrets_decryptable: 'Secrets',
|
|
102
103
|
services_reachable: 'Service reachability',
|
|
103
104
|
machines_reachable: 'Machine reachability',
|
|
105
|
+
transport_reads: 'Transport readability',
|
|
104
106
|
trusted_sources: 'Trusted networks',
|
|
105
107
|
};
|
|
106
108
|
|
|
@@ -24,6 +24,7 @@ import { type FailingKey, builtinAlertKey } from './keys';
|
|
|
24
24
|
*/
|
|
25
25
|
const TARGET_KIND_BY_CATEGORY: Partial<Record<DriftCategory, string>> = {
|
|
26
26
|
machines_reachable: 'machine',
|
|
27
|
+
transport_reads: 'module',
|
|
27
28
|
services_reachable: 'service',
|
|
28
29
|
services_credentials: 'service',
|
|
29
30
|
backups: 'module',
|
|
@@ -7,7 +7,12 @@ import { eq } from 'drizzle-orm';
|
|
|
7
7
|
import type { DbClient } from '../../db/client';
|
|
8
8
|
import { type Route, alerts, modules, monitors, notificationDeliveries } from '../../db/schema';
|
|
9
9
|
import { setupTestDatabase } from '../../test-utils/setup-test-db';
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
type InboundPollDeps,
|
|
12
|
+
type TransportReadRecord,
|
|
13
|
+
pollInbound,
|
|
14
|
+
transportsWithRoutes,
|
|
15
|
+
} from './inbound-poller';
|
|
11
16
|
import { moduleCheckAlertKey } from './keys';
|
|
12
17
|
import { createPerson, createRoute } from './people';
|
|
13
18
|
import { mintDelivery } from './tokens';
|
|
@@ -386,6 +391,63 @@ describe('pollInbound', () => {
|
|
|
386
391
|
expect(ackedBy()).toBe(peterRoute.personId);
|
|
387
392
|
});
|
|
388
393
|
|
|
394
|
+
// The gap this closes. The ONLY per-transport state celilo persisted was the
|
|
395
|
+
// cursor, and writeCursor early-returns when there is no cursor — which a
|
|
396
|
+
// failed read never produces. So the store could not REPRESENT a failure, and
|
|
397
|
+
// "unreadable for a week" and "nobody replied for a week" left identical
|
|
398
|
+
// traces (#501).
|
|
399
|
+
describe('recording what each read attempt produced', () => {
|
|
400
|
+
const records: Array<[string, TransportReadRecord]> = [];
|
|
401
|
+
const recording = (over: Partial<InboundPollDeps> = {}) =>
|
|
402
|
+
deps([], { recordRead: (t, r) => records.push([t, r]), ...over });
|
|
403
|
+
|
|
404
|
+
beforeEach(() => {
|
|
405
|
+
records.length = 0;
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
test('a FAILED read is recorded — the case the cursor could never express', async () => {
|
|
409
|
+
await pollInbound(
|
|
410
|
+
db,
|
|
411
|
+
recording({ receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
|
|
412
|
+
);
|
|
413
|
+
expect(records).toHaveLength(1);
|
|
414
|
+
expect(records[0][0]).toBe('signal');
|
|
415
|
+
expect(records[0][1]).toMatchObject({ outcome: 'failed', error: 'connection refused' });
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
test('a successful read is recorded with how many messages it returned', async () => {
|
|
419
|
+
await pollInbound(
|
|
420
|
+
db,
|
|
421
|
+
deps([inbound(PETER, 'hello')], { recordRead: (t, r) => records.push([t, r]) }),
|
|
422
|
+
);
|
|
423
|
+
expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 1 });
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
// Zero messages is not failure. Conflating them is the original bug.
|
|
427
|
+
test('an empty but successful read records received, not failed', async () => {
|
|
428
|
+
await pollInbound(db, recording());
|
|
429
|
+
expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 0 });
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
test('a unidirectional transport is recorded as such', async () => {
|
|
433
|
+
await pollInbound(db, recording({ receiveFrom: async () => ({ status: 'unidirectional' }) }));
|
|
434
|
+
expect(records[0][1].outcome).toBe('unidirectional');
|
|
435
|
+
});
|
|
436
|
+
|
|
437
|
+
// Every attempt, not just interesting ones — a gap in the record would be
|
|
438
|
+
// read as "nothing happened".
|
|
439
|
+
test('every attempt is recorded, and carries when it happened', async () => {
|
|
440
|
+
await pollInbound(db, recording());
|
|
441
|
+
expect(records[0][1].at).toBe(NOW.toISOString());
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
// Optional dep: a caller that does not persist must still poll.
|
|
445
|
+
test('polling works with no recorder attached', async () => {
|
|
446
|
+
const report = await pollInbound(db, deps([]));
|
|
447
|
+
expect(report.transportsPolled).toBe(1);
|
|
448
|
+
});
|
|
449
|
+
});
|
|
450
|
+
|
|
389
451
|
test('a route pointing at a transport nobody uses is not polled', () => {
|
|
390
452
|
db.delete(alerts).run();
|
|
391
453
|
expect(transportsWithRoutes(db)).toEqual(['signal']);
|
|
@@ -44,6 +44,35 @@ export interface TransportFailure {
|
|
|
44
44
|
error: string;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* The outcome of one attempt to read a transport, as recorded for later.
|
|
49
|
+
*
|
|
50
|
+
* Written on EVERY attempt, including failures — which is the whole point. The
|
|
51
|
+
* only per-transport state celilo persisted before this was the cursor, and a
|
|
52
|
+
* failed read produces no cursor, so `writeCursor` returned early and nothing
|
|
53
|
+
* was written. The store could not REPRESENT a failure, so an absence of
|
|
54
|
+
* recorded failures was never evidence there had been none: a transport that
|
|
55
|
+
* had not been readable for a week looked identical to one nobody had replied
|
|
56
|
+
* on (#501).
|
|
57
|
+
*/
|
|
58
|
+
export interface TransportReadRecord {
|
|
59
|
+
/** When the attempt happened, ISO-8601. */
|
|
60
|
+
at: string;
|
|
61
|
+
outcome: 'received' | 'unidirectional' | 'failed';
|
|
62
|
+
/** Present only when `failed`. */
|
|
63
|
+
error?: string;
|
|
64
|
+
/** How many messages the read returned. Zero is not the same as failure. */
|
|
65
|
+
messages: number;
|
|
66
|
+
/**
|
|
67
|
+
* When a read last SUCCEEDED, carried forward across failures.
|
|
68
|
+
*
|
|
69
|
+
* This is the field that answers the question the old state could not: a
|
|
70
|
+
* transport reporting "0 messages" for a week and one that has not been
|
|
71
|
+
* readable for a week are the same picture until you can see this.
|
|
72
|
+
*/
|
|
73
|
+
lastSuccessAt?: string;
|
|
74
|
+
}
|
|
75
|
+
|
|
47
76
|
/** A message that was read but not acted on, and why. */
|
|
48
77
|
export interface UnheardMessage {
|
|
49
78
|
senderAddress: string;
|
|
@@ -65,6 +94,13 @@ export interface InboundPollDeps {
|
|
|
65
94
|
/** Persisted receive cursor per transport. */
|
|
66
95
|
readCursor(transportModuleId: string): string | null;
|
|
67
96
|
writeCursor(transportModuleId: string, cursor: string | null): void;
|
|
97
|
+
/**
|
|
98
|
+
* Record what one read attempt produced. Called for EVERY attempt — a failed
|
|
99
|
+
* read must leave a trace, or "we could not read this transport" stays
|
|
100
|
+
* indistinguishable from "nobody replied". Optional so a caller that does not
|
|
101
|
+
* care (tests, one-off invocations) need not supply it.
|
|
102
|
+
*/
|
|
103
|
+
recordRead?(transportModuleId: string, record: TransportReadRecord): void;
|
|
68
104
|
now(): Date;
|
|
69
105
|
/**
|
|
70
106
|
* Transport for a route, so an ack can be broadcast to everyone else paged.
|
|
@@ -131,6 +167,12 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
|
|
|
131
167
|
for (const transportId of transportsWithRoutes(db)) {
|
|
132
168
|
const received = await deps.receiveFrom(transportId, deps.readCursor(transportId));
|
|
133
169
|
report.transportsPolled++;
|
|
170
|
+
deps.recordRead?.(transportId, {
|
|
171
|
+
at: deps.now().toISOString(),
|
|
172
|
+
outcome: received.status,
|
|
173
|
+
...(received.status === 'failed' ? { error: received.error } : {}),
|
|
174
|
+
messages: received.status === 'received' ? received.messages.length : 0,
|
|
175
|
+
});
|
|
134
176
|
// One dead transport must not stop the others being read — but it is
|
|
135
177
|
// RECORDED rather than skipped in silence, because "cannot read" and
|
|
136
178
|
// "nothing to read" are the two things an operator most needs to tell
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The per-transport record of whether celilo can still READ replies.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from the poll command because it now has two readers: the poller
|
|
5
|
+
* writes it, and the audit asserts on it. Keeping the key format private to the
|
|
6
|
+
* writer would have meant the audit re-deriving a string literal — the sort of
|
|
7
|
+
* duplication that silently stops matching.
|
|
8
|
+
*
|
|
9
|
+
* The shape it stores is the point. celilo's only per-transport state used to
|
|
10
|
+
* be the receive cursor, and `writeCursor` returns early when there is no
|
|
11
|
+
* cursor — which a failed read never produces. So the store could not
|
|
12
|
+
* REPRESENT a failure, and "unreadable since Tuesday" left exactly the same
|
|
13
|
+
* trace as "nobody replied since Tuesday" (#501).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { eq } from 'drizzle-orm';
|
|
17
|
+
import type { DbClient } from '../../db/client';
|
|
18
|
+
import { systemConfig } from '../../db/schema';
|
|
19
|
+
import { type TransportReadRecord, transportsWithRoutes } from './inbound-poller';
|
|
20
|
+
|
|
21
|
+
const READ_PREFIX = 'alerting.last_read.';
|
|
22
|
+
|
|
23
|
+
export function readLastRead(db: DbClient, transportModuleId: string): TransportReadRecord | null {
|
|
24
|
+
const row = db
|
|
25
|
+
.select()
|
|
26
|
+
.from(systemConfig)
|
|
27
|
+
.where(eq(systemConfig.key, `${READ_PREFIX}${transportModuleId}`))
|
|
28
|
+
.get();
|
|
29
|
+
if (!row?.value) return null;
|
|
30
|
+
try {
|
|
31
|
+
return JSON.parse(row.value) as TransportReadRecord;
|
|
32
|
+
} catch {
|
|
33
|
+
// A malformed row must not read as "never succeeded" — that would invent a
|
|
34
|
+
// fact and page someone about it.
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Record one attempt, carrying the last SUCCESS forward.
|
|
41
|
+
*
|
|
42
|
+
* Deliberately has no counterpart to `writeCursor`'s early return: a failure
|
|
43
|
+
* that writes nothing is why this record did not exist before. Carrying the
|
|
44
|
+
* success forward is what turns a log into an answer — a run of failures must
|
|
45
|
+
* not erase when the transport last actually worked.
|
|
46
|
+
*/
|
|
47
|
+
export function writeLastRead(
|
|
48
|
+
db: DbClient,
|
|
49
|
+
transportModuleId: string,
|
|
50
|
+
record: TransportReadRecord,
|
|
51
|
+
): void {
|
|
52
|
+
const key = `${READ_PREFIX}${transportModuleId}`;
|
|
53
|
+
const previous = readLastRead(db, transportModuleId);
|
|
54
|
+
const lastSuccessAt =
|
|
55
|
+
record.outcome === 'received' ? record.at : (previous?.lastSuccessAt ?? undefined);
|
|
56
|
+
const value = JSON.stringify({ ...record, ...(lastSuccessAt ? { lastSuccessAt } : {}) });
|
|
57
|
+
|
|
58
|
+
const existing = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
|
|
59
|
+
if (existing) {
|
|
60
|
+
db.update(systemConfig).set({ value }).where(eq(systemConfig.key, key)).run();
|
|
61
|
+
} else {
|
|
62
|
+
db.insert(systemConfig)
|
|
63
|
+
.values({ key, value, description: `Last inbound read attempt for ${transportModuleId}` })
|
|
64
|
+
.run();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface TransportReadStatus {
|
|
69
|
+
transportModuleId: string;
|
|
70
|
+
last: TransportReadRecord | null;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Read state for every transport that has a route pointing at it.
|
|
75
|
+
*
|
|
76
|
+
* Scoped to transports with routes on purpose: a transport nobody is routed to
|
|
77
|
+
* cannot fail to deliver anyone's acknowledgement, and paging about it would be
|
|
78
|
+
* noise that trains an operator to ignore this check.
|
|
79
|
+
*/
|
|
80
|
+
export function readAllTransportStatuses(db: DbClient): TransportReadStatus[] {
|
|
81
|
+
return transportsWithRoutes(db).map((transportModuleId) => ({
|
|
82
|
+
transportModuleId,
|
|
83
|
+
last: readLastRead(db, transportModuleId),
|
|
84
|
+
}));
|
|
85
|
+
}
|
|
@@ -32,6 +32,7 @@ const emptyDeps = {
|
|
|
32
32
|
secretsDecryptable: { results: [] },
|
|
33
33
|
servicesReachable: { results: [] },
|
|
34
34
|
machinesReachable: { results: [] },
|
|
35
|
+
transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
|
|
35
36
|
trustedSources: { firewalls: [] },
|
|
36
37
|
};
|
|
37
38
|
|
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
} from './services-credentials';
|
|
26
26
|
import { type ServicesReachableAuditDeps, auditServicesReachable } from './services-reachable';
|
|
27
27
|
import { type TerraformPlanAuditDeps, auditTerraformPlan } from './terraform-plan';
|
|
28
|
+
import { type TransportReadsAuditDeps, auditTransportReads } from './transport-reads';
|
|
28
29
|
import { type TrustedSourcesAuditDeps, auditTrustedSources } from './trusted-sources';
|
|
29
30
|
import {
|
|
30
31
|
type DriftCategory,
|
|
@@ -53,6 +54,7 @@ export interface AuditDeps {
|
|
|
53
54
|
secretsDecryptable: SecretsDecryptableAuditDeps;
|
|
54
55
|
servicesReachable: ServicesReachableAuditDeps;
|
|
55
56
|
machinesReachable: MachinesReachableAuditDeps;
|
|
57
|
+
transportReads: TransportReadsAuditDeps;
|
|
56
58
|
trustedSources: TrustedSourcesAuditDeps;
|
|
57
59
|
/** Defaults to `Date.now()`-based ISO string. */
|
|
58
60
|
now?: () => Date;
|
|
@@ -103,6 +105,7 @@ export async function runAudit(
|
|
|
103
105
|
wrap('secrets_decryptable', auditSecretsDecryptable(deps.secretsDecryptable)),
|
|
104
106
|
wrap('services_reachable', auditServicesReachable(deps.servicesReachable)),
|
|
105
107
|
wrap('machines_reachable', auditMachinesReachable(deps.machinesReachable)),
|
|
108
|
+
wrap('transport_reads', auditTransportReads(deps.transportReads)),
|
|
106
109
|
wrap('trusted_sources', auditTrustedSources(deps.trustedSources)),
|
|
107
110
|
]);
|
|
108
111
|
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import type { TransportReadStatus } from '../alerting/read-records';
|
|
3
|
+
import { auditTransportReads } from './transport-reads';
|
|
4
|
+
|
|
5
|
+
const NOW = new Date('2026-08-01T12:00:00Z');
|
|
6
|
+
const STALE_AFTER = 30 * 60_000;
|
|
7
|
+
|
|
8
|
+
const ago = (ms: number) => new Date(NOW.getTime() - ms).toISOString();
|
|
9
|
+
|
|
10
|
+
function status(over: Partial<TransportReadStatus['last']> | null): TransportReadStatus {
|
|
11
|
+
return {
|
|
12
|
+
transportModuleId: 'signal',
|
|
13
|
+
last:
|
|
14
|
+
over === null
|
|
15
|
+
? null
|
|
16
|
+
: { at: ago(0), outcome: 'received', messages: 0, lastSuccessAt: ago(0), ...over },
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const run = (statuses: TransportReadStatus[]) =>
|
|
21
|
+
auditTransportReads({ statuses, now: NOW, staleAfterMs: STALE_AFTER });
|
|
22
|
+
|
|
23
|
+
describe('auditTransportReads', () => {
|
|
24
|
+
test('a transport read successfully just now is not a finding', async () => {
|
|
25
|
+
expect(await run([status({})])).toEqual([]);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
// THE case this check exists for. Six tokens were issued and zero consumed
|
|
29
|
+
// over a week, and nothing anywhere was red (#501).
|
|
30
|
+
test('a transport not read successfully for hours is drift', async () => {
|
|
31
|
+
const findings = await run([
|
|
32
|
+
status({
|
|
33
|
+
at: ago(60_000),
|
|
34
|
+
outcome: 'failed',
|
|
35
|
+
error: 'refused',
|
|
36
|
+
lastSuccessAt: ago(3 * 3600_000),
|
|
37
|
+
}),
|
|
38
|
+
]);
|
|
39
|
+
expect(findings).toHaveLength(1);
|
|
40
|
+
expect(findings[0]).toMatchObject({
|
|
41
|
+
category: 'transport_reads',
|
|
42
|
+
code: 'transport_reads_stale',
|
|
43
|
+
severity: 'drift',
|
|
44
|
+
subject: 'signal',
|
|
45
|
+
});
|
|
46
|
+
expect(findings[0].message).toContain('3h ago');
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// The decision that makes staleness safe to page on: an EMPTY read is a
|
|
50
|
+
// success, so a quiet transport keeps refreshing lastSuccessAt. If empty
|
|
51
|
+
// reads counted as failure this check would page on every quiet afternoon
|
|
52
|
+
// and be switched off within a week.
|
|
53
|
+
test('a quiet transport — successful reads, zero messages — is NOT stale', async () => {
|
|
54
|
+
const findings = await run([
|
|
55
|
+
status({ at: ago(0), outcome: 'received', messages: 0, lastSuccessAt: ago(0) }),
|
|
56
|
+
]);
|
|
57
|
+
expect(findings).toEqual([]);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test('a transport that has never succeeded is drift, even if attempts are recent', async () => {
|
|
61
|
+
const findings = await run([
|
|
62
|
+
status({
|
|
63
|
+
at: ago(0),
|
|
64
|
+
outcome: 'failed',
|
|
65
|
+
error: 'connection refused',
|
|
66
|
+
lastSuccessAt: undefined,
|
|
67
|
+
}),
|
|
68
|
+
]);
|
|
69
|
+
expect(findings[0]).toMatchObject({ code: 'transport_never_read', severity: 'drift' });
|
|
70
|
+
expect(findings[0].details).toContain('connection refused');
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test('a transport with no record at all is drift, not silently fine', async () => {
|
|
74
|
+
const findings = await run([status(null)]);
|
|
75
|
+
expect(findings[0]).toMatchObject({ code: 'transport_never_polled', severity: 'drift' });
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
// A transport with no `receive` is working as designed. Paging about it
|
|
79
|
+
// would be paging about a healthy system, which teaches operators to ignore
|
|
80
|
+
// the check.
|
|
81
|
+
test('a unidirectional transport is never a finding', async () => {
|
|
82
|
+
const findings = await run([
|
|
83
|
+
status({ at: ago(10 * 3600_000), outcome: 'unidirectional', lastSuccessAt: undefined }),
|
|
84
|
+
]);
|
|
85
|
+
expect(findings).toEqual([]);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test('just inside the threshold is not yet drift', async () => {
|
|
89
|
+
expect(await run([status({ lastSuccessAt: ago(STALE_AFTER - 1_000) })])).toEqual([]);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
test('just outside the threshold is drift', async () => {
|
|
93
|
+
const findings = await run([status({ lastSuccessAt: ago(STALE_AFTER + 1_000) })]);
|
|
94
|
+
expect(findings).toHaveLength(1);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
// The remediation must point at something that does not consume the queue.
|
|
98
|
+
// `celilo alerts poll` would READ, and a suggestion that eats the operator's
|
|
99
|
+
// acknowledgement is worse than no suggestion (#541).
|
|
100
|
+
test('a stale finding sends you to the journal, not to a read', async () => {
|
|
101
|
+
const findings = await run([status({ lastSuccessAt: ago(3 * 3600_000) })]);
|
|
102
|
+
expect(findings[0].remediation).toBe('celilo module journal signal');
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test('each transport is judged on its own record', async () => {
|
|
106
|
+
const findings = await run([
|
|
107
|
+
{ transportModuleId: 'signal', last: status({}).last },
|
|
108
|
+
{ transportModuleId: 'sms', last: status({ lastSuccessAt: ago(9 * 3600_000) }).last },
|
|
109
|
+
]);
|
|
110
|
+
expect(findings).toHaveLength(1);
|
|
111
|
+
expect(findings[0].subject).toBe('sms');
|
|
112
|
+
});
|
|
113
|
+
});
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transport-reads check — is celilo still able to READ replies?
|
|
3
|
+
*
|
|
4
|
+
* Every other signal that a notification transport is working proves the wrong
|
|
5
|
+
* half. `send` succeeding proves outbound. A daemon answering its API proves a
|
|
6
|
+
* socket. Neither can see the return leg fail, and for a week none of them did:
|
|
7
|
+
* six alert tokens were issued, zero were consumed, and nothing anywhere was
|
|
8
|
+
* red (#501).
|
|
9
|
+
*
|
|
10
|
+
* This is the check that would have caught it. It asserts the contract —
|
|
11
|
+
* celilo can read this transport — using first-hand evidence rather than a
|
|
12
|
+
* proxy: the recorded outcome of the reads the poller already performs every
|
|
13
|
+
* few minutes.
|
|
14
|
+
*
|
|
15
|
+
* WHY STALENESS IS A SOUND SIGNAL HERE, and it rests on one decision:
|
|
16
|
+
* an empty read is recorded as a SUCCESS. A transport nobody has replied on
|
|
17
|
+
* still records a successful read on every poll. So "no successful read in a
|
|
18
|
+
* while" cannot mean "quiet" — it can only mean the reads stopped happening or
|
|
19
|
+
* stopped working. Had zero-messages been recorded as failure, this check would
|
|
20
|
+
* page every time an operator had a peaceful afternoon, and would be turned off
|
|
21
|
+
* within a week.
|
|
22
|
+
*
|
|
23
|
+
* Deliberately consumes pre-computed state rather than reading the store
|
|
24
|
+
* itself, so it stays unit-testable and cannot become a second thing that
|
|
25
|
+
* performs reads. A check that drained the queue to find out whether the queue
|
|
26
|
+
* could be drained would eat the acknowledgement it was protecting (#541).
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import type { TransportReadStatus } from '../alerting/read-records';
|
|
30
|
+
import type { DriftFinding } from './types';
|
|
31
|
+
|
|
32
|
+
export type { TransportReadStatus };
|
|
33
|
+
|
|
34
|
+
export interface TransportReadsAuditDeps {
|
|
35
|
+
/** One entry per transport that has at least one route pointing at it. */
|
|
36
|
+
statuses: TransportReadStatus[];
|
|
37
|
+
now: Date;
|
|
38
|
+
/**
|
|
39
|
+
* How long without a successful read before it counts as drift.
|
|
40
|
+
*
|
|
41
|
+
* The poller runs every five minutes, so this is a multiple of that rather
|
|
42
|
+
* than a guess: it has to absorb a missed tick, a slow sweep, and a restart
|
|
43
|
+
* without crying wolf, while still catching a transport that has genuinely
|
|
44
|
+
* stopped being readable.
|
|
45
|
+
*/
|
|
46
|
+
staleAfterMs: number;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const CATEGORY = 'transport_reads' as const;
|
|
50
|
+
|
|
51
|
+
function describeAge(ms: number): string {
|
|
52
|
+
const mins = Math.floor(ms / 60_000);
|
|
53
|
+
if (mins < 60) return `${mins}m`;
|
|
54
|
+
const hours = Math.floor(mins / 60);
|
|
55
|
+
return hours < 48 ? `${hours}h` : `${Math.floor(hours / 24)}d`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function auditTransportReads(deps: TransportReadsAuditDeps): Promise<DriftFinding[]> {
|
|
59
|
+
const findings: DriftFinding[] = [];
|
|
60
|
+
|
|
61
|
+
for (const status of deps.statuses) {
|
|
62
|
+
const id = status.transportModuleId;
|
|
63
|
+
|
|
64
|
+
// Nothing recorded at all. Either the poller has never run, or this
|
|
65
|
+
// transport was added and never polled. Both mean no reply from here has
|
|
66
|
+
// ever been collectable, which is worth saying out loud rather than
|
|
67
|
+
// treating an empty store as "fine so far".
|
|
68
|
+
if (!status.last) {
|
|
69
|
+
findings.push({
|
|
70
|
+
category: CATEGORY,
|
|
71
|
+
severity: 'drift',
|
|
72
|
+
code: 'transport_never_polled',
|
|
73
|
+
message: `${id}: celilo has never recorded a read attempt`,
|
|
74
|
+
details:
|
|
75
|
+
'No inbound read has been attempted for this transport, so a\n' +
|
|
76
|
+
'reply sent to it would not be collected. This is the state a\n' +
|
|
77
|
+
'newly-added transport is in until the first poll runs.',
|
|
78
|
+
remediation: 'celilo alerts poll',
|
|
79
|
+
actionable: true,
|
|
80
|
+
subject: id,
|
|
81
|
+
});
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// A transport with no `receive` is unidirectional BY DESIGN — pages go out,
|
|
86
|
+
// replies were never possible, and the route's can_ack already records
|
|
87
|
+
// that. Flagging it would be flagging a working system.
|
|
88
|
+
if (status.last.outcome === 'unidirectional') continue;
|
|
89
|
+
|
|
90
|
+
if (!status.last.lastSuccessAt) {
|
|
91
|
+
findings.push({
|
|
92
|
+
category: CATEGORY,
|
|
93
|
+
severity: 'drift',
|
|
94
|
+
code: 'transport_never_read',
|
|
95
|
+
message: `${id}: no read has ever SUCCEEDED`,
|
|
96
|
+
details: `Reads have been attempted — the most recent was ${status.last.outcome}${status.last.error ? ` (${status.last.error})` : ''} — but none has\never succeeded. Acknowledgements sent to this transport cannot be\ncollected, and outbound paging will keep working, so nothing else\nwill report this.`,
|
|
97
|
+
remediation: `celilo module journal ${id}`,
|
|
98
|
+
actionable: true,
|
|
99
|
+
subject: id,
|
|
100
|
+
});
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const age = deps.now.getTime() - new Date(status.last.lastSuccessAt).getTime();
|
|
105
|
+
if (age > deps.staleAfterMs) {
|
|
106
|
+
findings.push({
|
|
107
|
+
category: CATEGORY,
|
|
108
|
+
severity: 'drift',
|
|
109
|
+
code: 'transport_reads_stale',
|
|
110
|
+
message: `${id}: last successful read ${describeAge(age)} ago`,
|
|
111
|
+
details: `The most recent attempt was ${status.last.outcome}${status.last.error ? `: ${status.last.error}` : ''}.\nAn empty read still counts as a success, so this is not "nobody\nreplied" — reads are either not happening or not working, and a\nreply sent now would not be collected.`,
|
|
112
|
+
remediation: `celilo module journal ${id}`,
|
|
113
|
+
actionable: true,
|
|
114
|
+
subject: id,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return findings;
|
|
120
|
+
}
|
|
@@ -104,6 +104,35 @@ export async function addBackupStorage(params: {
|
|
|
104
104
|
};
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
+
/**
|
|
108
|
+
* Replace a storage destination's credentials.
|
|
109
|
+
*
|
|
110
|
+
* Clears the verification stamp in the same statement. A `verified`
|
|
111
|
+
* flag describes the destination the credentials pointed at; once they
|
|
112
|
+
* change it describes somewhere else, and carrying it forward is how
|
|
113
|
+
* celilo-mgr ended up reporting `✓ Verified` for a macOS path on a
|
|
114
|
+
* Linux host (#566). Callers re-verify against the new destination.
|
|
115
|
+
*/
|
|
116
|
+
export async function updateStorageCredentials(
|
|
117
|
+
id: string,
|
|
118
|
+
credentials: Record<string, unknown>,
|
|
119
|
+
): Promise<void> {
|
|
120
|
+
const masterKey = await getOrCreateMasterKey();
|
|
121
|
+
const encrypted = encryptSecret(JSON.stringify(credentials), masterKey);
|
|
122
|
+
|
|
123
|
+
getDb()
|
|
124
|
+
.update(backupStorages)
|
|
125
|
+
.set({
|
|
126
|
+
credentialsEncrypted: JSON.stringify(encrypted),
|
|
127
|
+
verified: false,
|
|
128
|
+
verifiedAt: null,
|
|
129
|
+
verificationError: null,
|
|
130
|
+
updatedAt: new Date(),
|
|
131
|
+
})
|
|
132
|
+
.where(eq(backupStorages.id, id))
|
|
133
|
+
.run();
|
|
134
|
+
}
|
|
135
|
+
|
|
107
136
|
/**
|
|
108
137
|
* Get backup storage by storage ID (user-facing identifier)
|
|
109
138
|
*/
|
|
@@ -14,7 +14,8 @@ import {
|
|
|
14
14
|
import { dirname, join, relative } from 'node:path';
|
|
15
15
|
import type { StorageProvider, StorageVerifyResult } from './types';
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
/** Subdirectory of the configured path that actually holds archives. */
|
|
18
|
+
export const BACKUP_PREFIX = 'celilo-backups';
|
|
18
19
|
|
|
19
20
|
export interface LocalStorageConfig {
|
|
20
21
|
path: string;
|
|
@@ -82,6 +82,7 @@ const cleanAudit: AuditDeps = {
|
|
|
82
82
|
secretsDecryptable: { results: [] },
|
|
83
83
|
servicesReachable: { results: [] },
|
|
84
84
|
machinesReachable: { results: [] },
|
|
85
|
+
transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
|
|
85
86
|
trustedSources: { firewalls: [] },
|
|
86
87
|
};
|
|
87
88
|
|