@celilo/cli 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CELILO_CORE_MODULES.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +27 -7
  3. package/package.json +6 -5
  4. package/src/cli/commands/alerts-act.ts +1 -1
  5. package/src/cli/commands/backup-create.ts +26 -11
  6. package/src/cli/commands/backup-list.test.ts +83 -0
  7. package/src/cli/commands/backup-list.ts +67 -3
  8. package/src/cli/commands/backup-prune.ts +17 -17
  9. package/src/cli/commands/backup-sweep.ts +20 -8
  10. package/src/cli/commands/firewall-interface-list.test.ts +85 -0
  11. package/src/cli/commands/firewall-interface-list.ts +123 -0
  12. package/src/cli/commands/machine-add.ts +30 -2
  13. package/src/cli/commands/module-config.test.ts +64 -2
  14. package/src/cli/commands/module-config.ts +159 -8
  15. package/src/cli/commands/module-status.ts +124 -0
  16. package/src/cli/commands/monitor.ts +116 -19
  17. package/src/cli/commands/system-migrate.ts +14 -0
  18. package/src/cli/commands/system-update.ts +4 -1
  19. package/src/cli/completion.ts +35 -9
  20. package/src/cli/index.ts +59 -2
  21. package/src/cli/tui/audit-state.ts +2 -0
  22. package/src/hooks/capability-loader.ts +130 -4
  23. package/src/hooks/types.ts +2 -1
  24. package/src/manifest/contracts/v1.ts +16 -0
  25. package/src/manifest/schema.ts +40 -65
  26. package/src/services/alerting/builtin-monitors.test.ts +18 -10
  27. package/src/services/alerting/cadence-migration.test.ts +155 -0
  28. package/src/services/alerting/cadence-migration.ts +90 -0
  29. package/src/services/alerting/coverage-source.ts +8 -11
  30. package/src/services/alerting/deploy-hooks.test.ts +16 -7
  31. package/src/services/alerting/deploy-hooks.ts +11 -5
  32. package/src/services/alerting/health-cadence.test.ts +58 -0
  33. package/src/services/alerting/health-cadence.ts +128 -0
  34. package/src/services/alerting/health-coverage.ts +18 -8
  35. package/src/services/alerting/monitors.ts +50 -15
  36. package/src/services/alerting/sweep-runner.test.ts +51 -3
  37. package/src/services/alerting/sweep-runner.ts +30 -7
  38. package/src/services/audit/backup-source.ts +24 -1
  39. package/src/services/audit/backups.test.ts +95 -10
  40. package/src/services/audit/backups.ts +40 -37
  41. package/src/services/audit/interface-classification.test.ts +220 -0
  42. package/src/services/audit/interface-classification.ts +167 -0
  43. package/src/services/audit/types.ts +2 -1
  44. package/src/services/backup-age-agreement.test.ts +118 -0
  45. package/src/services/backup-create.ts +36 -30
  46. package/src/services/backup-metadata.ts +52 -1
  47. package/src/services/backup-retention.test.ts +123 -0
  48. package/src/services/backup-retention.ts +66 -5
  49. package/src/services/backup-schedule.test.ts +166 -0
  50. package/src/services/backup-schedule.ts +105 -15
  51. package/src/services/backup-staging.ts +14 -1
  52. package/src/services/backup-sweep.test.ts +22 -3
  53. package/src/services/backup-sweep.ts +15 -5
  54. package/src/services/cadence.test.ts +97 -0
  55. package/src/services/cadence.ts +165 -0
  56. package/src/services/machine-detector.ts +23 -1
  57. package/src/services/module-config.ts +33 -0
  58. package/src/services/storage-providers/s3.test.ts +96 -13
  59. package/src/services/storage-providers/s3.ts +48 -15
  60. package/src/services/zone-detector.test.ts +34 -3
  61. package/src/services/zone-detector.ts +33 -13
@@ -0,0 +1,167 @@
1
+ /**
2
+ * Firewall interface classification, reported where an operator already looks.
3
+ *
4
+ * The condition that produced `fw-keeper.sh` was not that celilo lacked
5
+ * information — it was that celilo said nothing. An interface it cannot
6
+ * attribute is an unowned network segment on a firewall, and one that is never
7
+ * surfaced is discovered as an outage rather than as a finding.
8
+ *
9
+ * What this reports, per firewall:
10
+ *
11
+ * - every interface celilo cannot attribute, by name AND address;
12
+ * - a carrier-grade NAT interface, which celilo refuses to converge on;
13
+ * - an ambiguous external edge — more than one public leg, none designated;
14
+ * - a default route leaving through neither `internal` nor `external`.
15
+ *
16
+ * The last three are BLOCKING, matching what the converge does with them. An
17
+ * audit that called them drift while the converge refused would be telling the
18
+ * operator something different from what the system does.
19
+ *
20
+ * Pure, like every other checker here: it consumes a pre-computed view so it is
21
+ * unit-testable without SSH or a live firewall.
22
+ *
23
+ * openspec: firewall-interface-classification §10 (design D1, D2, D11)
24
+ */
25
+
26
+ import {
27
+ type InterfaceAddress,
28
+ type ZoneDeclaration,
29
+ classifyInterfaces,
30
+ defaultRouteFinding,
31
+ externalEdge,
32
+ isPubliclyRoutable,
33
+ subnetContains,
34
+ } from '@celilo/capabilities';
35
+ import type { DriftFinding } from './types';
36
+
37
+ /** RFC 6598. celilo refuses to converge on one — see design D2 (amended). */
38
+ const CARRIER_GRADE_NAT = '100.64.0.0/10';
39
+
40
+ export interface FirewallInterfaceView {
41
+ /**
42
+ * User-facing hostname, and the key the finding is subjected on.
43
+ *
44
+ * NOT the machine UUID: suppression resolves a machine's ancestor key from
45
+ * its hostname, so a finding subjected on a UUID produces an alert key
46
+ * suppression can never match (#596). Users never see UUIDs either.
47
+ */
48
+ hostname: string;
49
+ /** Every address the firewall holds, as read from the box. */
50
+ interfaces: InterfaceAddress[];
51
+ /** The zones this firewall declares, with the subnets that define them. */
52
+ zones: ZoneDeclaration[];
53
+ /** The interface carrying the default route, when celilo knows it. */
54
+ defaultRouteInterface?: string | null;
55
+ /** The operator's designated edge, from the firewall's `zone.external.ip`. */
56
+ designatedExternalIp?: string | null;
57
+ }
58
+
59
+ /**
60
+ * Findings for one firewall's interfaces.
61
+ *
62
+ * Ordered most-severe first so the report reads top-down: the reasons a
63
+ * converge would refuse, then the interfaces that are merely unaccounted for.
64
+ */
65
+ export function auditFirewallInterfaces(view: FirewallInterfaceView): DriftFinding[] {
66
+ const findings: DriftFinding[] = [];
67
+ const classifications = classifyInterfaces(view.interfaces, view.zones);
68
+
69
+ // ── Blocking: a carrier address celilo cannot reason about ────────────────
70
+ const carrier = view.interfaces.filter((i) => subnetContains(CARRIER_GRADE_NAT, i.ip));
71
+ for (const iface of carrier) {
72
+ findings.push({
73
+ category: 'interface_classification',
74
+ severity: 'blocked',
75
+ code: 'carrier_grade_nat_interface',
76
+ subject: view.hostname,
77
+ message: `${view.hostname}: ${iface.name} holds a carrier-grade NAT address (${iface.ip})`,
78
+ details:
79
+ "RFC 6598 space belongs to the carrier and is shared with other subscribers, so it cannot tell celilo the fleet's real external address. celilo does not know what such an interface is for, whether to NAT on it, or what reach it should have, so it refuses to converge rather than guessing. Only ISP equipment should hold one.",
80
+ remediation:
81
+ 'Remove the carrier-grade NAT interface from this firewall, or place the ISP boundary on separate equipment (celilo drives an upstream router through the firewall capability).',
82
+ actionable: false,
83
+ });
84
+ }
85
+
86
+ // ── Blocking: two public legs and no designation ──────────────────────────
87
+ const edge = externalEdge(classifications, view.designatedExternalIp);
88
+ if (edge.kind === 'ambiguous') {
89
+ const named = edge.candidates.map((c) => `${c.name} (${c.ip})`).join(', ');
90
+ findings.push({
91
+ category: 'interface_classification',
92
+ severity: 'blocked',
93
+ code: 'ambiguous_external_edge',
94
+ subject: view.hostname,
95
+ message: `${view.hostname}: ${edge.candidates.length} publicly routable interfaces and none designated — ${named}`,
96
+ details:
97
+ 'celilo will not pick an external edge by interface order. Before this check the first non-private address won silently, so a box with two public addresses had its edge chosen by enumeration with no warning and no way to override.',
98
+ remediation: `celilo module config set <firewall-module> zone.external.ip <one of: ${edge.candidates
99
+ .map((c) => c.ip)
100
+ .join(', ')}>`,
101
+ actionable: false,
102
+ });
103
+ }
104
+
105
+ // ── Blocking: egress leaving through a segment celilo cannot describe ─────
106
+ if (view.defaultRouteInterface !== undefined) {
107
+ const routeFinding = defaultRouteFinding(classifications, view.defaultRouteInterface);
108
+ if (routeFinding) {
109
+ findings.push({
110
+ category: 'interface_classification',
111
+ severity: 'blocked',
112
+ code: 'default_route_off_edge',
113
+ subject: view.hostname,
114
+ message: `${view.hostname}: ${routeFinding.message}`,
115
+ details:
116
+ 'celilo supports two shapes: a firewall that owns its WAN egresses through its external leg, and a downstream firewall egresses via an upstream router on the internal segment. Egress anywhere else leaves through a segment celilo cannot describe.',
117
+ remediation:
118
+ 'Move the default route onto the external or internal leg, or declare a zone for the interface that carries it.',
119
+ actionable: false,
120
+ });
121
+ }
122
+ }
123
+
124
+ // ── Drift: interfaces nobody accounted for ────────────────────────────────
125
+ const alien = classifications.filter((c) => c.role === 'alien');
126
+ if (alien.length > 0) {
127
+ const named = alien.map((c) => `${c.name} (${c.ip})`).join(', ');
128
+ findings.push({
129
+ category: 'interface_classification',
130
+ severity: 'drift',
131
+ code: 'alien_interfaces',
132
+ subject: view.hostname,
133
+ message: `${view.hostname}: ${alien.length} interface(s) celilo cannot attribute — ${named}`,
134
+ details:
135
+ 'An interface in no declared zone is an unowned network segment on a firewall. celilo refuses to converge a firewall it is onboarding until every interface is accounted for; on one it has already brought to a known-good state, a NEWLY appeared interface like this is disabled.',
136
+ remediation: `celilo system config set network.<zone>.subnet <cidr> # then add the zone to the firewall's declared zones`,
137
+ actionable: false,
138
+ });
139
+ }
140
+
141
+ return findings;
142
+ }
143
+
144
+ /** Every firewall's findings, flattened. */
145
+ export function auditInterfaceClassification(views: FirewallInterfaceView[]): DriftFinding[] {
146
+ return views.flatMap(auditFirewallInterfaces);
147
+ }
148
+
149
+ /**
150
+ * The classification itself, for the report body rather than as a finding.
151
+ *
152
+ * §10.2 requires a fully declared firewall to produce NO findings, so the
153
+ * per-interface roles cannot be findings — they are what an operator reads to
154
+ * confirm the picture is what they intended.
155
+ */
156
+ export function describeClassification(view: FirewallInterfaceView): string[] {
157
+ return classifyInterfaces(view.interfaces, view.zones).map((c) => {
158
+ if (c.role === 'zone') return `${c.name}: ${c.ip} → zone:${c.zone}`;
159
+ if (c.role === 'external') {
160
+ return `${c.name}: ${c.ip} → external (the WAN edge)`;
161
+ }
162
+ const why = isPubliclyRoutable(c.ip)
163
+ ? 'publicly routable but unclaimed'
164
+ : 'no declared subnet contains it';
165
+ return `${c.name}: ${c.ip} → alien (${why})`;
166
+ });
167
+ }
@@ -40,7 +40,8 @@ export type DriftCategory =
40
40
  | 'public_dns'
41
41
  | 'disk_space'
42
42
  | 'transport_reads'
43
- | 'trusted_sources';
43
+ | 'trusted_sources'
44
+ | 'interface_classification';
44
45
 
45
46
  export type DriftSeverity = 'todo' | 'drift' | 'blocked';
46
47
 
@@ -0,0 +1,118 @@
1
+ /**
2
+ * The run path and the alert path must agree about how old a backup is.
3
+ *
4
+ * They did not. `isBackupDue` measured from `backups.startedAt` while the
5
+ * freshness audit measured from `completedAt`, so the two disagreed by however
6
+ * long the backup itself took — 5.5 minutes for forgejo, and unbounded for
7
+ * anything larger. The shared accessor prevented drift where it was looking and
8
+ * the drift moved into the columns underneath it (design.md D6).
9
+ *
10
+ * This is the gate for that: one completed backup whose start and finish are
11
+ * far apart, read by both paths, which must return the same instant.
12
+ */
13
+
14
+ import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
15
+ import { mkdtempSync, rmSync } from 'node:fs';
16
+ import { tmpdir } from 'node:os';
17
+ import { join } from 'node:path';
18
+ import { closeDb, getDb } from '../db/client';
19
+ import { runMigrations } from '../db/migrate';
20
+ import { backupStorages, backups, modules } from '../db/schema';
21
+ import { loadBackupAuditInfo } from './audit/backup-source';
22
+ import { findBackupEligibleModules } from './backup-create';
23
+ import { loadBackupHistory } from './backup-metadata';
24
+ import { BACKUP_SCHEDULE_CONFIG_KEY, effectiveBackupSchedule } from './backup-schedule';
25
+ import { parseCadence } from './cadence';
26
+ import { configOverride, upsertModuleConfig } from './module-config';
27
+
28
+ const STARTED = new Date('2026-04-25T00:00:00Z');
29
+ // Six hours later — an extreme but not absurd gap for a multi-gigabyte module.
30
+ const COMPLETED = new Date('2026-04-25T06:00:00Z');
31
+
32
+ describe('backup age: the run path and the audit path read the same column', () => {
33
+ let dir: string;
34
+
35
+ beforeEach(async () => {
36
+ dir = mkdtempSync(join(tmpdir(), 'celilo-backup-age-'));
37
+ process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
38
+ await runMigrations(process.env.CELILO_DB_PATH);
39
+
40
+ const db = getDb();
41
+ db.insert(modules)
42
+ .values({
43
+ id: 'forgejo',
44
+ name: 'Forgejo',
45
+ sourcePath: dir,
46
+ version: '1.0.0',
47
+ state: 'INSTALLED',
48
+ manifestData: {
49
+ id: 'forgejo',
50
+ hooks: { on_backup: { script: './backup.ts' } },
51
+ backup: { schedule: 'daily' },
52
+ },
53
+ })
54
+ .run();
55
+ db.insert(backupStorages)
56
+ .values({
57
+ id: 'storage-1',
58
+ storageId: 'local',
59
+ name: 'Local',
60
+ providerName: 'local',
61
+ credentialsEncrypted: '',
62
+ providerConfig: {},
63
+ verified: true,
64
+ })
65
+ .run();
66
+ db.insert(backups)
67
+ .values({
68
+ id: 'backup-1',
69
+ moduleId: 'forgejo',
70
+ storageId: 'storage-1',
71
+ storagePath: '2026-04-25/forgejo.backup',
72
+ backupType: 'module_data',
73
+ status: 'completed',
74
+ startedAt: STARTED,
75
+ completedAt: COMPLETED,
76
+ })
77
+ .run();
78
+ });
79
+
80
+ afterEach(() => {
81
+ closeDb();
82
+ process.env.CELILO_DB_PATH = undefined;
83
+ rmSync(dir, { recursive: true, force: true });
84
+ });
85
+
86
+ test('both measure from when the backup COMPLETED, not when it started', () => {
87
+ const history = loadBackupHistory('forgejo');
88
+ const [audited] = loadBackupAuditInfo(getDb());
89
+
90
+ expect(history.lastSuccessAt?.getTime()).toBe(COMPLETED.getTime());
91
+ expect(audited.lastSuccessfulBackupAt).toBe(COMPLETED.getTime());
92
+ expect(history.lastSuccessAt?.getTime()).toBe(audited.lastSuccessfulBackupAt ?? 0);
93
+ });
94
+
95
+ test('both resolve the same cadence for the same module', () => {
96
+ const [audited] = loadBackupAuditInfo(getDb());
97
+ const fromAudit = effectiveBackupSchedule(audited.manifest, audited.scheduleOverride);
98
+
99
+ expect(fromAudit).toEqual({ minutes: 24 * 60 });
100
+ expect(parseCadence('daily')).toEqual(fromAudit);
101
+ });
102
+
103
+ test("an operator's override reaches the sweep and the audit alike", () => {
104
+ upsertModuleConfig(getDb(), 'forgejo', BACKUP_SCHEDULE_CONFIG_KEY, '6h');
105
+
106
+ const [audited] = loadBackupAuditInfo(getDb());
107
+ const [eligible] = findBackupEligibleModules();
108
+
109
+ const fromAudit = effectiveBackupSchedule(audited.manifest, audited.scheduleOverride);
110
+ const fromSweep = effectiveBackupSchedule(
111
+ eligible.manifest,
112
+ configOverride(eligible.configs, BACKUP_SCHEDULE_CONFIG_KEY),
113
+ );
114
+
115
+ expect(fromAudit).toEqual({ minutes: 360 });
116
+ expect(fromSweep).toEqual(fromAudit);
117
+ });
118
+ });
@@ -17,8 +17,13 @@ import { getOrCreateMasterKey } from '../secrets/master-key';
17
17
  import { shellEscape } from '../utils/shell';
18
18
  import { encryptFileToFile } from './backup-cipher';
19
19
  import { buildManifest } from './backup-manifest';
20
- import { completeBackup, createBackupRecord, failBackup, listBackups } from './backup-metadata';
21
- import type { BackupSchedule } from './backup-schedule';
20
+ import {
21
+ completeBackup,
22
+ createBackupRecord,
23
+ failBackup,
24
+ loadBackupHistory,
25
+ } from './backup-metadata';
26
+ import { isBackupDueFromHistory } from './backup-schedule';
22
27
  import { stagingDirFor } from './backup-staging';
23
28
  import {
24
29
  createStorageProvider,
@@ -26,8 +31,10 @@ import {
26
31
  getBackupStorageByStorageId,
27
32
  getDefaultBackupStorage,
28
33
  } from './backup-storage';
34
+ import type { Cadence } from './cadence';
29
35
  import { materializeCrossModuleRoot, moduleHasCrossModuleRead } from './cross-module-read';
30
36
  import { getModuleSystems } from './deployed-systems';
37
+ import { parseStoredConfigValue } from './module-config';
31
38
  import {
32
39
  completeOperation,
33
40
  failOperation,
@@ -49,8 +56,6 @@ export interface BackupCreateResult {
49
56
  error?: string;
50
57
  }
51
58
 
52
- export type { BackupSchedule } from './backup-schedule';
53
-
54
59
  /**
55
60
  * Resolve the target storage destination
56
61
  */
@@ -198,46 +203,47 @@ export async function createSystemStateBackup(
198
203
  }
199
204
 
200
205
  /**
201
- * Schedule intervals in milliseconds
202
- */
203
- const SCHEDULE_INTERVALS: Record<BackupSchedule, number> = {
204
- hourly: 60 * 60 * 1000,
205
- daily: 24 * 60 * 60 * 1000,
206
- weekly: 7 * 24 * 60 * 60 * 1000,
207
- monthly: 30 * 24 * 60 * 60 * 1000,
208
- manual: Number.POSITIVE_INFINITY,
209
- };
210
-
211
- /**
212
- * Check if a module is due for backup based on its schedule
206
+ * Check if a module is due for backup based on its schedule.
207
+ *
208
+ * The policy lives in `isBackupDueFromHistory`; this reads the history it
209
+ * needs. Worth knowing what the previous version did, because its shape was
210
+ * the bug: it asked for `limit: 1` the single most recent row, whatever its
211
+ * outcome and then `.find()`-ed a *completed* one within that. Those two
212
+ * lines contradict each other. Once an attempt failed, the newest row was that
213
+ * failure, the find matched nothing, and "no successful backup exists" meant
214
+ * unconditionally due — at every hourly tick, forever, however the module's
215
+ * cadence read (celilo#685).
213
216
  */
214
- export function isBackupDue(moduleId: string, schedule: BackupSchedule): boolean {
215
- if (schedule === 'manual') return false;
216
-
217
- const existing = listBackups({ moduleId, limit: 1 });
218
- const lastBackup = existing.find(
219
- (b) => b.moduleId === moduleId && b.status === 'completed' && b.backupType === 'module_data',
220
- );
221
-
222
- if (!lastBackup) return true;
223
-
224
- const elapsed = Date.now() - new Date(lastBackup.startedAt).getTime();
225
- return elapsed >= SCHEDULE_INTERVALS[schedule];
217
+ export function isBackupDue(moduleId: string, schedule: Cadence): boolean {
218
+ return isBackupDueFromHistory(schedule, loadBackupHistory(moduleId), Date.now());
226
219
  }
227
220
 
228
221
  /**
229
- * Find all installed modules that have an on_backup hook
222
+ * Find all installed modules that have an on_backup hook.
223
+ *
224
+ * Each carries its operator config alongside its manifest, because every
225
+ * caller then has to resolve a cadence or a retention policy out of the two
226
+ * together — and a caller that got the manifest alone would silently ignore
227
+ * the operator's override.
230
228
  */
231
229
  export function findBackupEligibleModules(): Array<{
232
230
  module: typeof modules.$inferSelect;
233
231
  manifest: ModuleManifest;
232
+ configs: Record<string, unknown>;
234
233
  }> {
235
234
  const db = getDb();
236
235
  const allModules = db.select().from(modules).all();
236
+ const configsByModule = new Map<string, Record<string, unknown>>();
237
+ for (const row of db.select().from(moduleConfigs).all()) {
238
+ const forModule = configsByModule.get(row.moduleId) ?? {};
239
+ forModule[row.key] = parseStoredConfigValue(row);
240
+ configsByModule.set(row.moduleId, forModule);
241
+ }
237
242
 
238
243
  const eligible: Array<{
239
244
  module: typeof modules.$inferSelect;
240
245
  manifest: ModuleManifest;
246
+ configs: Record<string, unknown>;
241
247
  }> = [];
242
248
 
243
249
  for (const mod of allModules) {
@@ -246,7 +252,7 @@ export function findBackupEligibleModules(): Array<{
246
252
  const manifest = mod.manifestData as unknown as ModuleManifest;
247
253
  if (!manifest.hooks?.on_backup) continue;
248
254
 
249
- eligible.push({ module: mod, manifest });
255
+ eligible.push({ module: mod, manifest, configs: configsByModule.get(mod.id) ?? {} });
250
256
  }
251
257
 
252
258
  return eligible;
@@ -4,9 +4,10 @@
4
4
  */
5
5
 
6
6
  import { randomUUID } from 'node:crypto';
7
- import { desc, eq, like } from 'drizzle-orm';
7
+ import { and, desc, eq, gt, isNotNull, like } from 'drizzle-orm';
8
8
  import { getDb } from '../db/client';
9
9
  import { type Backup, type BackupStatus, type BackupType, backups } from '../db/schema';
10
+ import type { BackupHistory } from './backup-schedule';
10
11
 
11
12
  /** Short ID length used for display and lookup */
12
13
  export const SHORT_ID_LENGTH = 8;
@@ -156,6 +157,56 @@ export function listBackups(options?: {
156
157
  return db.select().from(backups).orderBy(desc(backups.startedAt)).limit(limit).all();
157
158
  }
158
159
 
160
+ /**
161
+ * What a module's backup history says, for the due-ness decision.
162
+ *
163
+ * Only `module_data` rows count. A module's cadence is about its own data; a
164
+ * system backup that happens to name it must not make it look covered.
165
+ *
166
+ * Split from the decision itself so the policy stays pure and testable
167
+ * (Rule 2.3) — see `isBackupDueFromHistory`.
168
+ */
169
+ export function loadBackupHistory(moduleId: string): BackupHistory {
170
+ const db = getDb();
171
+ const forThisModule = and(eq(backups.moduleId, moduleId), eq(backups.backupType, 'module_data'));
172
+
173
+ // Ordered and measured by `completedAt`, matching the freshness audit
174
+ // (services/audit/backup-source.ts). Measuring due-ness from `startedAt`
175
+ // while the audit measured from `completedAt` meant the two paths could
176
+ // disagree about a module's age by the duration of the backup itself —
177
+ // design.md D6. A `completed` row with no `completedAt` is not a backup
178
+ // either path will count.
179
+ const [lastSuccess] = db
180
+ .select()
181
+ .from(backups)
182
+ .where(and(forThisModule, eq(backups.status, 'completed'), isNotNull(backups.completedAt)))
183
+ .orderBy(desc(backups.completedAt))
184
+ .limit(1)
185
+ .all();
186
+
187
+ const [lastAttempt] = db
188
+ .select()
189
+ .from(backups)
190
+ .where(forThisModule)
191
+ .orderBy(desc(backups.startedAt))
192
+ .limit(1)
193
+ .all();
194
+
195
+ // Attempts since the last success — every row newer than it, or every row at
196
+ // all when there has never been one. An `in_progress` row is included: the
197
+ // decision only ever uses this to back OFF, so counting a live attempt is
198
+ // the safe direction.
199
+ const since = lastSuccess
200
+ ? and(forThisModule, gt(backups.startedAt, lastSuccess.startedAt))
201
+ : forThisModule;
202
+
203
+ return {
204
+ lastSuccessAt: lastSuccess?.completedAt ?? null,
205
+ lastAttemptAt: lastAttempt?.startedAt ?? null,
206
+ consecutiveFailures: db.select({ id: backups.id }).from(backups).where(since).all().length,
207
+ };
208
+ }
209
+
159
210
  /**
160
211
  * Every backup record still claiming to be in progress, oldest first.
161
212
  *
@@ -0,0 +1,123 @@
1
+ /**
2
+ * Retention resolves per dimension, and an unset dimension is UNBOUNDED.
3
+ *
4
+ * The trap this file exists for: `backup.retention` is an optional block, so a
5
+ * manifest that omits it prunes nothing at all — its inner `count: 7` /
6
+ * `max_age_days: 30` defaults never apply. If setting one dimension let the
7
+ * other fall back to those defaults, an operator asking to keep 3 copies would
8
+ * silently arm a 30-day deletion on a module that had been keeping everything.
9
+ * Every other failure here is recoverable; that one destroys backups.
10
+ */
11
+
12
+ import { describe, expect, test } from 'bun:test';
13
+ import type { Backup } from '../db/schema';
14
+ import type { ModuleManifest } from '../manifest/schema';
15
+ import {
16
+ BACKUP_RETENTION_COUNT_CONFIG_KEY,
17
+ BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY,
18
+ effectiveBackupRetention,
19
+ identifyExpiredBackups,
20
+ prunesNothing,
21
+ } from './backup-retention';
22
+
23
+ const DAY = 24 * 60 * 60 * 1000;
24
+ const UNBOUNDED = Number.POSITIVE_INFINITY;
25
+
26
+ function manifestWith(retention?: { count?: number; max_age_days?: number }): ModuleManifest {
27
+ return { backup: retention ? { retention } : undefined } as unknown as ModuleManifest;
28
+ }
29
+
30
+ /** Newest first, matching what `listCompletedBackupsForModule` returns. */
31
+ function backupsAgedDays(...ages: number[]): Backup[] {
32
+ return ages.map(
33
+ (days, i) =>
34
+ ({
35
+ id: `backup-${i}`,
36
+ storagePath: `path-${i}`,
37
+ storageId: 'storage-1',
38
+ startedAt: new Date(Date.now() - days * DAY),
39
+ }) as unknown as Backup,
40
+ );
41
+ }
42
+
43
+ describe('effectiveBackupRetention', () => {
44
+ test('nothing declared and nothing set is unbounded in both dimensions', () => {
45
+ const policy = effectiveBackupRetention(manifestWith(), {});
46
+ expect(policy).toEqual({ count: UNBOUNDED, maxAgeDays: UNBOUNDED });
47
+ expect(prunesNothing(policy)).toBe(true);
48
+ });
49
+
50
+ // THE data-loss trap. Setting one dimension must not arm the other.
51
+ test('setting one dimension leaves the other unbounded when the manifest declares none', () => {
52
+ const policy = effectiveBackupRetention(manifestWith(), {
53
+ [BACKUP_RETENTION_COUNT_CONFIG_KEY]: 3,
54
+ });
55
+
56
+ expect(policy.count).toBe(3);
57
+ expect(policy.maxAgeDays).toBe(UNBOUNDED);
58
+ expect(prunesNothing(policy)).toBe(false);
59
+ });
60
+
61
+ test('an override bounds one dimension while the manifest bounds the other', () => {
62
+ const policy = effectiveBackupRetention(manifestWith({ count: 7, max_age_days: 30 }), {
63
+ [BACKUP_RETENTION_COUNT_CONFIG_KEY]: 3,
64
+ });
65
+
66
+ expect(policy).toEqual({ count: 3, maxAgeDays: 30 });
67
+ });
68
+
69
+ test('both dimensions can be overridden independently', () => {
70
+ const policy = effectiveBackupRetention(manifestWith({ count: 7, max_age_days: 30 }), {
71
+ [BACKUP_RETENTION_COUNT_CONFIG_KEY]: 2,
72
+ [BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY]: 90,
73
+ });
74
+
75
+ expect(policy).toEqual({ count: 2, maxAgeDays: 90 });
76
+ });
77
+
78
+ test("the manifest's suggestion applies when nobody has overridden", () => {
79
+ expect(effectiveBackupRetention(manifestWith({ count: 7, max_age_days: 30 }), {})).toEqual({
80
+ count: 7,
81
+ maxAgeDays: 30,
82
+ });
83
+ });
84
+
85
+ test('a non-positive or unparseable override falls back rather than deleting everything', () => {
86
+ // Validated at SET time, so this is hand-edited state. A `0` read as "keep
87
+ // zero copies" would delete every backup the module has.
88
+ for (const bad of [0, -1, 'lots', 2.5]) {
89
+ const policy = effectiveBackupRetention(manifestWith({ count: 7, max_age_days: 30 }), {
90
+ [BACKUP_RETENTION_COUNT_CONFIG_KEY]: bad,
91
+ });
92
+ expect(policy.count).toBe(7);
93
+ }
94
+ });
95
+ });
96
+
97
+ describe('identifyExpiredBackups with unbounded dimensions', () => {
98
+ test('an unbounded count never prunes on count', () => {
99
+ const expired = identifyExpiredBackups(backupsAgedDays(1, 2, 3, 4, 5), {
100
+ count: UNBOUNDED,
101
+ maxAgeDays: UNBOUNDED,
102
+ });
103
+ expect(expired).toEqual([]);
104
+ });
105
+
106
+ test('a bounded count prunes past it, and age alone prunes nothing', () => {
107
+ const expired = identifyExpiredBackups(backupsAgedDays(1, 2, 100, 200), {
108
+ count: 3,
109
+ maxAgeDays: UNBOUNDED,
110
+ });
111
+
112
+ expect(expired.map((b) => b.id)).toEqual(['backup-3']);
113
+ });
114
+
115
+ test('a bounded age prunes past it while every copy is kept on count', () => {
116
+ const expired = identifyExpiredBackups(backupsAgedDays(1, 2, 100), {
117
+ count: UNBOUNDED,
118
+ maxAgeDays: 30,
119
+ });
120
+
121
+ expect(expired.map((b) => b.id)).toEqual(['backup-2']);
122
+ });
123
+ });