@celilo/cli 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CELILO_CORE_MODULES.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +27 -7
  3. package/package.json +6 -5
  4. package/src/cli/commands/alerts-act.ts +1 -1
  5. package/src/cli/commands/backup-create.ts +26 -11
  6. package/src/cli/commands/backup-list.test.ts +83 -0
  7. package/src/cli/commands/backup-list.ts +67 -3
  8. package/src/cli/commands/backup-prune.ts +17 -17
  9. package/src/cli/commands/backup-sweep.ts +20 -8
  10. package/src/cli/commands/firewall-interface-list.test.ts +85 -0
  11. package/src/cli/commands/firewall-interface-list.ts +123 -0
  12. package/src/cli/commands/machine-add.ts +30 -2
  13. package/src/cli/commands/module-config.test.ts +64 -2
  14. package/src/cli/commands/module-config.ts +159 -8
  15. package/src/cli/commands/module-status.ts +124 -0
  16. package/src/cli/commands/monitor.ts +116 -19
  17. package/src/cli/commands/system-migrate.ts +14 -0
  18. package/src/cli/commands/system-update.ts +4 -1
  19. package/src/cli/completion.ts +35 -9
  20. package/src/cli/index.ts +59 -2
  21. package/src/cli/tui/audit-state.ts +2 -0
  22. package/src/hooks/capability-loader.ts +130 -4
  23. package/src/hooks/types.ts +2 -1
  24. package/src/manifest/contracts/v1.ts +16 -0
  25. package/src/manifest/schema.ts +40 -65
  26. package/src/services/alerting/builtin-monitors.test.ts +18 -10
  27. package/src/services/alerting/cadence-migration.test.ts +155 -0
  28. package/src/services/alerting/cadence-migration.ts +90 -0
  29. package/src/services/alerting/coverage-source.ts +8 -11
  30. package/src/services/alerting/deploy-hooks.test.ts +16 -7
  31. package/src/services/alerting/deploy-hooks.ts +11 -5
  32. package/src/services/alerting/health-cadence.test.ts +58 -0
  33. package/src/services/alerting/health-cadence.ts +128 -0
  34. package/src/services/alerting/health-coverage.ts +18 -8
  35. package/src/services/alerting/monitors.ts +50 -15
  36. package/src/services/alerting/sweep-runner.test.ts +51 -3
  37. package/src/services/alerting/sweep-runner.ts +30 -7
  38. package/src/services/audit/backup-source.ts +24 -1
  39. package/src/services/audit/backups.test.ts +95 -10
  40. package/src/services/audit/backups.ts +40 -37
  41. package/src/services/audit/interface-classification.test.ts +220 -0
  42. package/src/services/audit/interface-classification.ts +167 -0
  43. package/src/services/audit/types.ts +2 -1
  44. package/src/services/backup-age-agreement.test.ts +118 -0
  45. package/src/services/backup-create.ts +36 -30
  46. package/src/services/backup-metadata.ts +52 -1
  47. package/src/services/backup-retention.test.ts +123 -0
  48. package/src/services/backup-retention.ts +66 -5
  49. package/src/services/backup-schedule.test.ts +166 -0
  50. package/src/services/backup-schedule.ts +105 -15
  51. package/src/services/backup-staging.ts +14 -1
  52. package/src/services/backup-sweep.test.ts +22 -3
  53. package/src/services/backup-sweep.ts +15 -5
  54. package/src/services/cadence.test.ts +97 -0
  55. package/src/services/cadence.ts +165 -0
  56. package/src/services/machine-detector.ts +23 -1
  57. package/src/services/module-config.ts +33 -0
  58. package/src/services/storage-providers/s3.test.ts +96 -13
  59. package/src/services/storage-providers/s3.ts +48 -15
  60. package/src/services/zone-detector.test.ts +34 -3
  61. package/src/services/zone-detector.ts +33 -13
@@ -1,19 +1,77 @@
1
1
  /**
2
- * Backup retention service.
3
- * Enforces retention policies defined in module manifests.
4
- * Policies are count-based (keep last N) and age-based (delete older than X days).
5
- * Whichever limit is hit first triggers deletion.
2
+ * Backup retention: how many copies to keep, and for how long.
3
+ *
4
+ * Policies are count-based (keep last N) and age-based (delete older than X
5
+ * days), and the two are INDEPENDENT — whichever limit is hit first triggers
6
+ * deletion. The manifest suggests; the operator's `backup_retention_count` and
7
+ * `backup_retention_max_age_days` overrides decide, each dimension resolved on
8
+ * its own.
9
+ *
10
+ * ⚠️ An unset dimension is UNBOUNDED, never a default bound. Today an absent
11
+ * `backup.retention` block means prune nothing at all — the block is optional,
12
+ * so its inner `count: 7` / `max_age_days: 30` defaults never apply. If setting
13
+ * one dimension made the other fall back to those defaults, an operator asking
14
+ * to keep 3 copies would silently also arm a 30-day deletion they never asked
15
+ * for, on a module that had been keeping everything. Deleting backups nobody
16
+ * asked to delete is the one failure here that cannot be undone.
6
17
  */
7
18
 
8
19
  import type { Backup } from '../db/schema';
20
+ import type { ModuleManifest } from '../manifest/schema';
9
21
  import { deleteBackupRecord, listCompletedBackupsForModule } from './backup-metadata';
10
22
  import { createStorageProvider } from './backup-storage';
23
+ import { configOverride } from './module-config';
24
+
25
+ /** The `module_configs` keys an operator's retention policy is stored under. */
26
+ export const BACKUP_RETENTION_COUNT_CONFIG_KEY = 'backup_retention_count';
27
+ export const BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY = 'backup_retention_max_age_days';
11
28
 
12
29
  export interface RetentionPolicy {
30
+ /** Copies to keep. `Infinity` means unbounded — keep every copy. */
13
31
  count: number;
32
+ /** Days to keep. `Infinity` means unbounded — never delete on age. */
14
33
  maxAgeDays: number;
15
34
  }
16
35
 
36
+ /** Neither dimension bounded: nothing is ever pruned, so the pass can be skipped. */
37
+ export function prunesNothing(policy: RetentionPolicy): boolean {
38
+ return (
39
+ policy.count === Number.POSITIVE_INFINITY && policy.maxAgeDays === Number.POSITIVE_INFINITY
40
+ );
41
+ }
42
+
43
+ /**
44
+ * Takes the module's operator config as loaded from `module_configs`. An
45
+ * unparseable or non-positive override leaves that dimension to the manifest:
46
+ * values are validated at SET time, so a bad one here means hand-edited state,
47
+ * and keeping too much is the only safe direction to fail in.
48
+ */
49
+ export function effectiveBackupRetention(
50
+ manifest: ModuleManifest,
51
+ configs: Record<string, unknown> | undefined,
52
+ ): RetentionPolicy {
53
+ const declared = manifest.backup?.retention;
54
+ return {
55
+ count: positiveIntegerOr(
56
+ configOverride(configs, BACKUP_RETENTION_COUNT_CONFIG_KEY),
57
+ declared?.count,
58
+ ),
59
+ maxAgeDays: positiveIntegerOr(
60
+ configOverride(configs, BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY),
61
+ declared?.max_age_days,
62
+ ),
63
+ };
64
+ }
65
+
66
+ /** Override, else the manifest's suggestion, else unbounded. Never a default bound. */
67
+ function positiveIntegerOr(override: string | undefined, suggested: number | undefined): number {
68
+ if (override !== undefined) {
69
+ const parsed = Number(override);
70
+ if (Number.isInteger(parsed) && parsed > 0) return parsed;
71
+ }
72
+ return suggested ?? Number.POSITIVE_INFINITY;
73
+ }
74
+
17
75
  export interface PruneResult {
18
76
  moduleId: string;
19
77
  deleted: number;
@@ -21,7 +79,10 @@ export interface PruneResult {
21
79
  }
22
80
 
23
81
  /**
24
- * Identify backups that should be pruned per the retention policy
82
+ * Identify backups that should be pruned per the retention policy.
83
+ *
84
+ * An unbounded dimension is `Infinity`, which needs no special case: no index
85
+ * reaches it and no age exceeds it.
25
86
  */
26
87
  export function identifyExpiredBackups(backupsList: Backup[], policy: RetentionPolicy): Backup[] {
27
88
  const now = Date.now();
@@ -0,0 +1,166 @@
1
+ /**
2
+ * Backup cadence: what a manifest declares, and when the sweep acts on it.
3
+ *
4
+ * The due-ness tests are the interesting half. celilo#685 was a *daily* module
5
+ * attempted 24 times a day for a day, each attempt assembling ~1.9 GB before
6
+ * being OOM-killed, because due-ness was measured only from the last SUCCESS
7
+ * and a module that cannot succeed never advances that timestamp.
8
+ */
9
+
10
+ import { describe, expect, test } from 'bun:test';
11
+ import type { ModuleManifest } from '../manifest/schema';
12
+ import {
13
+ DEFAULT_BACKUP_SCHEDULE,
14
+ MAX_RAPID_RETRIES,
15
+ effectiveBackupSchedule,
16
+ isBackupDueFromHistory,
17
+ } from './backup-schedule';
18
+ import { type Cadence, parseCadence } from './cadence';
19
+
20
+ const HOUR = 60 * 60 * 1000;
21
+ const DAY = 24 * HOUR;
22
+ const NOW = Date.UTC(2026, 7, 13, 12, 0, 0);
23
+
24
+ function ago(ms: number): Date {
25
+ return new Date(NOW - ms);
26
+ }
27
+
28
+ /** Parse a cadence in a test, failing loudly rather than passing null on (Rule 7.2). */
29
+ function cadence(value: string): Cadence {
30
+ const parsed = parseCadence(value);
31
+ if (parsed === null) throw new Error(`test fixture is not a cadence: ${value}`);
32
+ return parsed;
33
+ }
34
+
35
+ function manifestWith(schedule?: string): ModuleManifest {
36
+ return { backup: schedule ? { schedule } : undefined } as unknown as ModuleManifest;
37
+ }
38
+
39
+ describe('effectiveBackupSchedule', () => {
40
+ test('an absent cadence from both sources means daily, not manual', () => {
41
+ expect(effectiveBackupSchedule(manifestWith(), undefined)).toEqual(cadence('daily'));
42
+ expect(DEFAULT_BACKUP_SCHEDULE).toEqual(cadence('daily'));
43
+ });
44
+
45
+ test("the manifest's suggestion is honoured when nobody has overridden", () => {
46
+ expect(effectiveBackupSchedule(manifestWith('manual'), undefined)).toBe('manual');
47
+ expect(effectiveBackupSchedule(manifestWith('weekly'), undefined)).toEqual(cadence('weekly'));
48
+ });
49
+
50
+ test('the operator override wins over the suggestion', () => {
51
+ expect(effectiveBackupSchedule(manifestWith('daily'), 'hourly')).toEqual(cadence('hourly'));
52
+ expect(effectiveBackupSchedule(manifestWith('daily'), '6h')).toEqual(cadence('6h'));
53
+ });
54
+
55
+ test('`manual` is reachable from either source', () => {
56
+ expect(effectiveBackupSchedule(manifestWith('daily'), 'manual')).toBe('manual');
57
+ expect(effectiveBackupSchedule(manifestWith('manual'), undefined)).toBe('manual');
58
+ });
59
+
60
+ test('an override applies to a module whose manifest suggests nothing', () => {
61
+ expect(effectiveBackupSchedule(manifestWith(), 'weekly')).toEqual(cadence('weekly'));
62
+ });
63
+
64
+ test('an unparseable override falls back to the manifest, never to manual', () => {
65
+ // Values are validated at SET time, so this is hand-edited state. Backing
66
+ // up MORE often than asked is the safe direction; silently never backing
67
+ // up is not.
68
+ expect(effectiveBackupSchedule(manifestWith('weekly'), 'dailyy')).toEqual(cadence('weekly'));
69
+ });
70
+ });
71
+
72
+ describe('isBackupDueFromHistory', () => {
73
+ test('manual never runs on a schedule', () => {
74
+ expect(
75
+ isBackupDueFromHistory(
76
+ 'manual',
77
+ { lastSuccessAt: null, lastAttemptAt: null, consecutiveFailures: 0 },
78
+ NOW,
79
+ ),
80
+ ).toBe(false);
81
+ });
82
+
83
+ test('a module that has never been backed up is due', () => {
84
+ expect(
85
+ isBackupDueFromHistory(
86
+ cadence('daily'),
87
+ { lastSuccessAt: null, lastAttemptAt: null, consecutiveFailures: 0 },
88
+ NOW,
89
+ ),
90
+ ).toBe(true);
91
+ });
92
+
93
+ test('a fresh success is not due again until its interval has passed', () => {
94
+ const history = {
95
+ lastSuccessAt: ago(2 * HOUR),
96
+ lastAttemptAt: ago(2 * HOUR),
97
+ consecutiveFailures: 0,
98
+ };
99
+ expect(isBackupDueFromHistory(cadence('daily'), history, NOW)).toBe(false);
100
+ expect(isBackupDueFromHistory(cadence('hourly'), history, NOW)).toBe(true);
101
+ });
102
+
103
+ test('a single failure retries on the next tick', () => {
104
+ // Most failures are transient. Waiting a whole cadence period after one bad
105
+ // minute at the storage endpoint would cost more coverage than it saves.
106
+ expect(
107
+ isBackupDueFromHistory(
108
+ cadence('daily'),
109
+ {
110
+ lastSuccessAt: ago(2 * DAY),
111
+ lastAttemptAt: ago(5 * 60 * 1000),
112
+ consecutiveFailures: 1,
113
+ },
114
+ NOW,
115
+ ),
116
+ ).toBe(true);
117
+ });
118
+
119
+ test('a run of failures backs off to the module cadence', () => {
120
+ // The celilo#685 shape: daily module, no success in a week, failing every
121
+ // hour. Under the old rule this was due at every tick forever.
122
+ const forgejo = {
123
+ lastSuccessAt: ago(7 * DAY),
124
+ lastAttemptAt: ago(1 * HOUR),
125
+ consecutiveFailures: 20,
126
+ };
127
+ expect(isBackupDueFromHistory(cadence('daily'), forgejo, NOW)).toBe(false);
128
+
129
+ // ...and still runs once its own interval has elapsed. Backing off is not
130
+ // giving up.
131
+ expect(
132
+ isBackupDueFromHistory(cadence('daily'), { ...forgejo, lastAttemptAt: ago(25 * HOUR) }, NOW),
133
+ ).toBe(true);
134
+ });
135
+
136
+ test('the back-off boundary is MAX_RAPID_RETRIES', () => {
137
+ const justFailed = { lastSuccessAt: ago(7 * DAY), lastAttemptAt: ago(1 * HOUR) };
138
+ expect(
139
+ isBackupDueFromHistory(
140
+ cadence('daily'),
141
+ { ...justFailed, consecutiveFailures: MAX_RAPID_RETRIES - 1 },
142
+ NOW,
143
+ ),
144
+ ).toBe(true);
145
+ expect(
146
+ isBackupDueFromHistory(
147
+ cadence('daily'),
148
+ { ...justFailed, consecutiveFailures: MAX_RAPID_RETRIES },
149
+ NOW,
150
+ ),
151
+ ).toBe(false);
152
+ });
153
+
154
+ test('a never-succeeded module also backs off once it is clearly failing', () => {
155
+ // `signal` on celilo-mgr: no successful backup has ever existed. Without
156
+ // this branch "never succeeded" reads as "always due" and the doomed
157
+ // attempt runs every tick.
158
+ expect(
159
+ isBackupDueFromHistory(
160
+ cadence('daily'),
161
+ { lastSuccessAt: null, lastAttemptAt: ago(1 * HOUR), consecutiveFailures: 8 },
162
+ NOW,
163
+ ),
164
+ ).toBe(false);
165
+ });
166
+ });
@@ -6,30 +6,120 @@
6
6
  * one. If those two read the manifest differently, a module can be
7
7
  * alerted-on-but-never-backed-up — an alert no human action can clear.
8
8
  *
9
- * Absent means `daily`, not `manual`. Treating "the author didn't say"
9
+ * The manifest is the author's SUGGESTION about a fleet they have never
10
+ * seen; the operator's `backup_schedule` override wins. Resolution
11
+ * happens HERE, at read time, so a corrected manifest still reaches
12
+ * every install that has not overridden it.
13
+ *
14
+ * Absent from both means `daily`, not `manual`. Treating "nobody said"
10
15
  * as "never check and never run" is what let celilo-mgmt go 55 days
11
16
  * without a backup and forgejo and signal go without one entirely, all
12
17
  * silently. Opting out is a decision worth writing down, so it takes an
13
- * explicit `schedule: manual`.
18
+ * explicit `manual`.
14
19
  */
15
20
 
16
21
  import type { ModuleManifest } from '../manifest/schema';
22
+ import { type Cadence, cadenceMs, parseCadence } from './cadence';
17
23
 
18
- export type BackupSchedule = 'hourly' | 'daily' | 'weekly' | 'monthly' | 'manual';
24
+ /** The `module_configs` key an operator's backup cadence is stored under. */
25
+ export const BACKUP_SCHEDULE_CONFIG_KEY = 'backup_schedule';
19
26
 
20
- /** Used when a manifest declares an `on_backup` hook but no cadence. */
21
- export const DEFAULT_BACKUP_SCHEDULE: BackupSchedule = 'daily';
27
+ /** Used when neither the operator nor the manifest says anything. */
28
+ export const DEFAULT_BACKUP_SCHEDULE: Cadence = { minutes: 24 * 60 };
22
29
 
23
- export function effectiveBackupSchedule(manifest: ModuleManifest): BackupSchedule {
30
+ /**
31
+ * `override` is the raw stored value of `backup_schedule`, or undefined when
32
+ * the operator has set none. An unparseable override falls back to the
33
+ * manifest rather than to `manual`: values are validated at SET time, so a bad
34
+ * one here means hand-edited state, and the safe direction is backing up more
35
+ * often than asked, never less.
36
+ */
37
+ export function effectiveBackupSchedule(
38
+ manifest: ModuleManifest,
39
+ override: string | undefined,
40
+ ): Cadence {
41
+ if (override !== undefined) {
42
+ const chosen = parseCadence(override);
43
+ if (chosen !== null) return chosen;
44
+ }
24
45
  const declared = manifest.backup?.schedule;
25
- switch (declared) {
26
- case 'hourly':
27
- case 'daily':
28
- case 'weekly':
29
- case 'monthly':
30
- case 'manual':
31
- return declared;
32
- default:
33
- return DEFAULT_BACKUP_SCHEDULE;
46
+ if (declared !== undefined) {
47
+ const suggested = parseCadence(declared);
48
+ if (suggested !== null) return suggested;
34
49
  }
50
+ return DEFAULT_BACKUP_SCHEDULE;
51
+ }
52
+
53
+ /**
54
+ * Failures in a row after which retrying stops being worth the resources.
55
+ *
56
+ * Under this many, a failure is assumed transient and the module stays due on
57
+ * the next tick, because most failures ARE transient — a storage endpoint
58
+ * having a bad minute should not cost a full cadence period of coverage.
59
+ *
60
+ * At or over it, the evidence says otherwise and the retry slows to the
61
+ * module's own cadence. Three is deliberately small: the useful information
62
+ * from a retry is almost entirely in the first one or two, and the cost of
63
+ * being wrong in this direction is bounded (one delayed backup) while the cost
64
+ * of the other direction is not.
65
+ */
66
+ export const MAX_RAPID_RETRIES = 3;
67
+
68
+ /** What the backup history says about one module, for the due-ness decision. */
69
+ export interface BackupHistory {
70
+ /**
71
+ * When the last successful backup COMPLETED, or null if there has never
72
+ * been one. Completion, not start — the same instant the freshness audit
73
+ * measures from, so the run path and the alert path cannot disagree about
74
+ * how old a backup is (design.md D6).
75
+ */
76
+ lastSuccessAt: Date | null;
77
+ /** Last attempt of any outcome, or null if none has ever been made. */
78
+ lastAttemptAt: Date | null;
79
+ /** Attempts since the last success. */
80
+ consecutiveFailures: number;
81
+ }
82
+
83
+ /**
84
+ * Whether to start a backup for a module right now.
85
+ *
86
+ * Due-ness used to be "has it been an interval since the last SUCCESS", which
87
+ * is correct in the healthy case and degenerate in the failing one: a module
88
+ * that cannot back up never advances that timestamp, so it is due at every
89
+ * tick forever. The sweep runs hourly, so a *daily* module that started failing
90
+ * was re-picked 24 times a day. On celilo-mgr that meant 20+ consecutive
91
+ * forgejo attempts, each one assembling ~1.9 GB of staging and then dying, none
92
+ * of them ever going to succeed for a reason no retry could change (celilo#685).
93
+ *
94
+ * Retrying is still right — it just cannot be unconditional. So the failure
95
+ * count decides which clock applies: under `MAX_RAPID_RETRIES` the module stays
96
+ * due against its last success, and at or over it the interval is measured from
97
+ * the last ATTEMPT instead, which turns 24 doomed attempts a day into one.
98
+ *
99
+ * Backing off is not the same as going quiet. The `backups` drift monitor
100
+ * measures staleness from the last success and is unaffected by this, so a
101
+ * module that has slowed to one attempt a day still alerts as stale on exactly
102
+ * the schedule it would have before — see services/audit/backups.ts.
103
+ *
104
+ * Pure, and time is a parameter, so the policy tests without a database.
105
+ */
106
+ export function isBackupDueFromHistory(
107
+ schedule: Cadence,
108
+ history: BackupHistory,
109
+ now: number,
110
+ ): boolean {
111
+ if (schedule === 'manual') return false;
112
+
113
+ const interval = cadenceMs(schedule);
114
+
115
+ if (history.consecutiveFailures >= MAX_RAPID_RETRIES) {
116
+ // A run of failures with no attempt recorded is not a state the sweep can
117
+ // produce, but "cannot prove an interval has passed" must not mean "start
118
+ // another 1.9 GB attempt".
119
+ if (!history.lastAttemptAt) return false;
120
+ return now - history.lastAttemptAt.getTime() >= interval;
121
+ }
122
+
123
+ if (!history.lastSuccessAt) return true;
124
+ return now - history.lastSuccessAt.getTime() >= interval;
35
125
  }
@@ -95,9 +95,22 @@ export interface ReapedStaging {
95
95
  * drift check can read a module as recently backed up when every attempt in
96
96
  * fact died. Nine such rows were live on celilo-mgr while forgejo had no usable
97
97
  * backup at all.
98
+ *
99
+ * It names no cause because this pass genuinely cannot know one: it runs later,
100
+ * in a different process, and infers the death from a pid that is no longer
101
+ * there. The observer that DOES know is the dispatcher, which holds the exit
102
+ * code and translates it (`describeHandlerExit` in packages/event-bus) — a
103
+ * SIGKILL, say, and the OOM killer that most likely sent it.
104
+ *
105
+ * The two facts existing in two places is fine; the operator having no way to
106
+ * learn that is not. In celilo#685 this string was the whole of what an
107
+ * operator saw for twenty consecutive OOM kills, so it now points at the record
108
+ * that has the answer rather than terminating the trail.
98
109
  */
99
110
  export const ABANDONED_BACKUP_MESSAGE =
100
- 'abandoned — the backup process ended without recording an outcome';
111
+ 'abandoned — the backup process ended without recording an outcome. ' +
112
+ 'Only the process that ran it saw how it died, so the cause is recorded ' +
113
+ 'against the event delivery rather than here: run `celilo system doctor`.';
101
114
 
102
115
  /**
103
116
  * Whether reclaiming this directory also means its record was lying about
@@ -1,6 +1,5 @@
1
1
  import { describe, expect, test } from 'bun:test';
2
2
  import type { ModuleManifest } from '../manifest/schema';
3
- import type { BackupSchedule } from './backup-schedule';
4
3
  import {
5
4
  BACKUP_SWEEP_MAX_ATTEMPTS,
6
5
  BACKUP_SWEEP_PATTERN,
@@ -13,13 +12,13 @@ import {
13
12
  } from './backup-sweep';
14
13
  import { InFlightError } from './module-operations';
15
14
 
16
- function moduleWith(id: string, schedule?: BackupSchedule): BackupSweepModule {
15
+ function moduleWith(id: string, schedule?: string, scheduleOverride?: string): BackupSweepModule {
17
16
  const manifest = {
18
17
  id,
19
18
  hooks: { on_backup: { script: 'backup.ts' } },
20
19
  ...(schedule ? { backup: { schedule } } : {}),
21
20
  } as unknown as ModuleManifest;
22
- return { id, manifest };
21
+ return { id, manifest, scheduleOverride };
23
22
  }
24
23
 
25
24
  function deps(
@@ -123,6 +122,26 @@ describe('runBackupSweep', () => {
123
122
  expect(report.skippedManual).toEqual(['scratch']);
124
123
  });
125
124
 
125
+ test("an operator's override decides the cadence, not the manifest", async () => {
126
+ const d = deps([moduleWith('caddy', 'daily', '6h')]);
127
+ const seen: Array<[string, unknown]> = [];
128
+ d.isDue = (moduleId, schedule) => {
129
+ seen.push([moduleId, schedule]);
130
+ return true;
131
+ };
132
+
133
+ await runBackupSweep(d);
134
+
135
+ expect(seen).toEqual([['caddy', { minutes: 360 }]]);
136
+ });
137
+
138
+ test('an override of manual stops a module the manifest wanted backed up', async () => {
139
+ const report = await runBackupSweep(deps([moduleWith('caddy', 'daily', 'manual')]));
140
+
141
+ expect(report.backedUp).toEqual([]);
142
+ expect(report.skippedManual).toEqual(['caddy']);
143
+ });
144
+
126
145
  test('an undeclared schedule is backed up, not treated as manual', async () => {
127
146
  // The regression this whole subsystem exists for: forgejo and signal
128
147
  // declare no `backup:` block and had never been backed up.
@@ -17,12 +17,16 @@
17
17
  */
18
18
 
19
19
  import type { ModuleManifest } from '../manifest/schema';
20
- import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
20
+ import { effectiveBackupSchedule } from './backup-schedule';
21
21
  import type { ReapStagingReport, ResolveAbandonedReport } from './backup-staging';
22
+ import { BACKUP_SWEEP_PATTERN, type Cadence } from './cadence';
22
23
  import { InFlightError } from './module-operations';
23
24
 
24
25
  export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
25
- export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
26
+ // The tick itself lives in services/cadence.ts, next to the floor derived from
27
+ // it — a sweep whose tick and whose finest servable cadence are stated in two
28
+ // files is the pair that drifts.
29
+ export { BACKUP_SWEEP_PATTERN };
26
30
 
27
31
  /**
28
32
  * How long the sweep may run before the dispatcher kills it.
@@ -99,12 +103,18 @@ export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
99
103
  export interface BackupSweepModule {
100
104
  id: string;
101
105
  manifest: ModuleManifest;
106
+ /**
107
+ * The operator's `backup_schedule` override, or undefined when they have set
108
+ * none. Carried rather than pre-resolved so the cadence is resolved through
109
+ * the one shared accessor here, the same way the freshness audit resolves it.
110
+ */
111
+ scheduleOverride: string | undefined;
102
112
  }
103
113
 
104
114
  export interface BackupSweepDeps {
105
115
  /** Installed modules that declare an `on_backup` hook. */
106
116
  listEligible(): BackupSweepModule[];
107
- isDue(moduleId: string, schedule: BackupSchedule): boolean;
117
+ isDue(moduleId: string, schedule: Cadence): boolean;
108
118
  backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
109
119
  /** Apply the module's declared retention. No-op when it declares none. */
110
120
  prune(module: BackupSweepModule): Promise<void>;
@@ -126,7 +136,7 @@ export interface BackupSweepReport {
126
136
  /** Records corrected from a stale `in_progress`. */
127
137
  records: ResolveAbandonedReport;
128
138
  backedUp: string[];
129
- /** Explicit `schedule: manual` — the author opted out. */
139
+ /** Effective cadence `manual` — the operator or the author opted out. */
130
140
  skippedManual: string[];
131
141
  skippedNotDue: string[];
132
142
  /** Another module operation held the lock. Not a failure; retried next tick. */
@@ -157,7 +167,7 @@ export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweep
157
167
  };
158
168
 
159
169
  for (const module of deps.listEligible()) {
160
- const schedule = effectiveBackupSchedule(module.manifest);
170
+ const schedule = effectiveBackupSchedule(module.manifest, module.scheduleOverride);
161
171
  if (schedule === 'manual') {
162
172
  report.skippedManual.push(module.id);
163
173
  continue;
@@ -0,0 +1,97 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import {
3
+ ALERTING_SWEEP_PATTERN,
4
+ BACKUP_CADENCE_FLOOR_MINUTES,
5
+ BACKUP_SWEEP_PATTERN,
6
+ MONITOR_INTERVAL_FLOOR_MINUTES,
7
+ cadenceMs,
8
+ cadenceSchema,
9
+ formatCadence,
10
+ parseCadence,
11
+ tickIntervalMinutes,
12
+ } from './cadence';
13
+
14
+ describe('parseCadence', () => {
15
+ test('every named period equals its duration equivalent', () => {
16
+ expect(parseCadence('hourly')).toEqual(parseCadence('1h'));
17
+ expect(parseCadence('daily')).toEqual(parseCadence('24h'));
18
+ expect(parseCadence('weekly')).toEqual(parseCadence('7d'));
19
+ expect(parseCadence('monthly')).toEqual(parseCadence('30d'));
20
+ });
21
+
22
+ test('durations normalise to minutes', () => {
23
+ expect(parseCadence('90m')).toEqual({ minutes: 90 });
24
+ expect(parseCadence('6h')).toEqual({ minutes: 360 });
25
+ expect(parseCadence('3d')).toEqual({ minutes: 4320 });
26
+ });
27
+
28
+ test('manual is its own value, not a number', () => {
29
+ expect(parseCadence('manual')).toBe('manual');
30
+ expect(cadenceMs('manual')).toBe(Number.POSITIVE_INFINITY);
31
+ });
32
+
33
+ test('rejects malformed, zero and negative values', () => {
34
+ for (const bad of ['dailyy', '', '6', 'h6', '6w', '0h', '-1h', '1.5h', 'never']) {
35
+ expect(parseCadence(bad)).toBeNull();
36
+ }
37
+ });
38
+ });
39
+
40
+ describe('formatCadence', () => {
41
+ test('prefers the word an operator would have typed', () => {
42
+ expect(formatCadence({ minutes: 60 })).toBe('hourly');
43
+ expect(formatCadence({ minutes: 1440 })).toBe('daily');
44
+ expect(formatCadence('manual')).toBe('manual');
45
+ });
46
+
47
+ test('falls back to the coarsest exact duration', () => {
48
+ expect(formatCadence({ minutes: 360 })).toBe('6h');
49
+ expect(formatCadence({ minutes: 2880 })).toBe('2d');
50
+ expect(formatCadence({ minutes: 90 })).toBe('90m');
51
+ });
52
+ });
53
+
54
+ describe('the floor follows the sweep tick', () => {
55
+ test('each floor is the tick of the sweep that would serve it', () => {
56
+ expect(BACKUP_CADENCE_FLOOR_MINUTES).toBe(tickIntervalMinutes(BACKUP_SWEEP_PATTERN));
57
+ expect(MONITOR_INTERVAL_FLOOR_MINUTES).toBe(tickIntervalMinutes(ALERTING_SWEEP_PATTERN));
58
+ });
59
+
60
+ test('a finer tick admits a cadence the coarser one refused', () => {
61
+ const coarse = cadenceSchema({ floorMinutes: tickIntervalMinutes('timer.tick.1h') });
62
+ const fine = cadenceSchema({ floorMinutes: tickIntervalMinutes('timer.tick.5m') });
63
+ expect(coarse.safeParse('15m').success).toBe(false);
64
+ expect(fine.safeParse('15m').success).toBe(true);
65
+ });
66
+
67
+ test('a pattern that is not a timer tick is a programming error, not a floor of zero', () => {
68
+ expect(() => tickIntervalMinutes('module.deployed')).toThrow();
69
+ });
70
+ });
71
+
72
+ describe('cadenceSchema', () => {
73
+ const schema = cadenceSchema({ floorMinutes: 60 });
74
+
75
+ test('accepts words, durations at or above the floor, and manual', () => {
76
+ for (const good of ['hourly', 'daily', 'weekly', 'monthly', 'manual', '6h', '2d', '60m']) {
77
+ expect(schema.safeParse(good).success).toBe(true);
78
+ }
79
+ });
80
+
81
+ test('names the permitted forms when the value is unparseable', () => {
82
+ const result = schema.safeParse('dailyy');
83
+ expect(result.success).toBe(false);
84
+ if (!result.success) {
85
+ expect(result.error.issues[0].message).toContain('named period');
86
+ expect(result.error.issues[0].message).toContain('manual');
87
+ }
88
+ });
89
+
90
+ test('names the finest servable cadence when the value is below the floor', () => {
91
+ const result = schema.safeParse('5m');
92
+ expect(result.success).toBe(false);
93
+ if (!result.success) {
94
+ expect(result.error.issues[0].message).toContain('hourly');
95
+ }
96
+ });
97
+ });