@celilo/cli 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +27 -7
- package/package.json +6 -5
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +64 -2
- package/src/cli/commands/module-config.ts +159 -8
- package/src/cli/commands/module-status.ts +124 -0
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +35 -9
- package/src/cli/index.ts +59 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/sweep-runner.test.ts +51 -3
- package/src/services/alerting/sweep-runner.ts +30 -7
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +33 -13
|
@@ -1,19 +1,77 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Backup retention
|
|
3
|
-
*
|
|
4
|
-
* Policies are count-based (keep last N) and age-based (delete older than X
|
|
5
|
-
*
|
|
2
|
+
* Backup retention: how many copies to keep, and for how long.
|
|
3
|
+
*
|
|
4
|
+
* Policies are count-based (keep last N) and age-based (delete older than X
|
|
5
|
+
* days), and the two are INDEPENDENT — whichever limit is hit first triggers
|
|
6
|
+
* deletion. The manifest suggests; the operator's `backup_retention_count` and
|
|
7
|
+
* `backup_retention_max_age_days` overrides decide, each dimension resolved on
|
|
8
|
+
* its own.
|
|
9
|
+
*
|
|
10
|
+
* ⚠️ An unset dimension is UNBOUNDED, never a default bound. Today an absent
|
|
11
|
+
* `backup.retention` block means prune nothing at all — the block is optional,
|
|
12
|
+
* so its inner `count: 7` / `max_age_days: 30` defaults never apply. If setting
|
|
13
|
+
* one dimension made the other fall back to those defaults, an operator asking
|
|
14
|
+
* to keep 3 copies would silently also arm a 30-day deletion they never asked
|
|
15
|
+
* for, on a module that had been keeping everything. Deleting backups nobody
|
|
16
|
+
* asked to delete is the one failure here that cannot be undone.
|
|
6
17
|
*/
|
|
7
18
|
|
|
8
19
|
import type { Backup } from '../db/schema';
|
|
20
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
9
21
|
import { deleteBackupRecord, listCompletedBackupsForModule } from './backup-metadata';
|
|
10
22
|
import { createStorageProvider } from './backup-storage';
|
|
23
|
+
import { configOverride } from './module-config';
|
|
24
|
+
|
|
25
|
+
/** The `module_configs` keys an operator's retention policy is stored under. */
|
|
26
|
+
export const BACKUP_RETENTION_COUNT_CONFIG_KEY = 'backup_retention_count';
|
|
27
|
+
export const BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY = 'backup_retention_max_age_days';
|
|
11
28
|
|
|
12
29
|
export interface RetentionPolicy {
|
|
30
|
+
/** Copies to keep. `Infinity` means unbounded — keep every copy. */
|
|
13
31
|
count: number;
|
|
32
|
+
/** Days to keep. `Infinity` means unbounded — never delete on age. */
|
|
14
33
|
maxAgeDays: number;
|
|
15
34
|
}
|
|
16
35
|
|
|
36
|
+
/** Neither dimension bounded: nothing is ever pruned, so the pass can be skipped. */
|
|
37
|
+
export function prunesNothing(policy: RetentionPolicy): boolean {
|
|
38
|
+
return (
|
|
39
|
+
policy.count === Number.POSITIVE_INFINITY && policy.maxAgeDays === Number.POSITIVE_INFINITY
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Takes the module's operator config as loaded from `module_configs`. An
|
|
45
|
+
* unparseable or non-positive override leaves that dimension to the manifest:
|
|
46
|
+
* values are validated at SET time, so a bad one here means hand-edited state,
|
|
47
|
+
* and keeping too much is the only safe direction to fail in.
|
|
48
|
+
*/
|
|
49
|
+
export function effectiveBackupRetention(
|
|
50
|
+
manifest: ModuleManifest,
|
|
51
|
+
configs: Record<string, unknown> | undefined,
|
|
52
|
+
): RetentionPolicy {
|
|
53
|
+
const declared = manifest.backup?.retention;
|
|
54
|
+
return {
|
|
55
|
+
count: positiveIntegerOr(
|
|
56
|
+
configOverride(configs, BACKUP_RETENTION_COUNT_CONFIG_KEY),
|
|
57
|
+
declared?.count,
|
|
58
|
+
),
|
|
59
|
+
maxAgeDays: positiveIntegerOr(
|
|
60
|
+
configOverride(configs, BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY),
|
|
61
|
+
declared?.max_age_days,
|
|
62
|
+
),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Override, else the manifest's suggestion, else unbounded. Never a default bound. */
|
|
67
|
+
function positiveIntegerOr(override: string | undefined, suggested: number | undefined): number {
|
|
68
|
+
if (override !== undefined) {
|
|
69
|
+
const parsed = Number(override);
|
|
70
|
+
if (Number.isInteger(parsed) && parsed > 0) return parsed;
|
|
71
|
+
}
|
|
72
|
+
return suggested ?? Number.POSITIVE_INFINITY;
|
|
73
|
+
}
|
|
74
|
+
|
|
17
75
|
export interface PruneResult {
|
|
18
76
|
moduleId: string;
|
|
19
77
|
deleted: number;
|
|
@@ -21,7 +79,10 @@ export interface PruneResult {
|
|
|
21
79
|
}
|
|
22
80
|
|
|
23
81
|
/**
|
|
24
|
-
* Identify backups that should be pruned per the retention policy
|
|
82
|
+
* Identify backups that should be pruned per the retention policy.
|
|
83
|
+
*
|
|
84
|
+
* An unbounded dimension is `Infinity`, which needs no special case: no index
|
|
85
|
+
* reaches it and no age exceeds it.
|
|
25
86
|
*/
|
|
26
87
|
export function identifyExpiredBackups(backupsList: Backup[], policy: RetentionPolicy): Backup[] {
|
|
27
88
|
const now = Date.now();
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backup cadence: what a manifest declares, and when the sweep acts on it.
|
|
3
|
+
*
|
|
4
|
+
* The due-ness tests are the interesting half. celilo#685 was a *daily* module
|
|
5
|
+
* attempted 24 times a day for a day, each attempt assembling ~1.9 GB before
|
|
6
|
+
* being OOM-killed, because due-ness was measured only from the last SUCCESS
|
|
7
|
+
* and a module that cannot succeed never advances that timestamp.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { describe, expect, test } from 'bun:test';
|
|
11
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
12
|
+
import {
|
|
13
|
+
DEFAULT_BACKUP_SCHEDULE,
|
|
14
|
+
MAX_RAPID_RETRIES,
|
|
15
|
+
effectiveBackupSchedule,
|
|
16
|
+
isBackupDueFromHistory,
|
|
17
|
+
} from './backup-schedule';
|
|
18
|
+
import { type Cadence, parseCadence } from './cadence';
|
|
19
|
+
|
|
20
|
+
const HOUR = 60 * 60 * 1000;
|
|
21
|
+
const DAY = 24 * HOUR;
|
|
22
|
+
const NOW = Date.UTC(2026, 7, 13, 12, 0, 0);
|
|
23
|
+
|
|
24
|
+
function ago(ms: number): Date {
|
|
25
|
+
return new Date(NOW - ms);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Parse a cadence in a test, failing loudly rather than passing null on (Rule 7.2). */
|
|
29
|
+
function cadence(value: string): Cadence {
|
|
30
|
+
const parsed = parseCadence(value);
|
|
31
|
+
if (parsed === null) throw new Error(`test fixture is not a cadence: ${value}`);
|
|
32
|
+
return parsed;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function manifestWith(schedule?: string): ModuleManifest {
|
|
36
|
+
return { backup: schedule ? { schedule } : undefined } as unknown as ModuleManifest;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
describe('effectiveBackupSchedule', () => {
|
|
40
|
+
test('an absent cadence from both sources means daily, not manual', () => {
|
|
41
|
+
expect(effectiveBackupSchedule(manifestWith(), undefined)).toEqual(cadence('daily'));
|
|
42
|
+
expect(DEFAULT_BACKUP_SCHEDULE).toEqual(cadence('daily'));
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("the manifest's suggestion is honoured when nobody has overridden", () => {
|
|
46
|
+
expect(effectiveBackupSchedule(manifestWith('manual'), undefined)).toBe('manual');
|
|
47
|
+
expect(effectiveBackupSchedule(manifestWith('weekly'), undefined)).toEqual(cadence('weekly'));
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test('the operator override wins over the suggestion', () => {
|
|
51
|
+
expect(effectiveBackupSchedule(manifestWith('daily'), 'hourly')).toEqual(cadence('hourly'));
|
|
52
|
+
expect(effectiveBackupSchedule(manifestWith('daily'), '6h')).toEqual(cadence('6h'));
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test('`manual` is reachable from either source', () => {
|
|
56
|
+
expect(effectiveBackupSchedule(manifestWith('daily'), 'manual')).toBe('manual');
|
|
57
|
+
expect(effectiveBackupSchedule(manifestWith('manual'), undefined)).toBe('manual');
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test('an override applies to a module whose manifest suggests nothing', () => {
|
|
61
|
+
expect(effectiveBackupSchedule(manifestWith(), 'weekly')).toEqual(cadence('weekly'));
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test('an unparseable override falls back to the manifest, never to manual', () => {
|
|
65
|
+
// Values are validated at SET time, so this is hand-edited state. Backing
|
|
66
|
+
// up MORE often than asked is the safe direction; silently never backing
|
|
67
|
+
// up is not.
|
|
68
|
+
expect(effectiveBackupSchedule(manifestWith('weekly'), 'dailyy')).toEqual(cadence('weekly'));
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
describe('isBackupDueFromHistory', () => {
|
|
73
|
+
test('manual never runs on a schedule', () => {
|
|
74
|
+
expect(
|
|
75
|
+
isBackupDueFromHistory(
|
|
76
|
+
'manual',
|
|
77
|
+
{ lastSuccessAt: null, lastAttemptAt: null, consecutiveFailures: 0 },
|
|
78
|
+
NOW,
|
|
79
|
+
),
|
|
80
|
+
).toBe(false);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test('a module that has never been backed up is due', () => {
|
|
84
|
+
expect(
|
|
85
|
+
isBackupDueFromHistory(
|
|
86
|
+
cadence('daily'),
|
|
87
|
+
{ lastSuccessAt: null, lastAttemptAt: null, consecutiveFailures: 0 },
|
|
88
|
+
NOW,
|
|
89
|
+
),
|
|
90
|
+
).toBe(true);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test('a fresh success is not due again until its interval has passed', () => {
|
|
94
|
+
const history = {
|
|
95
|
+
lastSuccessAt: ago(2 * HOUR),
|
|
96
|
+
lastAttemptAt: ago(2 * HOUR),
|
|
97
|
+
consecutiveFailures: 0,
|
|
98
|
+
};
|
|
99
|
+
expect(isBackupDueFromHistory(cadence('daily'), history, NOW)).toBe(false);
|
|
100
|
+
expect(isBackupDueFromHistory(cadence('hourly'), history, NOW)).toBe(true);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test('a single failure retries on the next tick', () => {
|
|
104
|
+
// Most failures are transient. Waiting a whole cadence period after one bad
|
|
105
|
+
// minute at the storage endpoint would cost more coverage than it saves.
|
|
106
|
+
expect(
|
|
107
|
+
isBackupDueFromHistory(
|
|
108
|
+
cadence('daily'),
|
|
109
|
+
{
|
|
110
|
+
lastSuccessAt: ago(2 * DAY),
|
|
111
|
+
lastAttemptAt: ago(5 * 60 * 1000),
|
|
112
|
+
consecutiveFailures: 1,
|
|
113
|
+
},
|
|
114
|
+
NOW,
|
|
115
|
+
),
|
|
116
|
+
).toBe(true);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test('a run of failures backs off to the module cadence', () => {
|
|
120
|
+
// The celilo#685 shape: daily module, no success in a week, failing every
|
|
121
|
+
// hour. Under the old rule this was due at every tick forever.
|
|
122
|
+
const forgejo = {
|
|
123
|
+
lastSuccessAt: ago(7 * DAY),
|
|
124
|
+
lastAttemptAt: ago(1 * HOUR),
|
|
125
|
+
consecutiveFailures: 20,
|
|
126
|
+
};
|
|
127
|
+
expect(isBackupDueFromHistory(cadence('daily'), forgejo, NOW)).toBe(false);
|
|
128
|
+
|
|
129
|
+
// ...and still runs once its own interval has elapsed. Backing off is not
|
|
130
|
+
// giving up.
|
|
131
|
+
expect(
|
|
132
|
+
isBackupDueFromHistory(cadence('daily'), { ...forgejo, lastAttemptAt: ago(25 * HOUR) }, NOW),
|
|
133
|
+
).toBe(true);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
test('the back-off boundary is MAX_RAPID_RETRIES', () => {
|
|
137
|
+
const justFailed = { lastSuccessAt: ago(7 * DAY), lastAttemptAt: ago(1 * HOUR) };
|
|
138
|
+
expect(
|
|
139
|
+
isBackupDueFromHistory(
|
|
140
|
+
cadence('daily'),
|
|
141
|
+
{ ...justFailed, consecutiveFailures: MAX_RAPID_RETRIES - 1 },
|
|
142
|
+
NOW,
|
|
143
|
+
),
|
|
144
|
+
).toBe(true);
|
|
145
|
+
expect(
|
|
146
|
+
isBackupDueFromHistory(
|
|
147
|
+
cadence('daily'),
|
|
148
|
+
{ ...justFailed, consecutiveFailures: MAX_RAPID_RETRIES },
|
|
149
|
+
NOW,
|
|
150
|
+
),
|
|
151
|
+
).toBe(false);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
test('a never-succeeded module also backs off once it is clearly failing', () => {
|
|
155
|
+
// `signal` on celilo-mgr: no successful backup has ever existed. Without
|
|
156
|
+
// this branch "never succeeded" reads as "always due" and the doomed
|
|
157
|
+
// attempt runs every tick.
|
|
158
|
+
expect(
|
|
159
|
+
isBackupDueFromHistory(
|
|
160
|
+
cadence('daily'),
|
|
161
|
+
{ lastSuccessAt: null, lastAttemptAt: ago(1 * HOUR), consecutiveFailures: 8 },
|
|
162
|
+
NOW,
|
|
163
|
+
),
|
|
164
|
+
).toBe(false);
|
|
165
|
+
});
|
|
166
|
+
});
|
|
@@ -6,30 +6,120 @@
|
|
|
6
6
|
* one. If those two read the manifest differently, a module can be
|
|
7
7
|
* alerted-on-but-never-backed-up — an alert no human action can clear.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
9
|
+
* The manifest is the author's SUGGESTION about a fleet they have never
|
|
10
|
+
* seen; the operator's `backup_schedule` override wins. Resolution
|
|
11
|
+
* happens HERE, at read time, so a corrected manifest still reaches
|
|
12
|
+
* every install that has not overridden it.
|
|
13
|
+
*
|
|
14
|
+
* Absent from both means `daily`, not `manual`. Treating "nobody said"
|
|
10
15
|
* as "never check and never run" is what let celilo-mgmt go 55 days
|
|
11
16
|
* without a backup and forgejo and signal go without one entirely, all
|
|
12
17
|
* silently. Opting out is a decision worth writing down, so it takes an
|
|
13
|
-
* explicit `
|
|
18
|
+
* explicit `manual`.
|
|
14
19
|
*/
|
|
15
20
|
|
|
16
21
|
import type { ModuleManifest } from '../manifest/schema';
|
|
22
|
+
import { type Cadence, cadenceMs, parseCadence } from './cadence';
|
|
17
23
|
|
|
18
|
-
|
|
24
|
+
/** The `module_configs` key an operator's backup cadence is stored under. */
|
|
25
|
+
export const BACKUP_SCHEDULE_CONFIG_KEY = 'backup_schedule';
|
|
19
26
|
|
|
20
|
-
/** Used when
|
|
21
|
-
export const DEFAULT_BACKUP_SCHEDULE:
|
|
27
|
+
/** Used when neither the operator nor the manifest says anything. */
|
|
28
|
+
export const DEFAULT_BACKUP_SCHEDULE: Cadence = { minutes: 24 * 60 };
|
|
22
29
|
|
|
23
|
-
|
|
30
|
+
/**
|
|
31
|
+
* `override` is the raw stored value of `backup_schedule`, or undefined when
|
|
32
|
+
* the operator has set none. An unparseable override falls back to the
|
|
33
|
+
* manifest rather than to `manual`: values are validated at SET time, so a bad
|
|
34
|
+
* one here means hand-edited state, and the safe direction is backing up more
|
|
35
|
+
* often than asked, never less.
|
|
36
|
+
*/
|
|
37
|
+
export function effectiveBackupSchedule(
|
|
38
|
+
manifest: ModuleManifest,
|
|
39
|
+
override: string | undefined,
|
|
40
|
+
): Cadence {
|
|
41
|
+
if (override !== undefined) {
|
|
42
|
+
const chosen = parseCadence(override);
|
|
43
|
+
if (chosen !== null) return chosen;
|
|
44
|
+
}
|
|
24
45
|
const declared = manifest.backup?.schedule;
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
case 'weekly':
|
|
29
|
-
case 'monthly':
|
|
30
|
-
case 'manual':
|
|
31
|
-
return declared;
|
|
32
|
-
default:
|
|
33
|
-
return DEFAULT_BACKUP_SCHEDULE;
|
|
46
|
+
if (declared !== undefined) {
|
|
47
|
+
const suggested = parseCadence(declared);
|
|
48
|
+
if (suggested !== null) return suggested;
|
|
34
49
|
}
|
|
50
|
+
return DEFAULT_BACKUP_SCHEDULE;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Failures in a row after which retrying stops being worth the resources.
|
|
55
|
+
*
|
|
56
|
+
* Under this many, a failure is assumed transient and the module stays due on
|
|
57
|
+
* the next tick, because most failures ARE transient — a storage endpoint
|
|
58
|
+
* having a bad minute should not cost a full cadence period of coverage.
|
|
59
|
+
*
|
|
60
|
+
* At or over it, the evidence says otherwise and the retry slows to the
|
|
61
|
+
* module's own cadence. Three is deliberately small: the useful information
|
|
62
|
+
* from a retry is almost entirely in the first one or two, and the cost of
|
|
63
|
+
* being wrong in this direction is bounded (one delayed backup) while the cost
|
|
64
|
+
* of the other direction is not.
|
|
65
|
+
*/
|
|
66
|
+
export const MAX_RAPID_RETRIES = 3;
|
|
67
|
+
|
|
68
|
+
/** What the backup history says about one module, for the due-ness decision. */
|
|
69
|
+
export interface BackupHistory {
|
|
70
|
+
/**
|
|
71
|
+
* When the last successful backup COMPLETED, or null if there has never
|
|
72
|
+
* been one. Completion, not start — the same instant the freshness audit
|
|
73
|
+
* measures from, so the run path and the alert path cannot disagree about
|
|
74
|
+
* how old a backup is (design.md D6).
|
|
75
|
+
*/
|
|
76
|
+
lastSuccessAt: Date | null;
|
|
77
|
+
/** Last attempt of any outcome, or null if none has ever been made. */
|
|
78
|
+
lastAttemptAt: Date | null;
|
|
79
|
+
/** Attempts since the last success. */
|
|
80
|
+
consecutiveFailures: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Whether to start a backup for a module right now.
|
|
85
|
+
*
|
|
86
|
+
* Due-ness used to be "has it been an interval since the last SUCCESS", which
|
|
87
|
+
* is correct in the healthy case and degenerate in the failing one: a module
|
|
88
|
+
* that cannot back up never advances that timestamp, so it is due at every
|
|
89
|
+
* tick forever. The sweep runs hourly, so a *daily* module that started failing
|
|
90
|
+
* was re-picked 24 times a day. On celilo-mgr that meant 20+ consecutive
|
|
91
|
+
* forgejo attempts, each one assembling ~1.9 GB of staging and then dying, none
|
|
92
|
+
* of them ever going to succeed for a reason no retry could change (celilo#685).
|
|
93
|
+
*
|
|
94
|
+
* Retrying is still right — it just cannot be unconditional. So the failure
|
|
95
|
+
* count decides which clock applies: under `MAX_RAPID_RETRIES` the module stays
|
|
96
|
+
* due against its last success, and at or over it the interval is measured from
|
|
97
|
+
* the last ATTEMPT instead, which turns 24 doomed attempts a day into one.
|
|
98
|
+
*
|
|
99
|
+
* Backing off is not the same as going quiet. The `backups` drift monitor
|
|
100
|
+
* measures staleness from the last success and is unaffected by this, so a
|
|
101
|
+
* module that has slowed to one attempt a day still alerts as stale on exactly
|
|
102
|
+
* the schedule it would have before — see services/audit/backups.ts.
|
|
103
|
+
*
|
|
104
|
+
* Pure, and time is a parameter, so the policy tests without a database.
|
|
105
|
+
*/
|
|
106
|
+
export function isBackupDueFromHistory(
|
|
107
|
+
schedule: Cadence,
|
|
108
|
+
history: BackupHistory,
|
|
109
|
+
now: number,
|
|
110
|
+
): boolean {
|
|
111
|
+
if (schedule === 'manual') return false;
|
|
112
|
+
|
|
113
|
+
const interval = cadenceMs(schedule);
|
|
114
|
+
|
|
115
|
+
if (history.consecutiveFailures >= MAX_RAPID_RETRIES) {
|
|
116
|
+
// A run of failures with no attempt recorded is not a state the sweep can
|
|
117
|
+
// produce, but "cannot prove an interval has passed" must not mean "start
|
|
118
|
+
// another 1.9 GB attempt".
|
|
119
|
+
if (!history.lastAttemptAt) return false;
|
|
120
|
+
return now - history.lastAttemptAt.getTime() >= interval;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if (!history.lastSuccessAt) return true;
|
|
124
|
+
return now - history.lastSuccessAt.getTime() >= interval;
|
|
35
125
|
}
|
|
@@ -95,9 +95,22 @@ export interface ReapedStaging {
|
|
|
95
95
|
* drift check can read a module as recently backed up when every attempt in
|
|
96
96
|
* fact died. Nine such rows were live on celilo-mgr while forgejo had no usable
|
|
97
97
|
* backup at all.
|
|
98
|
+
*
|
|
99
|
+
* It names no cause because this pass genuinely cannot know one: it runs later,
|
|
100
|
+
* in a different process, and infers the death from a pid that is no longer
|
|
101
|
+
* there. The observer that DOES know is the dispatcher, which holds the exit
|
|
102
|
+
* code and translates it (`describeHandlerExit` in packages/event-bus) — a
|
|
103
|
+
* SIGKILL, say, and the OOM killer that most likely sent it.
|
|
104
|
+
*
|
|
105
|
+
* The two facts existing in two places is fine; the operator having no way to
|
|
106
|
+
* learn that is not. In celilo#685 this string was the whole of what an
|
|
107
|
+
* operator saw for twenty consecutive OOM kills, so it now points at the record
|
|
108
|
+
* that has the answer rather than terminating the trail.
|
|
98
109
|
*/
|
|
99
110
|
export const ABANDONED_BACKUP_MESSAGE =
|
|
100
|
-
'abandoned — the backup process ended without recording an outcome'
|
|
111
|
+
'abandoned — the backup process ended without recording an outcome. ' +
|
|
112
|
+
'Only the process that ran it saw how it died, so the cause is recorded ' +
|
|
113
|
+
'against the event delivery rather than here: run `celilo system doctor`.';
|
|
101
114
|
|
|
102
115
|
/**
|
|
103
116
|
* Whether reclaiming this directory also means its record was lying about
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
2
|
import type { ModuleManifest } from '../manifest/schema';
|
|
3
|
-
import type { BackupSchedule } from './backup-schedule';
|
|
4
3
|
import {
|
|
5
4
|
BACKUP_SWEEP_MAX_ATTEMPTS,
|
|
6
5
|
BACKUP_SWEEP_PATTERN,
|
|
@@ -13,13 +12,13 @@ import {
|
|
|
13
12
|
} from './backup-sweep';
|
|
14
13
|
import { InFlightError } from './module-operations';
|
|
15
14
|
|
|
16
|
-
function moduleWith(id: string, schedule?:
|
|
15
|
+
function moduleWith(id: string, schedule?: string, scheduleOverride?: string): BackupSweepModule {
|
|
17
16
|
const manifest = {
|
|
18
17
|
id,
|
|
19
18
|
hooks: { on_backup: { script: 'backup.ts' } },
|
|
20
19
|
...(schedule ? { backup: { schedule } } : {}),
|
|
21
20
|
} as unknown as ModuleManifest;
|
|
22
|
-
return { id, manifest };
|
|
21
|
+
return { id, manifest, scheduleOverride };
|
|
23
22
|
}
|
|
24
23
|
|
|
25
24
|
function deps(
|
|
@@ -123,6 +122,26 @@ describe('runBackupSweep', () => {
|
|
|
123
122
|
expect(report.skippedManual).toEqual(['scratch']);
|
|
124
123
|
});
|
|
125
124
|
|
|
125
|
+
test("an operator's override decides the cadence, not the manifest", async () => {
|
|
126
|
+
const d = deps([moduleWith('caddy', 'daily', '6h')]);
|
|
127
|
+
const seen: Array<[string, unknown]> = [];
|
|
128
|
+
d.isDue = (moduleId, schedule) => {
|
|
129
|
+
seen.push([moduleId, schedule]);
|
|
130
|
+
return true;
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
await runBackupSweep(d);
|
|
134
|
+
|
|
135
|
+
expect(seen).toEqual([['caddy', { minutes: 360 }]]);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test('an override of manual stops a module the manifest wanted backed up', async () => {
|
|
139
|
+
const report = await runBackupSweep(deps([moduleWith('caddy', 'daily', 'manual')]));
|
|
140
|
+
|
|
141
|
+
expect(report.backedUp).toEqual([]);
|
|
142
|
+
expect(report.skippedManual).toEqual(['caddy']);
|
|
143
|
+
});
|
|
144
|
+
|
|
126
145
|
test('an undeclared schedule is backed up, not treated as manual', async () => {
|
|
127
146
|
// The regression this whole subsystem exists for: forgejo and signal
|
|
128
147
|
// declare no `backup:` block and had never been backed up.
|
|
@@ -17,12 +17,16 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import type { ModuleManifest } from '../manifest/schema';
|
|
20
|
-
import {
|
|
20
|
+
import { effectiveBackupSchedule } from './backup-schedule';
|
|
21
21
|
import type { ReapStagingReport, ResolveAbandonedReport } from './backup-staging';
|
|
22
|
+
import { BACKUP_SWEEP_PATTERN, type Cadence } from './cadence';
|
|
22
23
|
import { InFlightError } from './module-operations';
|
|
23
24
|
|
|
24
25
|
export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
|
|
25
|
-
|
|
26
|
+
// The tick itself lives in services/cadence.ts, next to the floor derived from
|
|
27
|
+
// it — a sweep whose tick and whose finest servable cadence are stated in two
|
|
28
|
+
// files is the pair that drifts.
|
|
29
|
+
export { BACKUP_SWEEP_PATTERN };
|
|
26
30
|
|
|
27
31
|
/**
|
|
28
32
|
* How long the sweep may run before the dispatcher kills it.
|
|
@@ -99,12 +103,18 @@ export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
|
|
|
99
103
|
export interface BackupSweepModule {
|
|
100
104
|
id: string;
|
|
101
105
|
manifest: ModuleManifest;
|
|
106
|
+
/**
|
|
107
|
+
* The operator's `backup_schedule` override, or undefined when they have set
|
|
108
|
+
* none. Carried rather than pre-resolved so the cadence is resolved through
|
|
109
|
+
* the one shared accessor here, the same way the freshness audit resolves it.
|
|
110
|
+
*/
|
|
111
|
+
scheduleOverride: string | undefined;
|
|
102
112
|
}
|
|
103
113
|
|
|
104
114
|
export interface BackupSweepDeps {
|
|
105
115
|
/** Installed modules that declare an `on_backup` hook. */
|
|
106
116
|
listEligible(): BackupSweepModule[];
|
|
107
|
-
isDue(moduleId: string, schedule:
|
|
117
|
+
isDue(moduleId: string, schedule: Cadence): boolean;
|
|
108
118
|
backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
|
|
109
119
|
/** Apply the module's declared retention. No-op when it declares none. */
|
|
110
120
|
prune(module: BackupSweepModule): Promise<void>;
|
|
@@ -126,7 +136,7 @@ export interface BackupSweepReport {
|
|
|
126
136
|
/** Records corrected from a stale `in_progress`. */
|
|
127
137
|
records: ResolveAbandonedReport;
|
|
128
138
|
backedUp: string[];
|
|
129
|
-
/**
|
|
139
|
+
/** Effective cadence `manual` — the operator or the author opted out. */
|
|
130
140
|
skippedManual: string[];
|
|
131
141
|
skippedNotDue: string[];
|
|
132
142
|
/** Another module operation held the lock. Not a failure; retried next tick. */
|
|
@@ -157,7 +167,7 @@ export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweep
|
|
|
157
167
|
};
|
|
158
168
|
|
|
159
169
|
for (const module of deps.listEligible()) {
|
|
160
|
-
const schedule = effectiveBackupSchedule(module.manifest);
|
|
170
|
+
const schedule = effectiveBackupSchedule(module.manifest, module.scheduleOverride);
|
|
161
171
|
if (schedule === 'manual') {
|
|
162
172
|
report.skippedManual.push(module.id);
|
|
163
173
|
continue;
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import {
|
|
3
|
+
ALERTING_SWEEP_PATTERN,
|
|
4
|
+
BACKUP_CADENCE_FLOOR_MINUTES,
|
|
5
|
+
BACKUP_SWEEP_PATTERN,
|
|
6
|
+
MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
7
|
+
cadenceMs,
|
|
8
|
+
cadenceSchema,
|
|
9
|
+
formatCadence,
|
|
10
|
+
parseCadence,
|
|
11
|
+
tickIntervalMinutes,
|
|
12
|
+
} from './cadence';
|
|
13
|
+
|
|
14
|
+
describe('parseCadence', () => {
|
|
15
|
+
test('every named period equals its duration equivalent', () => {
|
|
16
|
+
expect(parseCadence('hourly')).toEqual(parseCadence('1h'));
|
|
17
|
+
expect(parseCadence('daily')).toEqual(parseCadence('24h'));
|
|
18
|
+
expect(parseCadence('weekly')).toEqual(parseCadence('7d'));
|
|
19
|
+
expect(parseCadence('monthly')).toEqual(parseCadence('30d'));
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test('durations normalise to minutes', () => {
|
|
23
|
+
expect(parseCadence('90m')).toEqual({ minutes: 90 });
|
|
24
|
+
expect(parseCadence('6h')).toEqual({ minutes: 360 });
|
|
25
|
+
expect(parseCadence('3d')).toEqual({ minutes: 4320 });
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test('manual is its own value, not a number', () => {
|
|
29
|
+
expect(parseCadence('manual')).toBe('manual');
|
|
30
|
+
expect(cadenceMs('manual')).toBe(Number.POSITIVE_INFINITY);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test('rejects malformed, zero and negative values', () => {
|
|
34
|
+
for (const bad of ['dailyy', '', '6', 'h6', '6w', '0h', '-1h', '1.5h', 'never']) {
|
|
35
|
+
expect(parseCadence(bad)).toBeNull();
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
describe('formatCadence', () => {
|
|
41
|
+
test('prefers the word an operator would have typed', () => {
|
|
42
|
+
expect(formatCadence({ minutes: 60 })).toBe('hourly');
|
|
43
|
+
expect(formatCadence({ minutes: 1440 })).toBe('daily');
|
|
44
|
+
expect(formatCadence('manual')).toBe('manual');
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test('falls back to the coarsest exact duration', () => {
|
|
48
|
+
expect(formatCadence({ minutes: 360 })).toBe('6h');
|
|
49
|
+
expect(formatCadence({ minutes: 2880 })).toBe('2d');
|
|
50
|
+
expect(formatCadence({ minutes: 90 })).toBe('90m');
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe('the floor follows the sweep tick', () => {
|
|
55
|
+
test('each floor is the tick of the sweep that would serve it', () => {
|
|
56
|
+
expect(BACKUP_CADENCE_FLOOR_MINUTES).toBe(tickIntervalMinutes(BACKUP_SWEEP_PATTERN));
|
|
57
|
+
expect(MONITOR_INTERVAL_FLOOR_MINUTES).toBe(tickIntervalMinutes(ALERTING_SWEEP_PATTERN));
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test('a finer tick admits a cadence the coarser one refused', () => {
|
|
61
|
+
const coarse = cadenceSchema({ floorMinutes: tickIntervalMinutes('timer.tick.1h') });
|
|
62
|
+
const fine = cadenceSchema({ floorMinutes: tickIntervalMinutes('timer.tick.5m') });
|
|
63
|
+
expect(coarse.safeParse('15m').success).toBe(false);
|
|
64
|
+
expect(fine.safeParse('15m').success).toBe(true);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('a pattern that is not a timer tick is a programming error, not a floor of zero', () => {
|
|
68
|
+
expect(() => tickIntervalMinutes('module.deployed')).toThrow();
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
describe('cadenceSchema', () => {
|
|
73
|
+
const schema = cadenceSchema({ floorMinutes: 60 });
|
|
74
|
+
|
|
75
|
+
test('accepts words, durations at or above the floor, and manual', () => {
|
|
76
|
+
for (const good of ['hourly', 'daily', 'weekly', 'monthly', 'manual', '6h', '2d', '60m']) {
|
|
77
|
+
expect(schema.safeParse(good).success).toBe(true);
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test('names the permitted forms when the value is unparseable', () => {
|
|
82
|
+
const result = schema.safeParse('dailyy');
|
|
83
|
+
expect(result.success).toBe(false);
|
|
84
|
+
if (!result.success) {
|
|
85
|
+
expect(result.error.issues[0].message).toContain('named period');
|
|
86
|
+
expect(result.error.issues[0].message).toContain('manual');
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test('names the finest servable cadence when the value is below the floor', () => {
|
|
91
|
+
const result = schema.safeParse('5m');
|
|
92
|
+
expect(result.success).toBe(false);
|
|
93
|
+
if (!result.success) {
|
|
94
|
+
expect(result.error.issues[0].message).toContain('hourly');
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
});
|