@celilo/cli 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +61 -9
- package/drizzle/0024_module_pause.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +7 -7
- package/src/__integration__/container-services-cli.integration.test.ts +8 -2
- package/src/api/remote-client.test.ts +6 -5
- package/src/api/serve.ts +41 -7
- package/src/api-clients/proxmox.ts +34 -0
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/alerts-sweep.ts +2 -0
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/events.ts +34 -3
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +64 -2
- package/src/cli/commands/module-config.ts +159 -8
- package/src/cli/commands/module-deploy.ts +2 -2
- package/src/cli/commands/module-health.ts +1 -0
- package/src/cli/commands/module-import.ts +3 -3
- package/src/cli/commands/module-list.ts +12 -1
- package/src/cli/commands/module-pause.ts +317 -0
- package/src/cli/commands/module-remove.ts +78 -40
- package/src/cli/commands/module-status.ts +127 -4
- package/src/cli/commands/module-update.test.ts +1 -1
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/proxmox-template-selection.ts +1 -1
- package/src/cli/commands/status.ts +25 -3
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +39 -9
- package/src/cli/fuel-gauge.ts +4 -4
- package/src/cli/index.ts +104 -22
- package/src/cli/json-output.test.ts +162 -0
- package/src/cli/prompts.ts +53 -74
- package/src/cli/service-credential.ts +3 -3
- package/src/cli/stdout-is-undecorated.test.ts +94 -0
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/cli/types.ts +7 -2
- package/src/db/schema.ts +73 -15
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/run-named-hook.ts +28 -0
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/suppression.test.ts +5 -0
- package/src/services/alerting/suppression.ts +18 -1
- package/src/services/alerting/sweep-runner.test.ts +52 -3
- package/src/services/alerting/sweep-runner.ts +41 -8
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/bus-interview.ts +2 -2
- package/src/services/bus-secret-flow.test.ts +1 -1
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/fleet-checks.ts +48 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/module-deploy.ts +1 -1
- package/src/services/module-pause-observability.test.ts +224 -0
- package/src/services/module-pause-quiescence.test.ts +163 -0
- package/src/services/module-pause.test.ts +573 -0
- package/src/services/module-pause.ts +544 -0
- package/src/services/remove-guard.test.ts +175 -0
- package/src/services/remove-guard.ts +109 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/terminal-responder.ts +16 -16
- package/src/services/update/dep-graph.test.ts +33 -4
- package/src/services/update/dep-graph.ts +39 -17
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +32 -49
- package/src/test-utils/cli.ts +15 -14
- package/src/test-utils/integration-guard.ts +26 -0
- package/src/test-utils/setup-test-db.ts +13 -23
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
3
|
+
import {
|
|
4
|
+
type DependentCandidate,
|
|
5
|
+
describeRemovalRefusal,
|
|
6
|
+
findRemovalBlockers,
|
|
7
|
+
} from './remove-guard';
|
|
8
|
+
|
|
9
|
+
function manifest(
|
|
10
|
+
id: string,
|
|
11
|
+
opts: { requires?: string[]; optional?: string[] } = {},
|
|
12
|
+
): ModuleManifest {
|
|
13
|
+
return {
|
|
14
|
+
id,
|
|
15
|
+
name: id,
|
|
16
|
+
version: '1.0.0',
|
|
17
|
+
celilo_contract: '1.0',
|
|
18
|
+
provides: { capabilities: [] },
|
|
19
|
+
requires: { capabilities: (opts.requires ?? []).map((name) => ({ name, version: '1.0.0' })) },
|
|
20
|
+
optional: opts.optional
|
|
21
|
+
? { capabilities: opts.optional.map((name) => ({ name, version: '1.0.0' })) }
|
|
22
|
+
: undefined,
|
|
23
|
+
} as unknown as ModuleManifest;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function candidate(
|
|
27
|
+
id: string,
|
|
28
|
+
opts: { requires?: string[]; optional?: string[]; paused?: boolean; deployed?: boolean } = {},
|
|
29
|
+
): DependentCandidate {
|
|
30
|
+
return {
|
|
31
|
+
id,
|
|
32
|
+
manifest: manifest(id, opts),
|
|
33
|
+
paused: opts.paused ?? false,
|
|
34
|
+
deployed: opts.deployed ?? true,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** What `greenwave` provides, and the consumers that wedged its removal. */
|
|
39
|
+
const GREENWAVE_CAPABILITIES = ['firewall', 'dhcp_server'];
|
|
40
|
+
|
|
41
|
+
describe('findRemovalBlockers — the guard that wedged the greenwave swap', () => {
|
|
42
|
+
test('a live consumer blocks removal', () => {
|
|
43
|
+
const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
44
|
+
candidate('caddy', { requires: ['firewall'] }),
|
|
45
|
+
]);
|
|
46
|
+
expect(blockers).toEqual([{ moduleId: 'caddy', capability: 'firewall', kind: 'requires' }]);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('a PAUSED consumer does not block (task 5.1)', () => {
|
|
50
|
+
// Sound because unpause cannot return it to service without a redeploy, and
|
|
51
|
+
// a redeploy re-resolves its capabilities against whatever is present then.
|
|
52
|
+
const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
53
|
+
candidate('caddy', { requires: ['firewall'], paused: true }),
|
|
54
|
+
]);
|
|
55
|
+
expect(blockers).toEqual([]);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('a PARTIALLY paused dependent set still blocks, naming only the live ones', () => {
|
|
59
|
+
const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
60
|
+
candidate('caddy', { requires: ['firewall'], paused: true }),
|
|
61
|
+
candidate('authentik', { requires: ['firewall'] }),
|
|
62
|
+
]);
|
|
63
|
+
expect(blockers.map((b) => b.moduleId)).toEqual(['authentik']);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
describe('findRemovalBlockers — the optional-dependency regression (task 5.4)', () => {
|
|
68
|
+
// technitium consumes dhcp_server under `optional:`, so the old guard —
|
|
69
|
+
// which read `requires` alone — let a greenwave removal orphan its
|
|
70
|
+
// DHCP-handed-out DNS silently while correctly blocking on caddy.
|
|
71
|
+
test('an optional consumer NOW blocks, where it used to be invisible', () => {
|
|
72
|
+
const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
73
|
+
candidate('technitium', { optional: ['dhcp_server'] }),
|
|
74
|
+
]);
|
|
75
|
+
expect(blockers).toEqual([
|
|
76
|
+
{ moduleId: 'technitium', capability: 'dhcp_server', kind: 'optional' },
|
|
77
|
+
]);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test('and stops blocking once paused', () => {
|
|
81
|
+
const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
82
|
+
candidate('technitium', { optional: ['dhcp_server'], paused: true }),
|
|
83
|
+
]);
|
|
84
|
+
expect(blockers).toEqual([]);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test('a module depending BOTH ways is reported once, as the stronger claim', () => {
|
|
88
|
+
const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
89
|
+
candidate('both', { requires: ['firewall'], optional: ['dhcp_server'] }),
|
|
90
|
+
]);
|
|
91
|
+
expect(blockers).toHaveLength(1);
|
|
92
|
+
expect(blockers[0].kind).toBe('requires');
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
describe('findRemovalBlockers — deliberate non-goals', () => {
|
|
97
|
+
test('another provider existing does NOT by itself permit removal (celilo#683, task 5.5)', () => {
|
|
98
|
+
// `axon` provides the same capabilities as `greenwave`. The guard still
|
|
99
|
+
// refuses, because celilo does not reason about provider substitutability —
|
|
100
|
+
// pausing is the mechanism this change relies on instead.
|
|
101
|
+
const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
102
|
+
candidate('axon'),
|
|
103
|
+
candidate('caddy', { requires: ['firewall'] }),
|
|
104
|
+
]);
|
|
105
|
+
expect(blockers.map((b) => b.moduleId)).toEqual(['caddy']);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test('a module consuming an unrelated capability is not a dependent', () => {
|
|
109
|
+
expect(
|
|
110
|
+
findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
111
|
+
candidate('blog', { requires: ['public_web'] }),
|
|
112
|
+
]),
|
|
113
|
+
).toEqual([]);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test('a provider of nothing blocks nobody', () => {
|
|
117
|
+
expect(findRemovalBlockers([], [candidate('caddy', { requires: ['firewall'] })])).toEqual([]);
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
describe('describeRemovalRefusal', () => {
|
|
122
|
+
test('names each blocker AND which declaration makes it one (task 5.3)', () => {
|
|
123
|
+
const message = describeRemovalRefusal('greenwave', [
|
|
124
|
+
{ moduleId: 'caddy', capability: 'firewall', kind: 'requires' },
|
|
125
|
+
{ moduleId: 'technitium', capability: 'dhcp_server', kind: 'optional' },
|
|
126
|
+
]);
|
|
127
|
+
|
|
128
|
+
// The operator needs to tell a hard prerequisite from an optional consumer
|
|
129
|
+
// to judge what pausing it implies.
|
|
130
|
+
expect(message).toContain("caddy — requires 'firewall'");
|
|
131
|
+
expect(message).toContain("technitium — optional 'dhcp_server'");
|
|
132
|
+
// And it must point at the way out, since that is the whole change.
|
|
133
|
+
expect(message).toContain('celilo module pause --cascade greenwave');
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
describe('an undeployed dependent does not block (guard/cascade agreement)', () => {
|
|
138
|
+
// The spec says removal is refused for another INSTALLED module. An imported
|
|
139
|
+
// module has never resolved the capability and cannot be orphaned by the
|
|
140
|
+
// provider going away.
|
|
141
|
+
//
|
|
142
|
+
// This is load-bearing for D3's guard/cascade agreement, not a nicety.
|
|
143
|
+
// `pause --cascade` SKIPS undeployed members — pausing needs a settled
|
|
144
|
+
// deployed state and there is nothing to quiesce. If the guard still counted
|
|
145
|
+
// them, the operator would pause the whole cascade and the removal would
|
|
146
|
+
// still refuse, naming a module that pause is structurally unable to act on.
|
|
147
|
+
// That is a dead end: no sequence of commands gets the operator out of it.
|
|
148
|
+
test('an IMPORTED consumer is not a blocker', () => {
|
|
149
|
+
expect(
|
|
150
|
+
findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
151
|
+
candidate('technitium', { optional: ['dhcp_server'], deployed: false }),
|
|
152
|
+
]),
|
|
153
|
+
).toEqual([]);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test('but a DEPLOYED one still is', () => {
|
|
157
|
+
expect(
|
|
158
|
+
findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
159
|
+
candidate('technitium', { optional: ['dhcp_server'], deployed: true }),
|
|
160
|
+
]).map((b) => b.moduleId),
|
|
161
|
+
).toEqual(['technitium']);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test('the guard and the cascade agree on every combination', () => {
|
|
165
|
+
// Whatever pause can act on, the guard must count; whatever pause skips,
|
|
166
|
+
// the guard must ignore. Enumerated so a future change to either side
|
|
167
|
+
// cannot silently break the pairing.
|
|
168
|
+
const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
|
|
169
|
+
candidate('live', { requires: ['firewall'] }),
|
|
170
|
+
candidate('paused', { requires: ['firewall'], paused: true }),
|
|
171
|
+
candidate('imported', { requires: ['firewall'], deployed: false }),
|
|
172
|
+
]);
|
|
173
|
+
expect(blockers.map((b) => b.moduleId)).toEqual(['live']);
|
|
174
|
+
});
|
|
175
|
+
});
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The dependency guard on `module remove`.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `cli/commands/module-remove.ts` so it can be unit-tested
|
|
5
|
+
* without a database, and so the two behaviour changes it carries are visible
|
|
6
|
+
* in one place (openspec/changes/module-pause-lifecycle, design D3):
|
|
7
|
+
*
|
|
8
|
+
* 1. A PAUSED module is not a dependent. Sound rather than a loosening: a
|
|
9
|
+
* paused module cannot return to service without a redeploy, and that
|
|
10
|
+
* redeploy re-resolves its capabilities against whatever providers exist
|
|
11
|
+
* at the time. The invariant is "paused implies not currently bound, and
|
|
12
|
+
* guaranteed to rebind before going live".
|
|
13
|
+
*
|
|
14
|
+
* 2. A dependent is one that declares the capability under `requires` OR
|
|
15
|
+
* `optional`, which is what `services/update/dep-graph.ts` already means
|
|
16
|
+
* by an edge. The guard used to consider `requires` alone, and the two
|
|
17
|
+
* definitions disagreeing was silently harmful: `technitium` consumes
|
|
18
|
+
* `dhcp_server` under `optional:`, so removing its provider orphaned its
|
|
19
|
+
* DHCP-handed-out DNS without a word while correctly blocking on `caddy`.
|
|
20
|
+
* The guard and the cascade MUST agree on the set, or `pause --cascade`
|
|
21
|
+
* pauses a set the guard still rejects.
|
|
22
|
+
*
|
|
23
|
+
* Note (2) makes the guard STRICTER for optional consumers while (1) makes it
|
|
24
|
+
* looser for paused ones. Both directions are intended.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
28
|
+
|
|
29
|
+
/** How a module came to depend on the capability — reported, not just counted. */
|
|
30
|
+
export type DependencyKind = 'requires' | 'optional';
|
|
31
|
+
|
|
32
|
+
export interface Blocker {
|
|
33
|
+
moduleId: string;
|
|
34
|
+
capability: string;
|
|
35
|
+
kind: DependencyKind;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface DependentCandidate {
|
|
39
|
+
id: string;
|
|
40
|
+
manifest: ModuleManifest;
|
|
41
|
+
paused: boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Has this module actually been deployed? The spec says removal is refused
|
|
44
|
+
* when another **installed** module depends on the capability, and an
|
|
45
|
+
* imported-but-never-deployed module is not installed: it has never resolved
|
|
46
|
+
* the capability, holds no state derived from it, and cannot be orphaned by
|
|
47
|
+
* the provider going away.
|
|
48
|
+
*
|
|
49
|
+
* This is not a nicety — it is required for the guard and the cascade to
|
|
50
|
+
* agree (design D3). `pause --cascade` skips undeployed members, because
|
|
51
|
+
* pausing requires a settled deployed state and there is nothing to quiesce.
|
|
52
|
+
* If the guard still counted them, the operator would pause the full cascade
|
|
53
|
+
* and the removal would STILL refuse, naming a module that pause cannot act
|
|
54
|
+
* on — a dead end with no way forward.
|
|
55
|
+
*/
|
|
56
|
+
deployed: boolean;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Modules that block removal of a provider of `providedCapabilities`.
|
|
61
|
+
*
|
|
62
|
+
* Pure. Paused candidates are excluded; a candidate depending via both
|
|
63
|
+
* `requires` and `optional` is reported once, as `requires` (the stronger
|
|
64
|
+
* claim, and the one whose removal consequence is worse).
|
|
65
|
+
*/
|
|
66
|
+
export function findRemovalBlockers(
|
|
67
|
+
providedCapabilities: Iterable<string>,
|
|
68
|
+
candidates: DependentCandidate[],
|
|
69
|
+
): Blocker[] {
|
|
70
|
+
const provided = new Set(providedCapabilities);
|
|
71
|
+
if (provided.size === 0) return [];
|
|
72
|
+
|
|
73
|
+
const blockers: Blocker[] = [];
|
|
74
|
+
for (const candidate of candidates) {
|
|
75
|
+
if (candidate.paused) continue;
|
|
76
|
+
if (!candidate.deployed) continue;
|
|
77
|
+
|
|
78
|
+
const required = (candidate.manifest.requires?.capabilities ?? []).map((c) => c.name);
|
|
79
|
+
const optional = (candidate.manifest.optional?.capabilities ?? []).map((c) => c.name);
|
|
80
|
+
|
|
81
|
+
const hit =
|
|
82
|
+
required.find((name) => provided.has(name)) ?? optional.find((name) => provided.has(name));
|
|
83
|
+
if (!hit) continue;
|
|
84
|
+
|
|
85
|
+
blockers.push({
|
|
86
|
+
moduleId: candidate.id,
|
|
87
|
+
capability: hit,
|
|
88
|
+
kind: required.includes(hit) ? 'requires' : 'optional',
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return blockers.sort((a, b) => a.moduleId.localeCompare(b.moduleId));
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The refusal an operator reads. Names each blocker AND which declaration makes
|
|
97
|
+
* it one, so a hard prerequisite is distinguishable from an optional consumer —
|
|
98
|
+
* the operator needs that to judge what pausing it implies.
|
|
99
|
+
*/
|
|
100
|
+
export function describeRemovalRefusal(moduleId: string, blockers: Blocker[]): string {
|
|
101
|
+
const lines = blockers.map((b) => ` • ${b.moduleId} — ${b.kind} '${b.capability}'`);
|
|
102
|
+
return [
|
|
103
|
+
`Cannot remove '${moduleId}': the following installed modules depend on its capabilities:`,
|
|
104
|
+
...lines,
|
|
105
|
+
'',
|
|
106
|
+
'Pause them first so they rebind on their next deploy:',
|
|
107
|
+
` celilo module pause --cascade ${moduleId}`,
|
|
108
|
+
].join('\n');
|
|
109
|
+
}
|
|
@@ -1,17 +1,26 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Recurrence
|
|
3
|
-
* self-describing, replayable Buffer body — NOT a one-shot Node read stream.
|
|
2
|
+
* Recurrence gates for the two ways this provider's upload has been wrong.
|
|
4
3
|
*
|
|
5
|
-
*
|
|
6
|
-
* no ContentLength
|
|
7
|
-
* with "The request body terminated unexpectedly"
|
|
8
|
-
* stream across S3's retries
|
|
9
|
-
* (length known) and so never exercised the broken path
|
|
10
|
-
*
|
|
11
|
-
*
|
|
4
|
+
* ISS-0016 — a one-shot `createReadStream` passed straight to PutObject as
|
|
5
|
+
* `Body` with no ContentLength. AWS SDK v3 fell back to aws-chunked streaming
|
|
6
|
+
* and failed with "The request body terminated unexpectedly", and could not
|
|
7
|
+
* replay the stream across S3's retries and redirects. The verify path used a
|
|
8
|
+
* string body (length known) and so never exercised the broken path, which is
|
|
9
|
+
* why every real S3 backup silently failed.
|
|
10
|
+
*
|
|
11
|
+
* celilo#685 — the fix for ISS-0016 was `readFileSync` into one Buffer, which
|
|
12
|
+
* is replayable and was fine for the system envelopes this provider carried at
|
|
13
|
+
* the time. Module backups then started using the same provider at a thousand
|
|
14
|
+
* times the size, and a ~1.9 GB Buffer on a 3784 MB management server was
|
|
15
|
+
* OOM-killed every hour for a day.
|
|
16
|
+
*
|
|
17
|
+
* The two constrain opposite things — replayable versus not resident — so
|
|
18
|
+
* neither test is meaningful alone, and satisfying one by breaking the other is
|
|
19
|
+
* exactly the history here. `Upload` (multipart) satisfies both: each PART is a
|
|
20
|
+
* replayable buffer, and only a bounded number of parts exist at once.
|
|
12
21
|
*
|
|
13
22
|
* No live S3 / MinIO harness exists, so we intercept S3Client.prototype.send
|
|
14
|
-
* and inspect the
|
|
23
|
+
* and inspect the commands the provider builds.
|
|
15
24
|
*/
|
|
16
25
|
|
|
17
26
|
import { type Mock, afterEach, describe, expect, it, spyOn } from 'bun:test';
|
|
@@ -19,8 +28,20 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
|
19
28
|
import { tmpdir } from 'node:os';
|
|
20
29
|
import { join } from 'node:path';
|
|
21
30
|
import { Readable } from 'node:stream';
|
|
22
|
-
import {
|
|
23
|
-
|
|
31
|
+
import {
|
|
32
|
+
CompleteMultipartUploadCommand,
|
|
33
|
+
CreateMultipartUploadCommand,
|
|
34
|
+
GetObjectCommand,
|
|
35
|
+
PutObjectCommand,
|
|
36
|
+
S3Client,
|
|
37
|
+
UploadPartCommand,
|
|
38
|
+
} from '@aws-sdk/client-s3';
|
|
39
|
+
import {
|
|
40
|
+
type S3StorageConfig,
|
|
41
|
+
UPLOAD_PART_SIZE,
|
|
42
|
+
UPLOAD_QUEUE_SIZE,
|
|
43
|
+
createS3StorageProvider,
|
|
44
|
+
} from './s3';
|
|
24
45
|
|
|
25
46
|
const CONFIG: S3StorageConfig = {
|
|
26
47
|
bucket: 'test-bucket',
|
|
@@ -44,7 +65,7 @@ describe('s3 storage provider (ISS-0016)', () => {
|
|
|
44
65
|
}
|
|
45
66
|
});
|
|
46
67
|
|
|
47
|
-
it('uploads a
|
|
68
|
+
it('uploads a small file as a self-describing Buffer body, not a stream (ISS-0016)', async () => {
|
|
48
69
|
dir = mkdtempSync(join(tmpdir(), 's3-upload-'));
|
|
49
70
|
const file = join(dir, 'envelope.tar.gz');
|
|
50
71
|
// A multi-KB real file — the case that broke against live S3.
|
|
@@ -61,6 +82,8 @@ describe('s3 storage provider (ISS-0016)', () => {
|
|
|
61
82
|
const provider = createS3StorageProvider(CONFIG);
|
|
62
83
|
await provider.upload(file, 'celilo-mgmt/2026/envelope.tar.gz');
|
|
63
84
|
|
|
85
|
+
// Under one part, so no multipart ceremony — a single PutObject, exactly
|
|
86
|
+
// as before. The bound below is what changed, not the small-file path.
|
|
64
87
|
expect(sent).toHaveLength(1);
|
|
65
88
|
const command = sent[0];
|
|
66
89
|
expect(command).toBeInstanceOf(PutObjectCommand);
|
|
@@ -78,6 +101,66 @@ describe('s3 storage provider (ISS-0016)', () => {
|
|
|
78
101
|
expect(command.input.Bucket).toBe('test-bucket');
|
|
79
102
|
});
|
|
80
103
|
|
|
104
|
+
it('never holds more than one part per queue slot, however large the file (celilo#685)', async () => {
|
|
105
|
+
dir = mkdtempSync(join(tmpdir(), 's3-upload-large-'));
|
|
106
|
+
const file = join(dir, 'backup.tar.enc');
|
|
107
|
+
|
|
108
|
+
// Deliberately larger than one part, so the multipart path runs for real.
|
|
109
|
+
// It does not need to approach forgejo's 1.87 GB: the property under test
|
|
110
|
+
// is that peak residency is set by the part size rather than by the file,
|
|
111
|
+
// and a file that spans several parts demonstrates that at any scale. A
|
|
112
|
+
// test that had to allocate the failing size to prove the fix would be
|
|
113
|
+
// reproducing the bug rather than gating it.
|
|
114
|
+
const parts = 3;
|
|
115
|
+
const fileSize = UPLOAD_PART_SIZE * parts;
|
|
116
|
+
writeFileSync(file, Buffer.alloc(fileSize, 0x7a));
|
|
117
|
+
|
|
118
|
+
// Deliberately NOT a running total. Parts are uploaded concurrently and can
|
|
119
|
+
// arrive in any order, so the only way to show the artifact survives is to
|
|
120
|
+
// keep each part against its number and reassemble.
|
|
121
|
+
const received = new Map<number, Buffer>();
|
|
122
|
+
let inFlight = 0;
|
|
123
|
+
let peakInFlight = 0;
|
|
124
|
+
sendSpy = spyOn(S3Client.prototype, 'send').mockImplementation(async (command: unknown) => {
|
|
125
|
+
if (command instanceof CreateMultipartUploadCommand) return { UploadId: 'upload-1' };
|
|
126
|
+
if (command instanceof CompleteMultipartUploadCommand) return {};
|
|
127
|
+
|
|
128
|
+
expect(command).toBeInstanceOf(UploadPartCommand);
|
|
129
|
+
const part = command as UploadPartCommand;
|
|
130
|
+
const body = part.input.Body as Buffer;
|
|
131
|
+
const partNumber = part.input.PartNumber as number;
|
|
132
|
+
|
|
133
|
+
// Each part is still a replayable Buffer — ISS-0016 holds per part.
|
|
134
|
+
expect(Buffer.isBuffer(body)).toBe(true);
|
|
135
|
+
// A part number reused would silently lose data on reassembly.
|
|
136
|
+
expect(received.has(partNumber)).toBe(false);
|
|
137
|
+
received.set(partNumber, Buffer.from(body));
|
|
138
|
+
|
|
139
|
+
inFlight += 1;
|
|
140
|
+
peakInFlight = Math.max(peakInFlight, inFlight);
|
|
141
|
+
await new Promise((resolve) => setTimeout(resolve, 1));
|
|
142
|
+
inFlight -= 1;
|
|
143
|
+
return { ETag: `"etag-${partNumber}"` };
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
const provider = createS3StorageProvider(CONFIG);
|
|
147
|
+
await provider.upload(file, 'forgejo/2026-08-13/backup.tar.enc');
|
|
148
|
+
|
|
149
|
+
// The bound. No single request ever carries the whole artifact, and no more
|
|
150
|
+
// than the queue depth are resident at once, so peak bytes is
|
|
151
|
+
// UPLOAD_PART_SIZE * UPLOAD_QUEUE_SIZE no matter how big the file gets.
|
|
152
|
+
expect(received.size).toBe(parts);
|
|
153
|
+
for (const body of received.values()) expect(body.length).toBeLessThanOrEqual(UPLOAD_PART_SIZE);
|
|
154
|
+
expect(peakInFlight).toBeLessThanOrEqual(UPLOAD_QUEUE_SIZE);
|
|
155
|
+
|
|
156
|
+
// A bound is only worth having if the artifact still arrives. Reassembled
|
|
157
|
+
// in part order, the bytes must be the file — sizes summing correctly would
|
|
158
|
+
// not catch a swapped or duplicated part, and a backup that restores to
|
|
159
|
+
// scrambled bytes is worse than one that fails loudly.
|
|
160
|
+
const ordered = [...received.entries()].sort(([a], [b]) => a - b).map(([, body]) => body);
|
|
161
|
+
expect(Buffer.concat(ordered).equals(readFileSync(file))).toBe(true);
|
|
162
|
+
});
|
|
163
|
+
|
|
81
164
|
it('downloads a multi-chunk response body to disk intact (short-read safe)', async () => {
|
|
82
165
|
dir = mkdtempSync(join(tmpdir(), 's3-download-'));
|
|
83
166
|
const out = join(dir, 'restored.bin');
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Works with AWS S3, MinIO, Backblaze B2, Wasabi, and any S3-compatible service.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import {
|
|
6
|
+
import { createReadStream, createWriteStream } from 'node:fs';
|
|
7
7
|
import { mkdir } from 'node:fs/promises';
|
|
8
8
|
import { dirname } from 'node:path';
|
|
9
9
|
import { Readable } from 'node:stream';
|
|
@@ -15,10 +15,47 @@ import {
|
|
|
15
15
|
PutObjectCommand,
|
|
16
16
|
S3Client,
|
|
17
17
|
} from '@aws-sdk/client-s3';
|
|
18
|
+
import { Upload } from '@aws-sdk/lib-storage';
|
|
18
19
|
import type { StorageProvider, StorageVerifyResult } from './types';
|
|
19
20
|
|
|
20
21
|
const BACKUP_PREFIX = 'celilo-backups';
|
|
21
22
|
|
|
23
|
+
/**
|
|
24
|
+
* Multipart upload sizing. Peak resident bytes for an upload is
|
|
25
|
+
* `UPLOAD_PART_SIZE * UPLOAD_QUEUE_SIZE` — 64 MB — and does NOT grow with the
|
|
26
|
+
* artifact. That bound is the whole point of these two constants.
|
|
27
|
+
*
|
|
28
|
+
* `upload` used to `readFileSync` the artifact into one Buffer and hand it to
|
|
29
|
+
* PutObject. The comment justifying that read:
|
|
30
|
+
*
|
|
31
|
+
* > Upload a Buffer, NOT a read stream (ISS-0016). A streamed Body fails with
|
|
32
|
+
* > "The request body terminated unexpectedly" [...] Backup envelopes are
|
|
33
|
+
* > small (state + key, not provider binaries — ISS-0015), so buffering is
|
|
34
|
+
* > fine.
|
|
35
|
+
*
|
|
36
|
+
* The first half is true and still is: a one-shot Node stream as a PutObject
|
|
37
|
+
* `Body` cannot be replayed across the retries and redirects S3 issues, and
|
|
38
|
+
* with no ContentLength the SDK falls back to aws-chunked encoding on top.
|
|
39
|
+
* Passing `createReadStream` straight to PutObject would reintroduce exactly
|
|
40
|
+
* that bug.
|
|
41
|
+
*
|
|
42
|
+
* The second half stopped being true without anyone revisiting it. It was
|
|
43
|
+
* written when this provider only carried SYSTEM backups (celilo state plus a
|
|
44
|
+
* key, a few MB). MODULE backups now use the same provider and are three orders
|
|
45
|
+
* of magnitude larger — forgejo's envelope reached 1.87 GB — so "buffering is
|
|
46
|
+
* fine" became a ~1.9 GB Buffer on a 3784 MB management server, and the OOM
|
|
47
|
+
* killer took the backup every hour for a day while the on_backup hook itself
|
|
48
|
+
* reported success (celilo#685).
|
|
49
|
+
*
|
|
50
|
+
* `Upload` resolves the two halves rather than trading one for the other: it
|
|
51
|
+
* reads the stream a part at a time and each PART is a replayable buffer, so
|
|
52
|
+
* retries work without the whole object ever being resident. 16 MB parts keep
|
|
53
|
+
* a 1.87 GB artifact at ~117 requests, well inside S3's 10,000-part limit,
|
|
54
|
+
* which leaves headroom to ~160 GB.
|
|
55
|
+
*/
|
|
56
|
+
export const UPLOAD_PART_SIZE = 16 * 1024 * 1024;
|
|
57
|
+
export const UPLOAD_QUEUE_SIZE = 4;
|
|
58
|
+
|
|
22
59
|
export interface S3StorageConfig {
|
|
23
60
|
bucket: string;
|
|
24
61
|
region: string;
|
|
@@ -49,22 +86,18 @@ export function createS3StorageProvider(config: S3StorageConfig): StorageProvide
|
|
|
49
86
|
|
|
50
87
|
return {
|
|
51
88
|
async upload(localPath: string, remotePath: string): Promise<void> {
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
// binaries — ISS-0015), so buffering is fine.
|
|
60
|
-
const body = readFileSync(localPath);
|
|
61
|
-
await client.send(
|
|
62
|
-
new PutObjectCommand({
|
|
89
|
+
// Multipart, so peak memory is UPLOAD_PART_SIZE * UPLOAD_QUEUE_SIZE
|
|
90
|
+
// regardless of how large the artifact is.
|
|
91
|
+
await new Upload({
|
|
92
|
+
client,
|
|
93
|
+
partSize: UPLOAD_PART_SIZE,
|
|
94
|
+
queueSize: UPLOAD_QUEUE_SIZE,
|
|
95
|
+
params: {
|
|
63
96
|
Bucket: bucket,
|
|
64
97
|
Key: prefixedKey(remotePath),
|
|
65
|
-
Body:
|
|
66
|
-
}
|
|
67
|
-
);
|
|
98
|
+
Body: createReadStream(localPath),
|
|
99
|
+
},
|
|
100
|
+
}).done();
|
|
68
101
|
},
|
|
69
102
|
|
|
70
103
|
async download(remotePath: string, localPath: string): Promise<void> {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Terminal-responder: when a deploy runs on a TTY, this subscribes
|
|
3
3
|
* to `config.required.*`, `secret.required.*`, and `ensure.required.*`
|
|
4
|
-
* events and prompts the operator
|
|
5
|
-
* on the bus.
|
|
4
|
+
* events and prompts the operator on the terminal for each one,
|
|
5
|
+
* replying on the bus.
|
|
6
6
|
*
|
|
7
7
|
* Just one of several responder shapes — the bus query/reply path is
|
|
8
8
|
* a race, and the terminal is one racer. Other racers (Claude
|
|
@@ -26,9 +26,9 @@
|
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
28
|
import { hostname } from 'node:os';
|
|
29
|
+
import { confirm, isCancel, multiselect, select } from '@celilo/cli-display';
|
|
29
30
|
import { type Bus, openBus } from '@celilo/event-bus';
|
|
30
31
|
import { defineEvents } from '@celilo/event-bus';
|
|
31
|
-
import * as p from '@clack/prompts';
|
|
32
32
|
import { log, promptPassword, promptText } from '../cli/prompts';
|
|
33
33
|
import { getEventBusPath } from '../config/paths';
|
|
34
34
|
import { getDb } from '../db/client';
|
|
@@ -61,7 +61,7 @@ export interface TerminalResponderHandle {
|
|
|
61
61
|
|
|
62
62
|
/**
|
|
63
63
|
* Register transient subscriptions on the three interview event
|
|
64
|
-
* families. For each event, prompt the operator
|
|
64
|
+
* families. For each event, prompt the operator on the terminal, optionally
|
|
65
65
|
* inject the secret value out-of-band, and reply on the bus.
|
|
66
66
|
*
|
|
67
67
|
* Returns a handle the caller can close() when the deploy completes.
|
|
@@ -103,9 +103,9 @@ export function startTerminalResponder(): TerminalResponderHandle {
|
|
|
103
103
|
|
|
104
104
|
if (payload.options && payload.options.length > 0) {
|
|
105
105
|
// Multi-select prompt for vars with options[] declared in the
|
|
106
|
-
// manifest.
|
|
107
|
-
//
|
|
108
|
-
const selected = await
|
|
106
|
+
// manifest. multiselect returns an array of selected values
|
|
107
|
+
// directly — no JSON-typing to coerce.
|
|
108
|
+
const selected = await multiselect({
|
|
109
109
|
message,
|
|
110
110
|
options: payload.options.map((opt) => ({
|
|
111
111
|
value: opt.value,
|
|
@@ -114,7 +114,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
|
|
|
114
114
|
})),
|
|
115
115
|
required: payload.required,
|
|
116
116
|
});
|
|
117
|
-
if (
|
|
117
|
+
if (isCancel(selected)) {
|
|
118
118
|
log.warn(
|
|
119
119
|
`Terminal responder: cancelled prompt for ${payload.module}.${payload.key}; no reply emitted`,
|
|
120
120
|
);
|
|
@@ -308,11 +308,11 @@ export function startTerminalResponder(): TerminalResponderHandle {
|
|
|
308
308
|
let value: unknown;
|
|
309
309
|
|
|
310
310
|
if (payload.kind === 'confirm') {
|
|
311
|
-
const answer = await
|
|
311
|
+
const answer = await confirm({
|
|
312
312
|
message,
|
|
313
313
|
initialValue: payload.defaultValue === 'true',
|
|
314
314
|
});
|
|
315
|
-
if (
|
|
315
|
+
if (isCancel(answer)) {
|
|
316
316
|
log.warn(
|
|
317
317
|
`Terminal responder: cancelled prompt for ${payload.scope}.${payload.key}; no reply emitted`,
|
|
318
318
|
);
|
|
@@ -320,7 +320,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
|
|
|
320
320
|
}
|
|
321
321
|
value = answer;
|
|
322
322
|
} else if (payload.kind === 'select') {
|
|
323
|
-
const answer = await
|
|
323
|
+
const answer = await select({
|
|
324
324
|
message,
|
|
325
325
|
options: (payload.options ?? []).map((opt) => ({
|
|
326
326
|
value: opt.value,
|
|
@@ -329,7 +329,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
|
|
|
329
329
|
})),
|
|
330
330
|
initialValue: payload.defaultValue,
|
|
331
331
|
});
|
|
332
|
-
if (
|
|
332
|
+
if (isCancel(answer)) {
|
|
333
333
|
log.warn(
|
|
334
334
|
`Terminal responder: cancelled prompt for ${payload.scope}.${payload.key}; no reply emitted`,
|
|
335
335
|
);
|
|
@@ -337,7 +337,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
|
|
|
337
337
|
}
|
|
338
338
|
value = answer;
|
|
339
339
|
} else if (payload.kind === 'multiselect') {
|
|
340
|
-
const answer = await
|
|
340
|
+
const answer = await multiselect({
|
|
341
341
|
message,
|
|
342
342
|
options: (payload.options ?? []).map((opt) => ({
|
|
343
343
|
value: opt.value,
|
|
@@ -346,7 +346,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
|
|
|
346
346
|
})),
|
|
347
347
|
required: payload.required,
|
|
348
348
|
});
|
|
349
|
-
if (
|
|
349
|
+
if (isCancel(answer)) {
|
|
350
350
|
log.warn(
|
|
351
351
|
`Terminal responder: cancelled prompt for ${payload.scope}.${payload.key}; no reply emitted`,
|
|
352
352
|
);
|
|
@@ -624,8 +624,8 @@ function describeTypeHint(type: ConfigRequiredPayload['type']): string | null {
|
|
|
624
624
|
}
|
|
625
625
|
|
|
626
626
|
function coerceValue(raw: string | undefined, type: ConfigRequiredPayload['type']): unknown {
|
|
627
|
-
//
|
|
628
|
-
// that up as an error so validate sees it and the responder can
|
|
627
|
+
// The prompt layer yields undefined when the user cancels (Ctrl+C).
|
|
628
|
+
// Bubble that up as an error so validate sees it and the responder can
|
|
629
629
|
// skip the reply rather than emit a malformed one.
|
|
630
630
|
if (raw === undefined) {
|
|
631
631
|
throw new Error('Cancelled');
|