@celilo/cli 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CELILO_CORE_MODULES.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +61 -9
  3. package/drizzle/0024_module_pause.sql +20 -0
  4. package/drizzle/meta/_journal.json +8 -1
  5. package/package.json +7 -7
  6. package/src/__integration__/container-services-cli.integration.test.ts +8 -2
  7. package/src/api/remote-client.test.ts +6 -5
  8. package/src/api/serve.ts +41 -7
  9. package/src/api-clients/proxmox.ts +34 -0
  10. package/src/cli/commands/alerts-act.ts +1 -1
  11. package/src/cli/commands/alerts-sweep.ts +2 -0
  12. package/src/cli/commands/backup-create.ts +26 -11
  13. package/src/cli/commands/backup-list.test.ts +83 -0
  14. package/src/cli/commands/backup-list.ts +67 -3
  15. package/src/cli/commands/backup-prune.ts +17 -17
  16. package/src/cli/commands/backup-sweep.ts +20 -8
  17. package/src/cli/commands/events.ts +34 -3
  18. package/src/cli/commands/firewall-interface-list.test.ts +85 -0
  19. package/src/cli/commands/firewall-interface-list.ts +123 -0
  20. package/src/cli/commands/machine-add.ts +30 -2
  21. package/src/cli/commands/module-config.test.ts +64 -2
  22. package/src/cli/commands/module-config.ts +159 -8
  23. package/src/cli/commands/module-deploy.ts +2 -2
  24. package/src/cli/commands/module-health.ts +1 -0
  25. package/src/cli/commands/module-import.ts +3 -3
  26. package/src/cli/commands/module-list.ts +12 -1
  27. package/src/cli/commands/module-pause.ts +317 -0
  28. package/src/cli/commands/module-remove.ts +78 -40
  29. package/src/cli/commands/module-status.ts +127 -4
  30. package/src/cli/commands/module-update.test.ts +1 -1
  31. package/src/cli/commands/monitor.ts +116 -19
  32. package/src/cli/commands/proxmox-template-selection.ts +1 -1
  33. package/src/cli/commands/status.ts +25 -3
  34. package/src/cli/commands/system-migrate.ts +14 -0
  35. package/src/cli/commands/system-update.ts +4 -1
  36. package/src/cli/completion.ts +39 -9
  37. package/src/cli/fuel-gauge.ts +4 -4
  38. package/src/cli/index.ts +104 -22
  39. package/src/cli/json-output.test.ts +162 -0
  40. package/src/cli/prompts.ts +53 -74
  41. package/src/cli/service-credential.ts +3 -3
  42. package/src/cli/stdout-is-undecorated.test.ts +94 -0
  43. package/src/cli/tui/audit-state.ts +2 -0
  44. package/src/cli/types.ts +7 -2
  45. package/src/db/schema.ts +73 -15
  46. package/src/hooks/capability-loader.ts +130 -4
  47. package/src/hooks/run-named-hook.ts +28 -0
  48. package/src/hooks/types.ts +2 -1
  49. package/src/manifest/contracts/v1.ts +16 -0
  50. package/src/manifest/schema.ts +40 -65
  51. package/src/services/alerting/builtin-monitors.test.ts +18 -10
  52. package/src/services/alerting/cadence-migration.test.ts +155 -0
  53. package/src/services/alerting/cadence-migration.ts +90 -0
  54. package/src/services/alerting/coverage-source.ts +8 -11
  55. package/src/services/alerting/deploy-hooks.test.ts +16 -7
  56. package/src/services/alerting/deploy-hooks.ts +11 -5
  57. package/src/services/alerting/health-cadence.test.ts +58 -0
  58. package/src/services/alerting/health-cadence.ts +128 -0
  59. package/src/services/alerting/health-coverage.ts +18 -8
  60. package/src/services/alerting/monitors.ts +50 -15
  61. package/src/services/alerting/suppression.test.ts +5 -0
  62. package/src/services/alerting/suppression.ts +18 -1
  63. package/src/services/alerting/sweep-runner.test.ts +52 -3
  64. package/src/services/alerting/sweep-runner.ts +41 -8
  65. package/src/services/audit/backup-source.ts +24 -1
  66. package/src/services/audit/backups.test.ts +95 -10
  67. package/src/services/audit/backups.ts +40 -37
  68. package/src/services/audit/interface-classification.test.ts +220 -0
  69. package/src/services/audit/interface-classification.ts +167 -0
  70. package/src/services/audit/types.ts +2 -1
  71. package/src/services/backup-age-agreement.test.ts +118 -0
  72. package/src/services/backup-create.ts +36 -30
  73. package/src/services/backup-metadata.ts +52 -1
  74. package/src/services/backup-retention.test.ts +123 -0
  75. package/src/services/backup-retention.ts +66 -5
  76. package/src/services/backup-schedule.test.ts +166 -0
  77. package/src/services/backup-schedule.ts +105 -15
  78. package/src/services/backup-staging.ts +14 -1
  79. package/src/services/backup-sweep.test.ts +22 -3
  80. package/src/services/backup-sweep.ts +15 -5
  81. package/src/services/bus-interview.ts +2 -2
  82. package/src/services/bus-secret-flow.test.ts +1 -1
  83. package/src/services/cadence.test.ts +97 -0
  84. package/src/services/cadence.ts +165 -0
  85. package/src/services/fleet-checks.ts +48 -0
  86. package/src/services/machine-detector.ts +23 -1
  87. package/src/services/module-config.ts +33 -0
  88. package/src/services/module-deploy.ts +1 -1
  89. package/src/services/module-pause-observability.test.ts +224 -0
  90. package/src/services/module-pause-quiescence.test.ts +163 -0
  91. package/src/services/module-pause.test.ts +573 -0
  92. package/src/services/module-pause.ts +544 -0
  93. package/src/services/remove-guard.test.ts +175 -0
  94. package/src/services/remove-guard.ts +109 -0
  95. package/src/services/storage-providers/s3.test.ts +96 -13
  96. package/src/services/storage-providers/s3.ts +48 -15
  97. package/src/services/terminal-responder.ts +16 -16
  98. package/src/services/update/dep-graph.test.ts +33 -4
  99. package/src/services/update/dep-graph.ts +39 -17
  100. package/src/services/zone-detector.test.ts +34 -3
  101. package/src/services/zone-detector.ts +32 -49
  102. package/src/test-utils/cli.ts +15 -14
  103. package/src/test-utils/integration-guard.ts +26 -0
  104. package/src/test-utils/setup-test-db.ts +13 -23
@@ -0,0 +1,175 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import type { ModuleManifest } from '../manifest/schema';
3
+ import {
4
+ type DependentCandidate,
5
+ describeRemovalRefusal,
6
+ findRemovalBlockers,
7
+ } from './remove-guard';
8
+
9
+ function manifest(
10
+ id: string,
11
+ opts: { requires?: string[]; optional?: string[] } = {},
12
+ ): ModuleManifest {
13
+ return {
14
+ id,
15
+ name: id,
16
+ version: '1.0.0',
17
+ celilo_contract: '1.0',
18
+ provides: { capabilities: [] },
19
+ requires: { capabilities: (opts.requires ?? []).map((name) => ({ name, version: '1.0.0' })) },
20
+ optional: opts.optional
21
+ ? { capabilities: opts.optional.map((name) => ({ name, version: '1.0.0' })) }
22
+ : undefined,
23
+ } as unknown as ModuleManifest;
24
+ }
25
+
26
+ function candidate(
27
+ id: string,
28
+ opts: { requires?: string[]; optional?: string[]; paused?: boolean; deployed?: boolean } = {},
29
+ ): DependentCandidate {
30
+ return {
31
+ id,
32
+ manifest: manifest(id, opts),
33
+ paused: opts.paused ?? false,
34
+ deployed: opts.deployed ?? true,
35
+ };
36
+ }
37
+
38
+ /** What `greenwave` provides, and the consumers that wedged its removal. */
39
+ const GREENWAVE_CAPABILITIES = ['firewall', 'dhcp_server'];
40
+
41
+ describe('findRemovalBlockers — the guard that wedged the greenwave swap', () => {
42
+ test('a live consumer blocks removal', () => {
43
+ const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
44
+ candidate('caddy', { requires: ['firewall'] }),
45
+ ]);
46
+ expect(blockers).toEqual([{ moduleId: 'caddy', capability: 'firewall', kind: 'requires' }]);
47
+ });
48
+
49
+ test('a PAUSED consumer does not block (task 5.1)', () => {
50
+ // Sound because unpause cannot return it to service without a redeploy, and
51
+ // a redeploy re-resolves its capabilities against whatever is present then.
52
+ const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
53
+ candidate('caddy', { requires: ['firewall'], paused: true }),
54
+ ]);
55
+ expect(blockers).toEqual([]);
56
+ });
57
+
58
+ test('a PARTIALLY paused dependent set still blocks, naming only the live ones', () => {
59
+ const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
60
+ candidate('caddy', { requires: ['firewall'], paused: true }),
61
+ candidate('authentik', { requires: ['firewall'] }),
62
+ ]);
63
+ expect(blockers.map((b) => b.moduleId)).toEqual(['authentik']);
64
+ });
65
+ });
66
+
67
+ describe('findRemovalBlockers — the optional-dependency regression (task 5.4)', () => {
68
+ // technitium consumes dhcp_server under `optional:`, so the old guard —
69
+ // which read `requires` alone — let a greenwave removal orphan its
70
+ // DHCP-handed-out DNS silently while correctly blocking on caddy.
71
+ test('an optional consumer NOW blocks, where it used to be invisible', () => {
72
+ const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
73
+ candidate('technitium', { optional: ['dhcp_server'] }),
74
+ ]);
75
+ expect(blockers).toEqual([
76
+ { moduleId: 'technitium', capability: 'dhcp_server', kind: 'optional' },
77
+ ]);
78
+ });
79
+
80
+ test('and stops blocking once paused', () => {
81
+ const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
82
+ candidate('technitium', { optional: ['dhcp_server'], paused: true }),
83
+ ]);
84
+ expect(blockers).toEqual([]);
85
+ });
86
+
87
+ test('a module depending BOTH ways is reported once, as the stronger claim', () => {
88
+ const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
89
+ candidate('both', { requires: ['firewall'], optional: ['dhcp_server'] }),
90
+ ]);
91
+ expect(blockers).toHaveLength(1);
92
+ expect(blockers[0].kind).toBe('requires');
93
+ });
94
+ });
95
+
96
+ describe('findRemovalBlockers — deliberate non-goals', () => {
97
+ test('another provider existing does NOT by itself permit removal (celilo#683, task 5.5)', () => {
98
+ // `axon` provides the same capabilities as `greenwave`. The guard still
99
+ // refuses, because celilo does not reason about provider substitutability —
100
+ // pausing is the mechanism this change relies on instead.
101
+ const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
102
+ candidate('axon'),
103
+ candidate('caddy', { requires: ['firewall'] }),
104
+ ]);
105
+ expect(blockers.map((b) => b.moduleId)).toEqual(['caddy']);
106
+ });
107
+
108
+ test('a module consuming an unrelated capability is not a dependent', () => {
109
+ expect(
110
+ findRemovalBlockers(GREENWAVE_CAPABILITIES, [
111
+ candidate('blog', { requires: ['public_web'] }),
112
+ ]),
113
+ ).toEqual([]);
114
+ });
115
+
116
+ test('a provider of nothing blocks nobody', () => {
117
+ expect(findRemovalBlockers([], [candidate('caddy', { requires: ['firewall'] })])).toEqual([]);
118
+ });
119
+ });
120
+
121
+ describe('describeRemovalRefusal', () => {
122
+ test('names each blocker AND which declaration makes it one (task 5.3)', () => {
123
+ const message = describeRemovalRefusal('greenwave', [
124
+ { moduleId: 'caddy', capability: 'firewall', kind: 'requires' },
125
+ { moduleId: 'technitium', capability: 'dhcp_server', kind: 'optional' },
126
+ ]);
127
+
128
+ // The operator needs to tell a hard prerequisite from an optional consumer
129
+ // to judge what pausing it implies.
130
+ expect(message).toContain("caddy — requires 'firewall'");
131
+ expect(message).toContain("technitium — optional 'dhcp_server'");
132
+ // And it must point at the way out, since that is the whole change.
133
+ expect(message).toContain('celilo module pause --cascade greenwave');
134
+ });
135
+ });
136
+
137
+ describe('an undeployed dependent does not block (guard/cascade agreement)', () => {
138
+ // The spec says removal is refused for another INSTALLED module. An imported
139
+ // module has never resolved the capability and cannot be orphaned by the
140
+ // provider going away.
141
+ //
142
+ // This is load-bearing for D3's guard/cascade agreement, not a nicety.
143
+ // `pause --cascade` SKIPS undeployed members — pausing needs a settled
144
+ // deployed state and there is nothing to quiesce. If the guard still counted
145
+ // them, the operator would pause the whole cascade and the removal would
146
+ // still refuse, naming a module that pause is structurally unable to act on.
147
+ // That is a dead end: no sequence of commands gets the operator out of it.
148
+ test('an IMPORTED consumer is not a blocker', () => {
149
+ expect(
150
+ findRemovalBlockers(GREENWAVE_CAPABILITIES, [
151
+ candidate('technitium', { optional: ['dhcp_server'], deployed: false }),
152
+ ]),
153
+ ).toEqual([]);
154
+ });
155
+
156
+ test('but a DEPLOYED one still is', () => {
157
+ expect(
158
+ findRemovalBlockers(GREENWAVE_CAPABILITIES, [
159
+ candidate('technitium', { optional: ['dhcp_server'], deployed: true }),
160
+ ]).map((b) => b.moduleId),
161
+ ).toEqual(['technitium']);
162
+ });
163
+
164
+ test('the guard and the cascade agree on every combination', () => {
165
+ // Whatever pause can act on, the guard must count; whatever pause skips,
166
+ // the guard must ignore. Enumerated so a future change to either side
167
+ // cannot silently break the pairing.
168
+ const blockers = findRemovalBlockers(GREENWAVE_CAPABILITIES, [
169
+ candidate('live', { requires: ['firewall'] }),
170
+ candidate('paused', { requires: ['firewall'], paused: true }),
171
+ candidate('imported', { requires: ['firewall'], deployed: false }),
172
+ ]);
173
+ expect(blockers.map((b) => b.moduleId)).toEqual(['live']);
174
+ });
175
+ });
@@ -0,0 +1,109 @@
1
+ /**
2
+ * The dependency guard on `module remove`.
3
+ *
4
+ * Extracted from `cli/commands/module-remove.ts` so it can be unit-tested
5
+ * without a database, and so the two behaviour changes it carries are visible
6
+ * in one place (openspec/changes/module-pause-lifecycle, design D3):
7
+ *
8
+ * 1. A PAUSED module is not a dependent. Sound rather than a loosening: a
9
+ * paused module cannot return to service without a redeploy, and that
10
+ * redeploy re-resolves its capabilities against whatever providers exist
11
+ * at the time. The invariant is "paused implies not currently bound, and
12
+ * guaranteed to rebind before going live".
13
+ *
14
+ * 2. A dependent is one that declares the capability under `requires` OR
15
+ * `optional`, which is what `services/update/dep-graph.ts` already means
16
+ * by an edge. The guard used to consider `requires` alone, and the two
17
+ * definitions disagreeing was silently harmful: `technitium` consumes
18
+ * `dhcp_server` under `optional:`, so removing its provider orphaned its
19
+ * DHCP-handed-out DNS without a word while correctly blocking on `caddy`.
20
+ * The guard and the cascade MUST agree on the set, or `pause --cascade`
21
+ * pauses a set the guard still rejects.
22
+ *
23
+ * Note (2) makes the guard STRICTER for optional consumers while (1) makes it
24
+ * looser for paused ones. Both directions are intended.
25
+ */
26
+
27
+ import type { ModuleManifest } from '../manifest/schema';
28
+
29
+ /** How a module came to depend on the capability — reported, not just counted. */
30
+ export type DependencyKind = 'requires' | 'optional';
31
+
32
+ export interface Blocker {
33
+ moduleId: string;
34
+ capability: string;
35
+ kind: DependencyKind;
36
+ }
37
+
38
+ export interface DependentCandidate {
39
+ id: string;
40
+ manifest: ModuleManifest;
41
+ paused: boolean;
42
+ /**
43
+ * Has this module actually been deployed? The spec says removal is refused
44
+ * when another **installed** module depends on the capability, and an
45
+ * imported-but-never-deployed module is not installed: it has never resolved
46
+ * the capability, holds no state derived from it, and cannot be orphaned by
47
+ * the provider going away.
48
+ *
49
+ * This is not a nicety — it is required for the guard and the cascade to
50
+ * agree (design D3). `pause --cascade` skips undeployed members, because
51
+ * pausing requires a settled deployed state and there is nothing to quiesce.
52
+ * If the guard still counted them, the operator would pause the full cascade
53
+ * and the removal would STILL refuse, naming a module that pause cannot act
54
+ * on — a dead end with no way forward.
55
+ */
56
+ deployed: boolean;
57
+ }
58
+
59
+ /**
60
+ * Modules that block removal of a provider of `providedCapabilities`.
61
+ *
62
+ * Pure. Paused candidates are excluded; a candidate depending via both
63
+ * `requires` and `optional` is reported once, as `requires` (the stronger
64
+ * claim, and the one whose removal consequence is worse).
65
+ */
66
+ export function findRemovalBlockers(
67
+ providedCapabilities: Iterable<string>,
68
+ candidates: DependentCandidate[],
69
+ ): Blocker[] {
70
+ const provided = new Set(providedCapabilities);
71
+ if (provided.size === 0) return [];
72
+
73
+ const blockers: Blocker[] = [];
74
+ for (const candidate of candidates) {
75
+ if (candidate.paused) continue;
76
+ if (!candidate.deployed) continue;
77
+
78
+ const required = (candidate.manifest.requires?.capabilities ?? []).map((c) => c.name);
79
+ const optional = (candidate.manifest.optional?.capabilities ?? []).map((c) => c.name);
80
+
81
+ const hit =
82
+ required.find((name) => provided.has(name)) ?? optional.find((name) => provided.has(name));
83
+ if (!hit) continue;
84
+
85
+ blockers.push({
86
+ moduleId: candidate.id,
87
+ capability: hit,
88
+ kind: required.includes(hit) ? 'requires' : 'optional',
89
+ });
90
+ }
91
+
92
+ return blockers.sort((a, b) => a.moduleId.localeCompare(b.moduleId));
93
+ }
94
+
95
+ /**
96
+ * The refusal an operator reads. Names each blocker AND which declaration makes
97
+ * it one, so a hard prerequisite is distinguishable from an optional consumer —
98
+ * the operator needs that to judge what pausing it implies.
99
+ */
100
+ export function describeRemovalRefusal(moduleId: string, blockers: Blocker[]): string {
101
+ const lines = blockers.map((b) => ` • ${b.moduleId} — ${b.kind} '${b.capability}'`);
102
+ return [
103
+ `Cannot remove '${moduleId}': the following installed modules depend on its capabilities:`,
104
+ ...lines,
105
+ '',
106
+ 'Pause them first so they rebind on their next deploy:',
107
+ ` celilo module pause --cascade ${moduleId}`,
108
+ ].join('\n');
109
+ }
@@ -1,17 +1,26 @@
1
1
  /**
2
- * Recurrence gate for ISS-0016: the S3 provider's upload() must send a
3
- * self-describing, replayable Buffer body — NOT a one-shot Node read stream.
2
+ * Recurrence gates for the two ways this provider's upload has been wrong.
4
3
  *
5
- * The original bug passed `createReadStream(localPath)` as PutObject Body with
6
- * no ContentLength; AWS SDK v3 fell back to aws-chunked streaming and failed
7
- * with "The request body terminated unexpectedly" (and couldn't replay the
8
- * stream across S3's retries/redirects). The verify path used a string body
9
- * (length known) and so never exercised the broken path every real S3 backup
10
- * silently failed. These tests upload a REAL on-disk file and assert the body
11
- * is a Buffer, so a regression back to a stream is caught here.
4
+ * ISS-0016 a one-shot `createReadStream` passed straight to PutObject as
5
+ * `Body` with no ContentLength. AWS SDK v3 fell back to aws-chunked streaming
6
+ * and failed with "The request body terminated unexpectedly", and could not
7
+ * replay the stream across S3's retries and redirects. The verify path used a
8
+ * string body (length known) and so never exercised the broken path, which is
9
+ * why every real S3 backup silently failed.
10
+ *
11
+ * celilo#685 — the fix for ISS-0016 was `readFileSync` into one Buffer, which
12
+ * is replayable and was fine for the system envelopes this provider carried at
13
+ * the time. Module backups then started using the same provider at a thousand
14
+ * times the size, and a ~1.9 GB Buffer on a 3784 MB management server was
15
+ * OOM-killed every hour for a day.
16
+ *
17
+ * The two constrain opposite things — replayable versus not resident — so
18
+ * neither test is meaningful alone, and satisfying one by breaking the other is
19
+ * exactly the history here. `Upload` (multipart) satisfies both: each PART is a
20
+ * replayable buffer, and only a bounded number of parts exist at once.
12
21
  *
13
22
  * No live S3 / MinIO harness exists, so we intercept S3Client.prototype.send
14
- * and inspect the command the provider builds.
23
+ * and inspect the commands the provider builds.
15
24
  */
16
25
 
17
26
  import { type Mock, afterEach, describe, expect, it, spyOn } from 'bun:test';
@@ -19,8 +28,20 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
19
28
  import { tmpdir } from 'node:os';
20
29
  import { join } from 'node:path';
21
30
  import { Readable } from 'node:stream';
22
- import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
23
- import { type S3StorageConfig, createS3StorageProvider } from './s3';
31
+ import {
32
+ CompleteMultipartUploadCommand,
33
+ CreateMultipartUploadCommand,
34
+ GetObjectCommand,
35
+ PutObjectCommand,
36
+ S3Client,
37
+ UploadPartCommand,
38
+ } from '@aws-sdk/client-s3';
39
+ import {
40
+ type S3StorageConfig,
41
+ UPLOAD_PART_SIZE,
42
+ UPLOAD_QUEUE_SIZE,
43
+ createS3StorageProvider,
44
+ } from './s3';
24
45
 
25
46
  const CONFIG: S3StorageConfig = {
26
47
  bucket: 'test-bucket',
@@ -44,7 +65,7 @@ describe('s3 storage provider (ISS-0016)', () => {
44
65
  }
45
66
  });
46
67
 
47
- it('uploads a real on-disk file as a self-describing Buffer body, not a stream', async () => {
68
+ it('uploads a small file as a self-describing Buffer body, not a stream (ISS-0016)', async () => {
48
69
  dir = mkdtempSync(join(tmpdir(), 's3-upload-'));
49
70
  const file = join(dir, 'envelope.tar.gz');
50
71
  // A multi-KB real file — the case that broke against live S3.
@@ -61,6 +82,8 @@ describe('s3 storage provider (ISS-0016)', () => {
61
82
  const provider = createS3StorageProvider(CONFIG);
62
83
  await provider.upload(file, 'celilo-mgmt/2026/envelope.tar.gz');
63
84
 
85
+ // Under one part, so no multipart ceremony — a single PutObject, exactly
86
+ // as before. The bound below is what changed, not the small-file path.
64
87
  expect(sent).toHaveLength(1);
65
88
  const command = sent[0];
66
89
  expect(command).toBeInstanceOf(PutObjectCommand);
@@ -78,6 +101,66 @@ describe('s3 storage provider (ISS-0016)', () => {
78
101
  expect(command.input.Bucket).toBe('test-bucket');
79
102
  });
80
103
 
104
+ it('never holds more than one part per queue slot, however large the file (celilo#685)', async () => {
105
+ dir = mkdtempSync(join(tmpdir(), 's3-upload-large-'));
106
+ const file = join(dir, 'backup.tar.enc');
107
+
108
+ // Deliberately larger than one part, so the multipart path runs for real.
109
+ // It does not need to approach forgejo's 1.87 GB: the property under test
110
+ // is that peak residency is set by the part size rather than by the file,
111
+ // and a file that spans several parts demonstrates that at any scale. A
112
+ // test that had to allocate the failing size to prove the fix would be
113
+ // reproducing the bug rather than gating it.
114
+ const parts = 3;
115
+ const fileSize = UPLOAD_PART_SIZE * parts;
116
+ writeFileSync(file, Buffer.alloc(fileSize, 0x7a));
117
+
118
+ // Deliberately NOT a running total. Parts are uploaded concurrently and can
119
+ // arrive in any order, so the only way to show the artifact survives is to
120
+ // keep each part against its number and reassemble.
121
+ const received = new Map<number, Buffer>();
122
+ let inFlight = 0;
123
+ let peakInFlight = 0;
124
+ sendSpy = spyOn(S3Client.prototype, 'send').mockImplementation(async (command: unknown) => {
125
+ if (command instanceof CreateMultipartUploadCommand) return { UploadId: 'upload-1' };
126
+ if (command instanceof CompleteMultipartUploadCommand) return {};
127
+
128
+ expect(command).toBeInstanceOf(UploadPartCommand);
129
+ const part = command as UploadPartCommand;
130
+ const body = part.input.Body as Buffer;
131
+ const partNumber = part.input.PartNumber as number;
132
+
133
+ // Each part is still a replayable Buffer — ISS-0016 holds per part.
134
+ expect(Buffer.isBuffer(body)).toBe(true);
135
+ // A part number reused would silently lose data on reassembly.
136
+ expect(received.has(partNumber)).toBe(false);
137
+ received.set(partNumber, Buffer.from(body));
138
+
139
+ inFlight += 1;
140
+ peakInFlight = Math.max(peakInFlight, inFlight);
141
+ await new Promise((resolve) => setTimeout(resolve, 1));
142
+ inFlight -= 1;
143
+ return { ETag: `"etag-${partNumber}"` };
144
+ });
145
+
146
+ const provider = createS3StorageProvider(CONFIG);
147
+ await provider.upload(file, 'forgejo/2026-08-13/backup.tar.enc');
148
+
149
+ // The bound. No single request ever carries the whole artifact, and no more
150
+ // than the queue depth are resident at once, so peak bytes is
151
+ // UPLOAD_PART_SIZE * UPLOAD_QUEUE_SIZE no matter how big the file gets.
152
+ expect(received.size).toBe(parts);
153
+ for (const body of received.values()) expect(body.length).toBeLessThanOrEqual(UPLOAD_PART_SIZE);
154
+ expect(peakInFlight).toBeLessThanOrEqual(UPLOAD_QUEUE_SIZE);
155
+
156
+ // A bound is only worth having if the artifact still arrives. Reassembled
157
+ // in part order, the bytes must be the file — sizes summing correctly would
158
+ // not catch a swapped or duplicated part, and a backup that restores to
159
+ // scrambled bytes is worse than one that fails loudly.
160
+ const ordered = [...received.entries()].sort(([a], [b]) => a - b).map(([, body]) => body);
161
+ expect(Buffer.concat(ordered).equals(readFileSync(file))).toBe(true);
162
+ });
163
+
81
164
  it('downloads a multi-chunk response body to disk intact (short-read safe)', async () => {
82
165
  dir = mkdtempSync(join(tmpdir(), 's3-download-'));
83
166
  const out = join(dir, 'restored.bin');
@@ -3,7 +3,7 @@
3
3
  * Works with AWS S3, MinIO, Backblaze B2, Wasabi, and any S3-compatible service.
4
4
  */
5
5
 
6
- import { createWriteStream, readFileSync } from 'node:fs';
6
+ import { createReadStream, createWriteStream } from 'node:fs';
7
7
  import { mkdir } from 'node:fs/promises';
8
8
  import { dirname } from 'node:path';
9
9
  import { Readable } from 'node:stream';
@@ -15,10 +15,47 @@ import {
15
15
  PutObjectCommand,
16
16
  S3Client,
17
17
  } from '@aws-sdk/client-s3';
18
+ import { Upload } from '@aws-sdk/lib-storage';
18
19
  import type { StorageProvider, StorageVerifyResult } from './types';
19
20
 
20
21
  const BACKUP_PREFIX = 'celilo-backups';
21
22
 
23
+ /**
24
+ * Multipart upload sizing. Peak resident bytes for an upload is
25
+ * `UPLOAD_PART_SIZE * UPLOAD_QUEUE_SIZE` — 64 MB — and does NOT grow with the
26
+ * artifact. That bound is the whole point of these two constants.
27
+ *
28
+ * `upload` used to `readFileSync` the artifact into one Buffer and hand it to
29
+ * PutObject. The comment justifying that read:
30
+ *
31
+ * > Upload a Buffer, NOT a read stream (ISS-0016). A streamed Body fails with
32
+ * > "The request body terminated unexpectedly" [...] Backup envelopes are
33
+ * > small (state + key, not provider binaries — ISS-0015), so buffering is
34
+ * > fine.
35
+ *
36
+ * The first half is true and still is: a one-shot Node stream as a PutObject
37
+ * `Body` cannot be replayed across the retries and redirects S3 issues, and
38
+ * with no ContentLength the SDK falls back to aws-chunked encoding on top.
39
+ * Passing `createReadStream` straight to PutObject would reintroduce exactly
40
+ * that bug.
41
+ *
42
+ * The second half stopped being true without anyone revisiting it. It was
43
+ * written when this provider only carried SYSTEM backups (celilo state plus a
44
+ * key, a few MB). MODULE backups now use the same provider and are three orders
45
+ * of magnitude larger — forgejo's envelope reached 1.87 GB — so "buffering is
46
+ * fine" became a ~1.9 GB Buffer on a 3784 MB management server, and the OOM
47
+ * killer took the backup every hour for a day while the on_backup hook itself
48
+ * reported success (celilo#685).
49
+ *
50
+ * `Upload` resolves the two halves rather than trading one for the other: it
51
+ * reads the stream a part at a time and each PART is a replayable buffer, so
52
+ * retries work without the whole object ever being resident. 16 MB parts keep
53
+ * a 1.87 GB artifact at ~117 requests, well inside S3's 10,000-part limit,
54
+ * which leaves headroom to ~160 GB.
55
+ */
56
+ export const UPLOAD_PART_SIZE = 16 * 1024 * 1024;
57
+ export const UPLOAD_QUEUE_SIZE = 4;
58
+
22
59
  export interface S3StorageConfig {
23
60
  bucket: string;
24
61
  region: string;
@@ -49,22 +86,18 @@ export function createS3StorageProvider(config: S3StorageConfig): StorageProvide
49
86
 
50
87
  return {
51
88
  async upload(localPath: string, remotePath: string): Promise<void> {
52
- // Upload a Buffer, NOT a read stream (ISS-0016). A streamed Body fails
53
- // with "The request body terminated unexpectedly": the SDK can't replay a
54
- // one-shot Node stream across the retries/redirects S3 issues (e.g. a
55
- // region 301), and with no ContentLength it falls back to aws-chunked
56
- // encoding on top. A Buffer is replayable and self-describing — the same
57
- // reason the string-bodied verify write succeeds where the stream upload
58
- // did not. Backup envelopes are small (state + key, not provider
59
- // binaries — ISS-0015), so buffering is fine.
60
- const body = readFileSync(localPath);
61
- await client.send(
62
- new PutObjectCommand({
89
+ // Multipart, so peak memory is UPLOAD_PART_SIZE * UPLOAD_QUEUE_SIZE
90
+ // regardless of how large the artifact is.
91
+ await new Upload({
92
+ client,
93
+ partSize: UPLOAD_PART_SIZE,
94
+ queueSize: UPLOAD_QUEUE_SIZE,
95
+ params: {
63
96
  Bucket: bucket,
64
97
  Key: prefixedKey(remotePath),
65
- Body: body,
66
- }),
67
- );
98
+ Body: createReadStream(localPath),
99
+ },
100
+ }).done();
68
101
  },
69
102
 
70
103
  async download(remotePath: string, localPath: string): Promise<void> {
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Terminal-responder: when a deploy runs on a TTY, this subscribes
3
3
  * to `config.required.*`, `secret.required.*`, and `ensure.required.*`
4
- * events and prompts the operator via clack for each one, replying
5
- * on the bus.
4
+ * events and prompts the operator on the terminal for each one,
5
+ * replying on the bus.
6
6
  *
7
7
  * Just one of several responder shapes — the bus query/reply path is
8
8
  * a race, and the terminal is one racer. Other racers (Claude
@@ -26,9 +26,9 @@
26
26
  */
27
27
 
28
28
  import { hostname } from 'node:os';
29
+ import { confirm, isCancel, multiselect, select } from '@celilo/cli-display';
29
30
  import { type Bus, openBus } from '@celilo/event-bus';
30
31
  import { defineEvents } from '@celilo/event-bus';
31
- import * as p from '@clack/prompts';
32
32
  import { log, promptPassword, promptText } from '../cli/prompts';
33
33
  import { getEventBusPath } from '../config/paths';
34
34
  import { getDb } from '../db/client';
@@ -61,7 +61,7 @@ export interface TerminalResponderHandle {
61
61
 
62
62
  /**
63
63
  * Register transient subscriptions on the three interview event
64
- * families. For each event, prompt the operator via clack, optionally
64
+ * families. For each event, prompt the operator on the terminal, optionally
65
65
  * inject the secret value out-of-band, and reply on the bus.
66
66
  *
67
67
  * Returns a handle the caller can close() when the deploy completes.
@@ -103,9 +103,9 @@ export function startTerminalResponder(): TerminalResponderHandle {
103
103
 
104
104
  if (payload.options && payload.options.length > 0) {
105
105
  // Multi-select prompt for vars with options[] declared in the
106
- // manifest. clack's multiselect returns an array of selected
107
- // values directly — no JSON-typing to coerce.
108
- const selected = await p.multiselect({
106
+ // manifest. multiselect returns an array of selected values
107
+ // directly — no JSON-typing to coerce.
108
+ const selected = await multiselect({
109
109
  message,
110
110
  options: payload.options.map((opt) => ({
111
111
  value: opt.value,
@@ -114,7 +114,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
114
114
  })),
115
115
  required: payload.required,
116
116
  });
117
- if (p.isCancel(selected)) {
117
+ if (isCancel(selected)) {
118
118
  log.warn(
119
119
  `Terminal responder: cancelled prompt for ${payload.module}.${payload.key}; no reply emitted`,
120
120
  );
@@ -308,11 +308,11 @@ export function startTerminalResponder(): TerminalResponderHandle {
308
308
  let value: unknown;
309
309
 
310
310
  if (payload.kind === 'confirm') {
311
- const answer = await p.confirm({
311
+ const answer = await confirm({
312
312
  message,
313
313
  initialValue: payload.defaultValue === 'true',
314
314
  });
315
- if (p.isCancel(answer)) {
315
+ if (isCancel(answer)) {
316
316
  log.warn(
317
317
  `Terminal responder: cancelled prompt for ${payload.scope}.${payload.key}; no reply emitted`,
318
318
  );
@@ -320,7 +320,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
320
320
  }
321
321
  value = answer;
322
322
  } else if (payload.kind === 'select') {
323
- const answer = await p.select({
323
+ const answer = await select({
324
324
  message,
325
325
  options: (payload.options ?? []).map((opt) => ({
326
326
  value: opt.value,
@@ -329,7 +329,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
329
329
  })),
330
330
  initialValue: payload.defaultValue,
331
331
  });
332
- if (p.isCancel(answer)) {
332
+ if (isCancel(answer)) {
333
333
  log.warn(
334
334
  `Terminal responder: cancelled prompt for ${payload.scope}.${payload.key}; no reply emitted`,
335
335
  );
@@ -337,7 +337,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
337
337
  }
338
338
  value = answer;
339
339
  } else if (payload.kind === 'multiselect') {
340
- const answer = await p.multiselect({
340
+ const answer = await multiselect({
341
341
  message,
342
342
  options: (payload.options ?? []).map((opt) => ({
343
343
  value: opt.value,
@@ -346,7 +346,7 @@ export function startTerminalResponder(): TerminalResponderHandle {
346
346
  })),
347
347
  required: payload.required,
348
348
  });
349
- if (p.isCancel(answer)) {
349
+ if (isCancel(answer)) {
350
350
  log.warn(
351
351
  `Terminal responder: cancelled prompt for ${payload.scope}.${payload.key}; no reply emitted`,
352
352
  );
@@ -624,8 +624,8 @@ function describeTypeHint(type: ConfigRequiredPayload['type']): string | null {
624
624
  }
625
625
 
626
626
  function coerceValue(raw: string | undefined, type: ConfigRequiredPayload['type']): unknown {
627
- // clack returns undefined when the user cancels (Ctrl+C). Bubble
628
- // that up as an error so validate sees it and the responder can
627
+ // The prompt layer yields undefined when the user cancels (Ctrl+C).
628
+ // Bubble that up as an error so validate sees it and the responder can
629
629
  // skip the reply rather than emit a malformed one.
630
630
  if (raw === undefined) {
631
631
  throw new Error('Cancelled');