@celilo/cli 3.1.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_SUBSYSTEMS.md +1 -0
- package/package.json +5 -4
- package/src/cli/command-tree-parser.test.ts +18 -0
- package/src/cli/command-tree-parser.ts +12 -0
- package/src/cli/commands/api.ts +67 -14
- package/src/cli/commands/events.ts +5 -2
- package/src/cli/commands/machine-reclassify.ts +79 -0
- package/src/cli/commands/publish/workspace.ts +9 -0
- package/src/cli/commands/system-audit.ts +22 -0
- package/src/cli/commands/system-reboot.ts +126 -0
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +3 -2
- package/src/cli/index.ts +19 -0
- package/src/cli/tui/audit-state.ts +4 -0
- package/src/hooks/broker.ts +31 -0
- package/src/hooks/capability-loader.ts +87 -44
- package/src/hooks/executor.ts +20 -2
- package/src/hooks/test-fixtures/slow-capability-hook.ts +28 -0
- package/src/hooks/timeout-names-provider.test.ts +87 -0
- package/src/policy/module-business-baseline.ts +0 -6
- package/src/services/api-access.test.ts +44 -0
- package/src/services/audit/index.test.ts +7 -0
- package/src/services/audit/index.ts +15 -0
- package/src/services/audit/machine-interface-zones.test.ts +30 -0
- package/src/services/audit/machine-interface-zones.ts +43 -0
- package/src/services/audit/reboot-pending.test.ts +63 -0
- package/src/services/audit/reboot-pending.ts +85 -0
- package/src/services/audit/recurrence-gate.test.ts +21 -0
- package/src/services/audit/server-bun-pin.test.ts +44 -0
- package/src/services/audit/server-bun-pin.ts +95 -0
- package/src/services/audit/types.ts +2 -0
- package/src/services/capability-compat.test.ts +22 -0
- package/src/services/capability-compat.ts +15 -2
- package/src/services/celilo-events.test.ts +39 -0
- package/src/services/celilo-events.ts +41 -4
- package/src/services/deploy-preflight.ts +12 -3
- package/src/services/deploy-validation.test.ts +81 -2
- package/src/services/deploy-validation.ts +19 -0
- package/src/services/fleet-checks.ts +40 -10
- package/src/services/machine-interface-zones.test.ts +73 -0
- package/src/services/machine-interface-zones.ts +101 -0
- package/src/services/module-deploy.stale-provider-guard.test.ts +98 -0
- package/src/services/module-deploy.ts +48 -13
- package/src/services/provider-arrival.test.ts +62 -46
- package/src/services/provider-arrival.ts +41 -10
- package/src/services/provider-converge.test.ts +16 -0
- package/src/services/provider-converge.ts +106 -4
- package/src/services/system-reboot.test.ts +231 -0
- package/src/services/system-reboot.ts +357 -0
- package/src/services/update/orchestrator.test.ts +164 -0
- package/src/services/update/orchestrator.ts +43 -12
- package/src/services/update/types.ts +10 -1
- package/src/templates/generator.test.ts +15 -4
- package/src/templates/generator.ts +12 -4
- package/src/variables/context.ts +14 -1
- package/src/services/static-content-converge.test.ts +0 -516
- package/src/services/static-content-converge.ts +0 -390
|
@@ -21,6 +21,9 @@ function manifest(input: { requires?: string[]; optional?: string[] }): ModuleMa
|
|
|
21
21
|
} as unknown as ModuleManifest;
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
/** What the arriving provider registered. Consumers in these tests declare 1.0.0. */
|
|
25
|
+
const provides = (...names: string[]) => names.map((name) => ({ name, version: '1.0.0' }));
|
|
26
|
+
|
|
24
27
|
const consumer = (
|
|
25
28
|
moduleId: string,
|
|
26
29
|
caps: { requires?: string[]; optional?: string[] },
|
|
@@ -29,14 +32,10 @@ const consumer = (
|
|
|
29
32
|
|
|
30
33
|
describe('planProviderBackfill', () => {
|
|
31
34
|
it('re-runs consumers of the capabilities the arriving provider actually provides', () => {
|
|
32
|
-
const plan = planProviderBackfill(
|
|
33
|
-
'
|
|
34
|
-
['
|
|
35
|
-
|
|
36
|
-
consumer('caddy', { requires: ['firewall'] }),
|
|
37
|
-
consumer('forgejo', { requires: ['public_web'] }),
|
|
38
|
-
],
|
|
39
|
-
);
|
|
35
|
+
const plan = planProviderBackfill('iptables', provides('firewall'), [
|
|
36
|
+
consumer('caddy', { requires: ['firewall'] }),
|
|
37
|
+
consumer('forgejo', { requires: ['public_web'] }),
|
|
38
|
+
]);
|
|
40
39
|
|
|
41
40
|
expect(plan.map((t) => t.consumerId)).toEqual(['caddy']);
|
|
42
41
|
});
|
|
@@ -48,11 +47,9 @@ describe('planProviderBackfill', () => {
|
|
|
48
47
|
* provider appears. Counting only `requires` would leave it unregistered.
|
|
49
48
|
*/
|
|
50
49
|
it('includes an OPTIONAL dependent, not only a required one', () => {
|
|
51
|
-
const plan = planProviderBackfill(
|
|
52
|
-
'
|
|
53
|
-
|
|
54
|
-
[consumer('caddy-internal', { optional: ['dns_internal'] })],
|
|
55
|
-
);
|
|
50
|
+
const plan = planProviderBackfill('technitium', provides('dns_internal'), [
|
|
51
|
+
consumer('caddy-internal', { optional: ['dns_internal'] }),
|
|
52
|
+
]);
|
|
56
53
|
|
|
57
54
|
expect(plan.map((t) => t.consumerId)).toEqual(['caddy-internal']);
|
|
58
55
|
expect(plan[0]?.skip).toBeUndefined();
|
|
@@ -61,14 +58,10 @@ describe('planProviderBackfill', () => {
|
|
|
61
58
|
it('never treats the provider as its own consumer', () => {
|
|
62
59
|
// A module that both provides and requires a capability would otherwise
|
|
63
60
|
// have its on_install run a second time inside the deploy that just ran it.
|
|
64
|
-
const plan = planProviderBackfill(
|
|
65
|
-
'caddy',
|
|
66
|
-
['public_web'],
|
|
67
|
-
|
|
68
|
-
consumer('caddy', { requires: ['public_web', 'firewall'] }),
|
|
69
|
-
consumer('forgejo', { requires: ['public_web'] }),
|
|
70
|
-
],
|
|
71
|
-
);
|
|
61
|
+
const plan = planProviderBackfill('caddy', provides('public_web'), [
|
|
62
|
+
consumer('caddy', { requires: ['public_web', 'firewall'] }),
|
|
63
|
+
consumer('forgejo', { requires: ['public_web'] }),
|
|
64
|
+
]);
|
|
72
65
|
|
|
73
66
|
expect(plan.map((t) => t.consumerId)).toEqual(['forgejo']);
|
|
74
67
|
});
|
|
@@ -76,52 +69,75 @@ describe('planProviderBackfill', () => {
|
|
|
76
69
|
it('skips a consumer in a pre-deploy state, and says so rather than dropping it', () => {
|
|
77
70
|
// Capabilities are registered at IMPORT, not deploy, so an imported module
|
|
78
71
|
// has never resolved anything and has nothing to re-register.
|
|
79
|
-
const plan = planProviderBackfill(
|
|
80
|
-
'
|
|
81
|
-
['firewall'],
|
|
82
|
-
[
|
|
83
|
-
|
|
84
|
-
consumer('validated', { requires: ['firewall'] }, 'VALIDATED'),
|
|
85
|
-
consumer('configured', { requires: ['firewall'] }, 'CONFIGURED'),
|
|
86
|
-
],
|
|
87
|
-
);
|
|
72
|
+
const plan = planProviderBackfill('iptables', provides('firewall'), [
|
|
73
|
+
consumer('imported', { requires: ['firewall'] }, 'IMPORTED'),
|
|
74
|
+
consumer('validated', { requires: ['firewall'] }, 'VALIDATED'),
|
|
75
|
+
consumer('configured', { requires: ['firewall'] }, 'CONFIGURED'),
|
|
76
|
+
]);
|
|
88
77
|
|
|
89
78
|
expect(plan.map((t) => t.skip)).toEqual(['not-deployed', 'not-deployed', 'not-deployed']);
|
|
90
79
|
});
|
|
91
80
|
|
|
92
81
|
it('skips a PAUSED consumer separately from a never-deployed one', () => {
|
|
93
|
-
const plan = planProviderBackfill(
|
|
94
|
-
'
|
|
95
|
-
|
|
96
|
-
[consumer('paused', { requires: ['firewall'] }, 'PAUSED')],
|
|
97
|
-
);
|
|
82
|
+
const plan = planProviderBackfill('iptables', provides('firewall'), [
|
|
83
|
+
consumer('paused', { requires: ['firewall'] }, 'PAUSED'),
|
|
84
|
+
]);
|
|
98
85
|
|
|
99
86
|
expect(plan[0]?.skip).toBe('paused');
|
|
100
87
|
});
|
|
101
88
|
|
|
102
89
|
it('names every capability the consumer takes from this provider, once', () => {
|
|
103
|
-
const plan = planProviderBackfill(
|
|
104
|
-
'
|
|
105
|
-
|
|
106
|
-
[consumer('caddy', { requires: ['firewall'], optional: ['dhcp_server'] })],
|
|
107
|
-
);
|
|
90
|
+
const plan = planProviderBackfill('greenwave', provides('firewall', 'dhcp_server'), [
|
|
91
|
+
consumer('caddy', { requires: ['firewall'], optional: ['dhcp_server'] }),
|
|
92
|
+
]);
|
|
108
93
|
|
|
109
94
|
expect(plan).toHaveLength(1);
|
|
110
95
|
expect(plan[0]?.capabilityNames).toEqual(['dhcp_server', 'firewall']);
|
|
111
96
|
});
|
|
112
97
|
|
|
98
|
+
/**
|
|
99
|
+
* celilo#1384. An OPTIONAL consumer built against external_web@1.0.0 was
|
|
100
|
+
* re-run against a provider serving 2.0.0; the hook failed inside
|
|
101
|
+
* `on_install`, which is only a warning, so nothing published and no surface
|
|
102
|
+
* said so. A consumer on another major is skipped and REPORTED instead.
|
|
103
|
+
*/
|
|
104
|
+
it('skips a consumer whose declared version the arriving provider no longer serves', () => {
|
|
105
|
+
const plan = planProviderBackfill(
|
|
106
|
+
'generic-cpanel',
|
|
107
|
+
[{ name: 'external_web', version: '2.0.0' }],
|
|
108
|
+
[
|
|
109
|
+
consumer('tango-nexus', { optional: ['external_web'] }),
|
|
110
|
+
consumer('required-too', { requires: ['external_web'] }),
|
|
111
|
+
],
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
expect(plan.map((t) => [t.consumerId, t.skip])).toEqual([
|
|
115
|
+
['required-too', 'incompatible'],
|
|
116
|
+
['tango-nexus', 'incompatible'],
|
|
117
|
+
]);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it('re-runs a consumer the provider is merely NEWER in minor than', () => {
|
|
121
|
+
const plan = planProviderBackfill(
|
|
122
|
+
'generic-cpanel',
|
|
123
|
+
[{ name: 'external_web', version: '1.4.0' }],
|
|
124
|
+
[consumer('tango-nexus', { optional: ['external_web'] })],
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
expect(plan[0]?.skip).toBeUndefined();
|
|
128
|
+
});
|
|
129
|
+
|
|
113
130
|
it('is empty for a module that provides nothing', () => {
|
|
114
131
|
expect(
|
|
115
|
-
planProviderBackfill('forgejo',
|
|
132
|
+
planProviderBackfill('forgejo', provides(), [consumer('caddy', { requires: ['firewall'] })]),
|
|
116
133
|
).toEqual([]);
|
|
117
134
|
});
|
|
118
135
|
|
|
119
136
|
it('is ordered by consumer id, so a failure is reproducible', () => {
|
|
120
|
-
const plan = planProviderBackfill(
|
|
121
|
-
'
|
|
122
|
-
['firewall'],
|
|
123
|
-
|
|
124
|
-
);
|
|
137
|
+
const plan = planProviderBackfill('iptables', provides('firewall'), [
|
|
138
|
+
consumer('zulu', { requires: ['firewall'] }),
|
|
139
|
+
consumer('alpha', { requires: ['firewall'] }),
|
|
140
|
+
]);
|
|
125
141
|
|
|
126
142
|
expect(plan.map((t) => t.consumerId)).toEqual(['alpha', 'zulu']);
|
|
127
143
|
});
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
* happened".
|
|
28
28
|
*/
|
|
29
29
|
|
|
30
|
+
import { compareConsumerToProvider } from '@celilo/capabilities';
|
|
30
31
|
import { eq } from 'drizzle-orm';
|
|
31
32
|
import type { DbClient } from '../db/client';
|
|
32
33
|
import { capabilities, modules } from '../db/schema';
|
|
@@ -36,7 +37,13 @@ import type { HookLogger } from '../hooks/types';
|
|
|
36
37
|
import { type ModuleManifest, ModuleManifestSchema } from '../manifest/schema';
|
|
37
38
|
import { PRE_DEPLOY_STATES } from './consumer-cleanup';
|
|
38
39
|
|
|
39
|
-
export type BackfillSkipReason = 'paused' | 'not-deployed';
|
|
40
|
+
export type BackfillSkipReason = 'paused' | 'not-deployed' | 'incompatible';
|
|
41
|
+
|
|
42
|
+
/** A capability as the arriving provider registered it. */
|
|
43
|
+
export interface ProvidedCapability {
|
|
44
|
+
name: string;
|
|
45
|
+
version: string;
|
|
46
|
+
}
|
|
40
47
|
|
|
41
48
|
export interface BackfillTarget {
|
|
42
49
|
/** The consumer module to re-run. */
|
|
@@ -70,23 +77,42 @@ export interface ConsumerCandidate {
|
|
|
70
77
|
*/
|
|
71
78
|
export function planProviderBackfill(
|
|
72
79
|
provider: string,
|
|
73
|
-
providedCapabilities: Iterable<
|
|
80
|
+
providedCapabilities: Iterable<ProvidedCapability>,
|
|
74
81
|
candidates: ConsumerCandidate[],
|
|
75
82
|
): BackfillTarget[] {
|
|
76
|
-
const provided = new
|
|
83
|
+
const provided = new Map([...providedCapabilities].map((c) => [c.name, c.version]));
|
|
77
84
|
if (provided.size === 0) return [];
|
|
78
85
|
|
|
79
86
|
const targets: BackfillTarget[] = [];
|
|
80
87
|
for (const candidate of candidates) {
|
|
81
88
|
if (candidate.moduleId === provider) continue;
|
|
82
89
|
|
|
83
|
-
const consumed = new
|
|
84
|
-
|
|
85
|
-
...(candidate.manifest.optional?.capabilities ?? [])
|
|
86
|
-
|
|
87
|
-
|
|
90
|
+
const consumed = new Map<string, string>();
|
|
91
|
+
for (const c of [
|
|
92
|
+
...(candidate.manifest.optional?.capabilities ?? []),
|
|
93
|
+
...(candidate.manifest.requires?.capabilities ?? []),
|
|
94
|
+
]) {
|
|
95
|
+
consumed.set(c.name, c.version);
|
|
96
|
+
}
|
|
97
|
+
const capabilityNames = [...consumed.keys()].filter((name) => provided.has(name)).sort();
|
|
88
98
|
if (capabilityNames.length === 0) continue;
|
|
89
99
|
|
|
100
|
+
// The consumer was built against a contract the arriving provider no
|
|
101
|
+
// longer speaks. Re-running its `on_install` would call methods that are
|
|
102
|
+
// gone (celilo#1384: tango-nexus 0.8.0's optional external_web@1.0.0 was
|
|
103
|
+
// re-run against generic-cpanel's 2.0.0 and failed inside the hook, which
|
|
104
|
+
// is only a warning — so nothing published and nothing said so).
|
|
105
|
+
if (
|
|
106
|
+
capabilityNames.some(
|
|
107
|
+
(name) =>
|
|
108
|
+
// biome-ignore lint/style/noNonNullAssertion: name came from `consumed`/`provided`.
|
|
109
|
+
!compareConsumerToProvider(consumed.get(name)!, provided.get(name)!).compatible,
|
|
110
|
+
)
|
|
111
|
+
) {
|
|
112
|
+
targets.push({ consumerId: candidate.moduleId, capabilityNames, skip: 'incompatible' });
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
|
|
90
116
|
if (candidate.state === 'PAUSED') {
|
|
91
117
|
targets.push({ consumerId: candidate.moduleId, capabilityNames, skip: 'paused' });
|
|
92
118
|
continue;
|
|
@@ -111,11 +137,11 @@ export function planProviderBackfill(
|
|
|
111
137
|
*/
|
|
112
138
|
export function loadProviderBackfillPlan(provider: string, db: DbClient): BackfillTarget[] {
|
|
113
139
|
const provided = db
|
|
114
|
-
.select({ capabilityName: capabilities.capabilityName })
|
|
140
|
+
.select({ capabilityName: capabilities.capabilityName, version: capabilities.version })
|
|
115
141
|
.from(capabilities)
|
|
116
142
|
.where(eq(capabilities.moduleId, provider))
|
|
117
143
|
.all()
|
|
118
|
-
.map((row) => row.capabilityName);
|
|
144
|
+
.map((row) => ({ name: row.capabilityName, version: row.version }));
|
|
119
145
|
if (provided.length === 0) return [];
|
|
120
146
|
|
|
121
147
|
const candidates: ConsumerCandidate[] = [];
|
|
@@ -174,6 +200,11 @@ export async function runProviderBackfill(
|
|
|
174
200
|
`${target.consumerId} is paused, so it was not re-run against the new '${provider}' — it consumes ${target.capabilityNames.join(', ')} and will rebind on its next deploy.`,
|
|
175
201
|
);
|
|
176
202
|
}
|
|
203
|
+
if (target.skip === 'incompatible') {
|
|
204
|
+
logger.warn(
|
|
205
|
+
`${target.consumerId} was NOT re-run against the new '${provider}': it declares ${target.capabilityNames.join(', ')} on a version '${provider}' no longer serves. Rebuild ${target.consumerId} against the provider's current major, then redeploy it.`,
|
|
206
|
+
);
|
|
207
|
+
}
|
|
177
208
|
continue;
|
|
178
209
|
}
|
|
179
210
|
|
|
@@ -138,6 +138,22 @@ describe('recordUnresolvedConsumers — the failure belongs to the consumer', ()
|
|
|
138
138
|
return db.select().from(modules).where(eq(modules.id, id)).get();
|
|
139
139
|
}
|
|
140
140
|
|
|
141
|
+
it('leaves a PAUSED consumer paused — a provider cannot render it by design', () => {
|
|
142
|
+
db.update(modules).set({ state: 'PAUSED' }).where(eq(modules.id, 'byoi')).run();
|
|
143
|
+
|
|
144
|
+
const recorded = recordUnresolvedConsumers(db, 'caddy', [
|
|
145
|
+
{ moduleId: 'byoi', reason: "module 'byoi' is paused — its site is left as it is" },
|
|
146
|
+
]);
|
|
147
|
+
|
|
148
|
+
// Pause preserves state, and that includes the module's own recorded
|
|
149
|
+
// state. A paused module is unresolvable on EVERY converge, so marking it
|
|
150
|
+
// ERROR here would mean an operator could not pause a module without the
|
|
151
|
+
// next publish reporting it failed.
|
|
152
|
+
expect(recorded).toEqual([]);
|
|
153
|
+
expect(stateOf('byoi')?.state).toBe('PAUSED');
|
|
154
|
+
expect(stateOf('byoi')?.errorMessage ?? null).toBeNull();
|
|
155
|
+
});
|
|
156
|
+
|
|
141
157
|
it('records the consumer in ERROR with the reason it could not be rendered', () => {
|
|
142
158
|
recordUnresolvedConsumers(db, 'caddy', [
|
|
143
159
|
{ moduleId: 'byoi', reason: 'no built site at /var/celilo/modules/byoi/site/dist' },
|
|
@@ -15,14 +15,22 @@
|
|
|
15
15
|
* against that consumer.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import { existsSync, rmSync } from 'node:fs';
|
|
18
|
+
import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs';
|
|
19
19
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
20
20
|
import { join } from 'node:path';
|
|
21
|
-
import { eq } from 'drizzle-orm';
|
|
21
|
+
import { and, eq } from 'drizzle-orm';
|
|
22
22
|
import { stringify as stringifyYaml } from 'yaml';
|
|
23
23
|
import type { DbClient } from '../db/client';
|
|
24
|
-
import { modules } from '../db/schema';
|
|
25
|
-
import {
|
|
24
|
+
import { moduleConfigs, modules } from '../db/schema';
|
|
25
|
+
import { parseStoredConfigValue } from './module-config';
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Retention when the operator has not set one. Mirrors the manifest default
|
|
29
|
+
* declared on the provider module (`static_release_retention`); the manifest
|
|
30
|
+
* is the operator-facing source of truth, this only covers a provider whose
|
|
31
|
+
* manifest predates the variable.
|
|
32
|
+
*/
|
|
33
|
+
const DEFAULT_STATIC_RELEASE_RETENTION = 5;
|
|
26
34
|
|
|
27
35
|
/** One file the provider wants on its host, rendered by the provider. */
|
|
28
36
|
export interface ProviderConfigFile {
|
|
@@ -121,6 +129,14 @@ export function recordUnresolvedConsumers(
|
|
|
121
129
|
const existing = db.select().from(modules).where(eq(modules.id, consumer.moduleId)).get();
|
|
122
130
|
if (!existing) continue;
|
|
123
131
|
|
|
132
|
+
// A PAUSED module is not broken, and pause preserves state — including the
|
|
133
|
+
// module's own recorded state. A provider legitimately cannot render a
|
|
134
|
+
// paused consumer (its site is deliberately left alone), so this path is
|
|
135
|
+
// reached on every converge while any module is paused; writing ERROR
|
|
136
|
+
// there would mean an operator could not pause a module without the next
|
|
137
|
+
// publish marking it failed.
|
|
138
|
+
if (existing.state === 'PAUSED') continue;
|
|
139
|
+
|
|
124
140
|
db.update(modules)
|
|
125
141
|
.set({
|
|
126
142
|
state: 'ERROR',
|
|
@@ -239,3 +255,89 @@ export async function convergeProviderConfig(
|
|
|
239
255
|
rmSync(generatedPath, { recursive: true, force: true });
|
|
240
256
|
return { success: true, varsPath, unresolved };
|
|
241
257
|
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Resolve the retention count for the provider module.
|
|
261
|
+
*
|
|
262
|
+
* Operator config first (`static_release_retention` module config), then the
|
|
263
|
+
* manifest's declared default, then the constant above. ONE resolution — the
|
|
264
|
+
* role reads the variable bare and never carries a literal.
|
|
265
|
+
*/
|
|
266
|
+
export function resolveStaticContentRetention(db: DbClient, providerModuleId: string): number {
|
|
267
|
+
const configRow = db
|
|
268
|
+
.select()
|
|
269
|
+
.from(moduleConfigs)
|
|
270
|
+
.where(
|
|
271
|
+
and(
|
|
272
|
+
eq(moduleConfigs.moduleId, providerModuleId),
|
|
273
|
+
eq(moduleConfigs.key, 'static_release_retention'),
|
|
274
|
+
),
|
|
275
|
+
)
|
|
276
|
+
.get();
|
|
277
|
+
if (configRow) {
|
|
278
|
+
const parsed = parseStoredConfigValue(configRow);
|
|
279
|
+
const count = typeof parsed === 'number' ? parsed : Number.parseInt(String(parsed), 10);
|
|
280
|
+
if (Number.isInteger(count) && count >= 1) return count;
|
|
281
|
+
throw new Error(
|
|
282
|
+
`static_release_retention for ${providerModuleId} must be a positive integer, got: ${String(parsed)}`,
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
const module = db.select().from(modules).where(eq(modules.id, providerModuleId)).get();
|
|
287
|
+
if (module) {
|
|
288
|
+
const manifest = module.manifestData as {
|
|
289
|
+
variables?: { owns?: Array<{ name?: string; default?: unknown }> };
|
|
290
|
+
} | null;
|
|
291
|
+
const declared = manifest?.variables?.owns?.find((v) => v?.name === 'static_release_retention');
|
|
292
|
+
const fallback = typeof declared?.default === 'number' ? declared.default : undefined;
|
|
293
|
+
if (typeof fallback === 'number' && Number.isInteger(fallback) && fallback >= 1)
|
|
294
|
+
return fallback;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
return DEFAULT_STATIC_RELEASE_RETENTION;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Build the release set from the declared `web_routes` rows.
|
|
302
|
+
*
|
|
303
|
+
* Pure: reads the DB, decides nothing on the host (Rule 10.4). Grouping is by
|
|
304
|
+
* slug, NOT by route — one slug is one release directory, and a slug can span
|
|
305
|
+
* hostnames (lunacycle's two routes, both at `/`). A group that spans modules
|
|
306
|
+
* or disagrees with itself about the hash is a contradiction the converge
|
|
307
|
+
* cannot express, so it fails here with both rows named rather than silently
|
|
308
|
+
* converging one of them.
|
|
309
|
+
*/
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Whether any text file in a generated Ansible tree mentions `needle`.
|
|
313
|
+
*
|
|
314
|
+
* Scans the TREE rather than `playbook.yml`, because a provider's converge
|
|
315
|
+
* belongs in one of its role's task files and a generated `playbook.yml` is a
|
|
316
|
+
* thin hosts/vars/roles stanza that names no variable at all. caddy is the
|
|
317
|
+
* worked example: `static_releases` appears in
|
|
318
|
+
* `roles/caddy/tasks/static-content.yml`, `.../main.yml.tpl` and
|
|
319
|
+
* `.../converge-release.yml`, and zero times in its playbook.
|
|
320
|
+
*
|
|
321
|
+
* @psbanka - 2026-09: this used to read `ansible/playbook.yml` alone, which
|
|
322
|
+
* could not reach the string it was looking for. The guard below therefore
|
|
323
|
+
* refused EVERY provider that had the converge support, and its remedy told the
|
|
324
|
+
* operator to redeploy — which regenerates the same thin playbook and cannot
|
|
325
|
+
* help. Guard and role tasks landed in the same commit (1fd3595e), so it had
|
|
326
|
+
* never once passed, and it held three e2e suites red where nobody was looking.
|
|
327
|
+
* celilo#1248.
|
|
328
|
+
*/
|
|
329
|
+
export function ansibleTreeMentions(dir: string, needle: string): boolean {
|
|
330
|
+
if (!existsSync(dir)) return false;
|
|
331
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
332
|
+
const full = join(dir, entry.name);
|
|
333
|
+
if (entry.isDirectory()) {
|
|
334
|
+
if (ansibleTreeMentions(full, needle)) return true;
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
// Text only. A generated tree can carry static assets, and reading those
|
|
338
|
+
// as utf-8 to look for a variable name is waste at best.
|
|
339
|
+
if (!/\.(ya?ml|j2|tpl|cfg|ini|conf)$/.test(entry.name)) continue;
|
|
340
|
+
if (readFileSync(full, 'utf-8').includes(needle)) return true;
|
|
341
|
+
}
|
|
342
|
+
return false;
|
|
343
|
+
}
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The recurrence gate celilo#1378 asks for: drive the reboot against a FAKE
|
|
3
|
+
* host, with an operation in flight (it must refuse and name it) and with
|
|
4
|
+
* nothing in flight (it must reboot, and the return must be verified rather
|
|
5
|
+
* than assumed).
|
|
6
|
+
*
|
|
7
|
+
* Every dependency is injected, so nothing here reboots anything.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { describe, expect, it } from 'bun:test';
|
|
11
|
+
import type { InFlightConflict } from './module-operations';
|
|
12
|
+
import {
|
|
13
|
+
type RebootMarker,
|
|
14
|
+
collectRebootPendingFacts,
|
|
15
|
+
judgeRebootPending,
|
|
16
|
+
requestReboot,
|
|
17
|
+
verifyReturn,
|
|
18
|
+
} from './system-reboot';
|
|
19
|
+
|
|
20
|
+
const BOOT_A = '11111111-1111-1111-1111-111111111111';
|
|
21
|
+
const BOOT_B = '22222222-2222-2222-2222-222222222222';
|
|
22
|
+
|
|
23
|
+
function conflict(describe: string): InFlightConflict {
|
|
24
|
+
return { operation: {} as InFlightConflict['operation'], describe };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** A fake host: records what was asked of it, answers what it was told to. */
|
|
28
|
+
function fakeHost(opts: { conflicts?: string[]; bootId?: string | null } = {}) {
|
|
29
|
+
const state = { rebooted: 0, marker: null as RebootMarker | null };
|
|
30
|
+
return {
|
|
31
|
+
state,
|
|
32
|
+
deps: {
|
|
33
|
+
conflicts: () => (opts.conflicts ?? []).map(conflict),
|
|
34
|
+
bootId: () => (opts.bootId === undefined ? BOOT_A : opts.bootId),
|
|
35
|
+
issueReboot: () => {
|
|
36
|
+
state.rebooted += 1;
|
|
37
|
+
return { status: 0 };
|
|
38
|
+
},
|
|
39
|
+
writeMarker: (m: RebootMarker) => {
|
|
40
|
+
state.marker = m;
|
|
41
|
+
},
|
|
42
|
+
now: () => 1_700_000_000_000,
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
describe('requestReboot — refuses while the fleet is busy', () => {
|
|
48
|
+
it('refuses with an operation in flight, and names which one', () => {
|
|
49
|
+
const host = fakeHost({ conflicts: ['deploy of homebridge (pid 12345)'] });
|
|
50
|
+
|
|
51
|
+
const result = requestReboot(host.deps, 'pending kernel');
|
|
52
|
+
|
|
53
|
+
expect(result.success).toBe(false);
|
|
54
|
+
expect(result.error).toContain('deploy of homebridge (pid 12345)');
|
|
55
|
+
// The whole point: naming it is not enough if it rebooted anyway.
|
|
56
|
+
expect(host.state.rebooted).toBe(0);
|
|
57
|
+
expect(host.state.marker).toBe(null);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('refuses when the boot id cannot be read, because the return could never be verified', () => {
|
|
61
|
+
const host = fakeHost({ bootId: null });
|
|
62
|
+
|
|
63
|
+
const result = requestReboot(host.deps);
|
|
64
|
+
|
|
65
|
+
expect(result.success).toBe(false);
|
|
66
|
+
expect(result.error).toContain('could not be verified');
|
|
67
|
+
expect(host.state.rebooted).toBe(0);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it('reboots with nothing in flight, recording the boot id the return is checked against', () => {
|
|
71
|
+
const host = fakeHost();
|
|
72
|
+
|
|
73
|
+
const result = requestReboot(host.deps, 'pending kernel 6.8.0-1064-raspi');
|
|
74
|
+
|
|
75
|
+
expect(result.success).toBe(true);
|
|
76
|
+
expect(host.state.rebooted).toBe(1);
|
|
77
|
+
expect(host.state.marker?.bootIdBefore).toBe(BOOT_A);
|
|
78
|
+
expect(host.state.marker?.reason).toContain('6.8.0-1064-raspi');
|
|
79
|
+
// It reports the reboot as ISSUED, never as completed — the process
|
|
80
|
+
// asking for it dies in it (celilo#1378).
|
|
81
|
+
expect(result.message).toContain('--verify');
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('leaves the marker in place when the reboot command itself fails', () => {
|
|
85
|
+
const host = fakeHost();
|
|
86
|
+
const result = requestReboot(
|
|
87
|
+
{ ...host.deps, issueReboot: () => ({ status: 1 }) },
|
|
88
|
+
'pending kernel',
|
|
89
|
+
);
|
|
90
|
+
|
|
91
|
+
expect(result.success).toBe(false);
|
|
92
|
+
expect(result.error).toContain('was not rebooted');
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
describe('verifyReturn — only the host actually coming back is success', () => {
|
|
97
|
+
const marker: RebootMarker = { bootIdBefore: BOOT_A, requestedAt: 1, reason: 'kernel' };
|
|
98
|
+
const cameBack = {
|
|
99
|
+
bootId: () => BOOT_B,
|
|
100
|
+
readMarker: () => marker,
|
|
101
|
+
heartbeatLive: async () => true,
|
|
102
|
+
cliVersion: () => '3.0.1',
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
it('reports success once the boot id changed, the dispatcher is live and the CLI answers', async () => {
|
|
106
|
+
let cleared = 0;
|
|
107
|
+
const result = await verifyReturn({
|
|
108
|
+
...cameBack,
|
|
109
|
+
clearMarker: () => {
|
|
110
|
+
cleared += 1;
|
|
111
|
+
},
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
expect(result.success).toBe(true);
|
|
115
|
+
expect(result.message).toContain('3.0.1');
|
|
116
|
+
expect(cleared).toBe(1);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('fails when the boot id is unchanged — the host never went down', async () => {
|
|
120
|
+
const result = await verifyReturn({ ...cameBack, bootId: () => BOOT_A, clearMarker: () => {} });
|
|
121
|
+
|
|
122
|
+
expect(result.success).toBe(false);
|
|
123
|
+
expect(result.error).toContain('has NOT rebooted');
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('fails when the box booted but the dispatcher did not, and says so', async () => {
|
|
127
|
+
const result = await verifyReturn({
|
|
128
|
+
...cameBack,
|
|
129
|
+
heartbeatLive: async () => false,
|
|
130
|
+
clearMarker: () => {},
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
expect(result.success).toBe(false);
|
|
134
|
+
expect(result.error).toContain('dispatcher');
|
|
135
|
+
expect(result.error).toContain('unverified');
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it('fails when the CLI does not answer', async () => {
|
|
139
|
+
const result = await verifyReturn({
|
|
140
|
+
...cameBack,
|
|
141
|
+
cliVersion: () => null,
|
|
142
|
+
clearMarker: () => {},
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
expect(result.success).toBe(false);
|
|
146
|
+
expect(result.error).toContain('--version');
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('keeps the marker when verification fails, so the audit still sees it', async () => {
|
|
150
|
+
let cleared = 0;
|
|
151
|
+
await verifyReturn({
|
|
152
|
+
...cameBack,
|
|
153
|
+
heartbeatLive: async () => false,
|
|
154
|
+
clearMarker: () => {
|
|
155
|
+
cleared += 1;
|
|
156
|
+
},
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
expect(cleared).toBe(0);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it('says so plainly when no reboot was ever requested', async () => {
|
|
163
|
+
const result = await verifyReturn({ ...cameBack, readMarker: () => null });
|
|
164
|
+
|
|
165
|
+
expect(result.success).toBe(false);
|
|
166
|
+
expect(result.error).toContain('No reboot is pending');
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
describe('judgeRebootPending — two independent tells', () => {
|
|
171
|
+
const none = {
|
|
172
|
+
markerPresent: false,
|
|
173
|
+
deferredPackages: [],
|
|
174
|
+
runningKernel: '6.8.0-1064-raspi',
|
|
175
|
+
installedKernel: '6.8.0-1064-raspi',
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
it('is the #1378 case: a running kernel older than the installed one', () => {
|
|
179
|
+
const verdict = judgeRebootPending({ ...none, runningKernel: '6.8.0-1063-raspi' });
|
|
180
|
+
|
|
181
|
+
expect(verdict.pending).toBe(true);
|
|
182
|
+
expect(verdict.reasons[0]).toContain('6.8.0-1063-raspi');
|
|
183
|
+
expect(verdict.reasons[0]).toContain('6.8.0-1064-raspi');
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it("reports Debian's own marker, naming the packages that deferred", () => {
|
|
187
|
+
const verdict = judgeRebootPending({
|
|
188
|
+
...none,
|
|
189
|
+
markerPresent: true,
|
|
190
|
+
deferredPackages: ['docker.service', 'dbus.service'],
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
expect(verdict.pending).toBe(true);
|
|
194
|
+
expect(verdict.reasons.join(' ')).toContain('docker.service');
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it('is clean when the kernel matches and nothing deferred', () => {
|
|
198
|
+
expect(judgeRebootPending(none)).toEqual({ pending: false, unmeasured: false, reasons: [] });
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it('is UNMEASURED, not clean, when the host could not be asked', () => {
|
|
202
|
+
// An absent /var/run/reboot-required on a host that has no /var/run is not
|
|
203
|
+
// evidence of a healthy kernel. Reporting it clean is the #1326 shape.
|
|
204
|
+
const verdict = judgeRebootPending({
|
|
205
|
+
markerPresent: false,
|
|
206
|
+
deferredPackages: [],
|
|
207
|
+
runningKernel: null,
|
|
208
|
+
installedKernel: null,
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
expect(verdict).toEqual({ pending: false, unmeasured: true, reasons: [] });
|
|
212
|
+
});
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
describe('collectRebootPendingFacts', () => {
|
|
216
|
+
it('reads the marker and splits the deferred package list', () => {
|
|
217
|
+
const facts = collectRebootPendingFacts(
|
|
218
|
+
(path) => (path.endsWith('.pkgs') ? 'docker\n\ndbus\n' : ''),
|
|
219
|
+
'6.8.0-1063-raspi',
|
|
220
|
+
'6.8.0-1064-raspi',
|
|
221
|
+
);
|
|
222
|
+
|
|
223
|
+
expect(facts.markerPresent).toBe(true);
|
|
224
|
+
expect(facts.deferredPackages).toEqual(['docker', 'dbus']);
|
|
225
|
+
expect(facts.runningKernel).toBe('6.8.0-1063-raspi');
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
it('reports no marker when the file is absent', () => {
|
|
229
|
+
expect(collectRebootPendingFacts(() => null).markerPresent).toBe(false);
|
|
230
|
+
});
|
|
231
|
+
});
|