@celilo/cli 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/hooks/capability-loader.ts +49 -1
- package/src/policy/capability-shape-baseline.ts +6 -0
- package/src/policy/module-business-baseline.ts +17 -0
- package/src/services/provider-converge.test.ts +209 -0
- package/src/services/provider-converge.ts +241 -0
- package/src/services/static-content-converge.test.ts +39 -0
- package/src/services/static-content-converge.ts +26 -5
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/cli",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.1.0",
|
|
4
4
|
"description": "Celilo — home lab orchestration CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -58,7 +58,7 @@
|
|
|
58
58
|
"dependencies": {
|
|
59
59
|
"@aws-sdk/client-s3": "^3.1109.0",
|
|
60
60
|
"@aws-sdk/lib-storage": "^3.1101.0",
|
|
61
|
-
"@celilo/capabilities": "^5.
|
|
61
|
+
"@celilo/capabilities": "^5.1.0",
|
|
62
62
|
"@celilo/cli-display": "^0.2.0",
|
|
63
63
|
"@celilo/core": "^0.13.0",
|
|
64
64
|
"@celilo/event-bus": "^0.6.0",
|
|
@@ -23,6 +23,7 @@ import type {
|
|
|
23
23
|
DnsRegistrarCapability,
|
|
24
24
|
HookLogger,
|
|
25
25
|
PortForwardStore,
|
|
26
|
+
ProviderConvergeView,
|
|
26
27
|
RouteOps,
|
|
27
28
|
RouteReadView,
|
|
28
29
|
TrustedSourceStore,
|
|
@@ -654,7 +655,7 @@ export async function loadCapabilityFunctions(
|
|
|
654
655
|
// exact "served but silently unreachable" anti-pattern.
|
|
655
656
|
convergeStaticContent: async () => {
|
|
656
657
|
const { convergeStaticContent } = await import('../services/static-content-converge');
|
|
657
|
-
const result = await convergeStaticContent(db, provider.moduleId);
|
|
658
|
+
const result = await convergeStaticContent(db, provider.moduleId, consumingModuleId);
|
|
658
659
|
if (!result.success) {
|
|
659
660
|
throw new Error(result.error ?? 'static-content converge failed');
|
|
660
661
|
}
|
|
@@ -721,6 +722,53 @@ export async function loadCapabilityFunctions(
|
|
|
721
722
|
};
|
|
722
723
|
result.web_routes = routeView;
|
|
723
724
|
debugLog(`web_routes: read-only route view injected for provider ${consumingModuleId}`);
|
|
725
|
+
|
|
726
|
+
// The provider renders its own config; celilo makes the host match by
|
|
727
|
+
// running that provider's own role. The lookup is deliberately per
|
|
728
|
+
// module: core used to plan the whole fleet's sites and throw when one
|
|
729
|
+
// module had no built site, which stopped every other site converging
|
|
730
|
+
// (celilo#1383). See openspec/changes/providers-converge-declared-state.
|
|
731
|
+
const providerModuleId = provider.moduleId;
|
|
732
|
+
const providerConverge: ProviderConvergeView = {
|
|
733
|
+
resolveConsumerSite: async (moduleId: string) => {
|
|
734
|
+
const { resolveModuleStateWebRoot, resolveModuleWebRoot } = await import(
|
|
735
|
+
'../module/web-root'
|
|
736
|
+
);
|
|
737
|
+
const sourceDir = resolveModuleWebRoot(moduleId, db);
|
|
738
|
+
if (!sourceDir) {
|
|
739
|
+
return { unavailable: `module '${moduleId}' is not installed` };
|
|
740
|
+
}
|
|
741
|
+
if (!existsSync(sourceDir)) {
|
|
742
|
+
return {
|
|
743
|
+
unavailable: `no built site at ${sourceDir} — a module that serves a static site ships it at <module root>/site/dist`,
|
|
744
|
+
};
|
|
745
|
+
}
|
|
746
|
+
const stateWebRoot = resolveModuleStateWebRoot(moduleId, db);
|
|
747
|
+
const overlayDir =
|
|
748
|
+
stateWebRoot && existsSync(stateWebRoot) ? { overlayDir: stateWebRoot } : {};
|
|
749
|
+
return { sourceDir, ...overlayDir };
|
|
750
|
+
},
|
|
751
|
+
converge: async (artifacts) => {
|
|
752
|
+
const { convergeProviderConfig } = await import('../services/provider-converge');
|
|
753
|
+
const { resolveStaticContentRetention } = await import(
|
|
754
|
+
'../services/static-content-converge'
|
|
755
|
+
);
|
|
756
|
+
const result = await convergeProviderConfig(db, providerModuleId, {
|
|
757
|
+
...artifacts,
|
|
758
|
+
// Operator config, resolved here rather than in the provider: the
|
|
759
|
+
// module never carries the number (design D10 of
|
|
760
|
+
// capability-owned-tables).
|
|
761
|
+
retention: resolveStaticContentRetention(db, providerModuleId),
|
|
762
|
+
});
|
|
763
|
+
return {
|
|
764
|
+
success: result.success,
|
|
765
|
+
...(result.error ? { error: result.error } : {}),
|
|
766
|
+
unresolved: result.unresolved,
|
|
767
|
+
};
|
|
768
|
+
},
|
|
769
|
+
};
|
|
770
|
+
result.provider_converge = providerConverge;
|
|
771
|
+
debugLog(`provider_converge: injected for provider ${consumingModuleId}`);
|
|
724
772
|
}
|
|
725
773
|
} else {
|
|
726
774
|
debugLog('public_web: not registered in DB, skipping');
|
|
@@ -143,4 +143,10 @@ export const CAPABILITY_SHAPE_BASELINE: Readonly<Record<string, CapabilityShape>
|
|
|
143
143
|
requests:
|
|
144
144
|
'[{kind:"interface",members:[{name:"converge",optional:false,type:"Promise<void>"},{name:"withdrawConsumer",optional:false,type:"Promise<void>"}],name:"FirewallProviderView"}]',
|
|
145
145
|
},
|
|
146
|
+
provider_converge: {
|
|
147
|
+
version: '1.0.0',
|
|
148
|
+
hash: '1d828d70433ce4a15a73072883abad0c2acfef92c059bf40432b792722eb7d97',
|
|
149
|
+
requests:
|
|
150
|
+
'[{kind:"alias",members:[{name:"_self",optional:false,type:"{\\n sourceDir: string;\\n overlayDir?: string;\\n} | {\\n unavailable: string;\\n}"}],name:"ConsumerSiteResolution"},{kind:"interface",members:[{name:"content",optional:false,type:"string"},{name:"path",optional:false,type:"string"}],name:"ProviderConfigFile"},{kind:"interface",members:[{name:"converge",optional:false,type:"Promise<{\\n success: boolean;\\n error?: string;\\n unresolved: string[];\\n}>"},{name:"resolveConsumerSite",optional:false,type:"Promise<ConsumerSiteResolution>"}],name:"ProviderConvergeView"},{kind:"interface",members:[{name:"contentHash",optional:false,type:"string"},{name:"hostnames",optional:false,type:"string[]"},{name:"moduleId",optional:false,type:"string"},{name:"overlayDir",optional:true,type:"string"},{name:"slug",optional:false,type:"string"},{name:"sourceDir",optional:false,type:"string"}],name:"ProviderSiteRequest"}]',
|
|
151
|
+
},
|
|
146
152
|
};
|
|
@@ -99,6 +99,12 @@ export const CAPABILITY_NAME_BASELINE: readonly CapabilityNameRow[] = [
|
|
|
99
99
|
count: 4,
|
|
100
100
|
why: 'PERMANENT — a provider VIEW, not a module-provided capability. The name was registered so it stops being invisible to audits (celilo#1007); these references predate that and are framework plumbing, not core branching on a provider',
|
|
101
101
|
},
|
|
102
|
+
{
|
|
103
|
+
file: 'apps/celilo/src/hooks/capability-loader.ts',
|
|
104
|
+
capability: 'provider_converge',
|
|
105
|
+
count: 2,
|
|
106
|
+
why: 'PERMANENT — a provider VIEW, not a module-provided capability. Core injects the generic converge every provider renders through; naming it is framework plumbing, not core branching on a provider (providers-converge-declared-state)',
|
|
107
|
+
},
|
|
102
108
|
{
|
|
103
109
|
file: 'apps/celilo/src/db/schema.ts',
|
|
104
110
|
capability: 'web_routes',
|
|
@@ -129,6 +135,12 @@ export const CAPABILITY_NAME_BASELINE: readonly CapabilityNameRow[] = [
|
|
|
129
135
|
count: 1,
|
|
130
136
|
why: 'PERMANENT — a contract declaration with no implementation; this is the package doing its job',
|
|
131
137
|
},
|
|
138
|
+
{
|
|
139
|
+
file: 'packages/capabilities/src/capability-contract.ts',
|
|
140
|
+
capability: 'provider_converge',
|
|
141
|
+
count: 1,
|
|
142
|
+
why: 'PERMANENT — a contract declaration with no implementation; this is the package doing its job',
|
|
143
|
+
},
|
|
132
144
|
{
|
|
133
145
|
file: 'apps/celilo/src/capabilities/well-known.ts',
|
|
134
146
|
capability: 'dhcp_server',
|
|
@@ -466,6 +478,11 @@ export const SERVICE_FILENAME_BASELINE: readonly ServiceFilenameRow[] = [
|
|
|
466
478
|
capability: 'cross_module_read',
|
|
467
479
|
why: 'PERMANENT — cross_module_read is a framework privilege, not a module capability',
|
|
468
480
|
},
|
|
481
|
+
{
|
|
482
|
+
file: 'apps/celilo/src/services/provider-converge.ts',
|
|
483
|
+
capability: 'provider_converge',
|
|
484
|
+
why: 'PERMANENT — provider_converge is a framework-granted view, not a module capability, and this service IS the generic mechanism it grants: it carries what a provider rendered and runs that provider own role. Naming it for one provider is what the scan exists to catch; naming it for the mechanism is the fix the scan asks for (providers-converge-declared-state)',
|
|
485
|
+
},
|
|
469
486
|
{
|
|
470
487
|
file: 'apps/celilo/src/services/dns-internal-records.ts',
|
|
471
488
|
capability: 'dns_internal',
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { eq } from 'drizzle-orm';
|
|
6
|
+
import { parse as parseYaml } from 'yaml';
|
|
7
|
+
import type { DbClient } from '../db/client';
|
|
8
|
+
import { modules } from '../db/schema';
|
|
9
|
+
import { setupTestDatabaseAt } from '../test-utils/database';
|
|
10
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
11
|
+
import {
|
|
12
|
+
type ProviderArtifacts,
|
|
13
|
+
convergeProviderConfig,
|
|
14
|
+
providerConfigVarsPath,
|
|
15
|
+
providerConfigVarsYaml,
|
|
16
|
+
recordUnresolvedConsumers,
|
|
17
|
+
} from './provider-converge';
|
|
18
|
+
|
|
19
|
+
const HASH = 'a'.repeat(64);
|
|
20
|
+
|
|
21
|
+
function artifacts(overrides: Partial<ProviderArtifacts> = {}): ProviderArtifacts {
|
|
22
|
+
return {
|
|
23
|
+
files: [{ path: '/etc/caddy/Caddyfile', content: 'peterbanka.org {\n\ttls internal\n}\n' }],
|
|
24
|
+
sites: [
|
|
25
|
+
{
|
|
26
|
+
slug: 'peterbanka-org',
|
|
27
|
+
moduleId: 'peterbanka-org',
|
|
28
|
+
contentHash: HASH,
|
|
29
|
+
hostnames: ['peterbanka.org'],
|
|
30
|
+
sourceDir: '/var/celilo/modules/peterbanka-org/site/dist',
|
|
31
|
+
},
|
|
32
|
+
],
|
|
33
|
+
unresolved: [],
|
|
34
|
+
retention: 5,
|
|
35
|
+
...overrides,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
describe('providerConfigVarsYaml — the provider renders, celilo carries', () => {
|
|
40
|
+
it('carries each rendered file so the role can place it verbatim', () => {
|
|
41
|
+
const vars = parseYaml(providerConfigVarsYaml(artifacts()));
|
|
42
|
+
|
|
43
|
+
expect(vars.provider_config_files).toEqual([
|
|
44
|
+
{ path: '/etc/caddy/Caddyfile', content: 'peterbanka.org {\n\ttls internal\n}\n' },
|
|
45
|
+
]);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('carries the sites under the variable the static-content role already reads', () => {
|
|
49
|
+
const vars = parseYaml(providerConfigVarsYaml(artifacts()));
|
|
50
|
+
|
|
51
|
+
expect(vars.static_release_retention).toBe(5);
|
|
52
|
+
expect(vars.static_releases).toEqual([
|
|
53
|
+
{
|
|
54
|
+
slug: 'peterbanka-org',
|
|
55
|
+
content_hash: HASH,
|
|
56
|
+
hostnames: ['peterbanka.org'],
|
|
57
|
+
source_dir: '/var/celilo/modules/peterbanka-org/site/dist',
|
|
58
|
+
},
|
|
59
|
+
]);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('omits overlay_dir when there is none, so the role guard reads a clean variable', () => {
|
|
63
|
+
const withOverlay = artifacts({
|
|
64
|
+
sites: [
|
|
65
|
+
{ ...artifacts().sites[0], overlayDir: '/var/celilo/modules/peterbanka-org/state/site' },
|
|
66
|
+
],
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
expect(parseYaml(providerConfigVarsYaml(artifacts())).static_releases[0].overlay_dir).toBe(
|
|
70
|
+
undefined,
|
|
71
|
+
);
|
|
72
|
+
expect(parseYaml(providerConfigVarsYaml(withOverlay)).static_releases[0].overlay_dir).toBe(
|
|
73
|
+
'/var/celilo/modules/peterbanka-org/state/site',
|
|
74
|
+
);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("one consumer the provider could not render does not remove another's site", () => {
|
|
78
|
+
const partial = artifacts({
|
|
79
|
+
unresolved: [
|
|
80
|
+
{ moduleId: 'byoi', reason: 'no built site at /var/celilo/modules/byoi/site/dist' },
|
|
81
|
+
],
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
const vars = parseYaml(providerConfigVarsYaml(partial));
|
|
85
|
+
|
|
86
|
+
expect(vars.static_releases.map((r: { slug: string }) => r.slug)).toEqual(['peterbanka-org']);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('is deterministic, so an unchanged desired state writes an unchanged file', () => {
|
|
90
|
+
expect(providerConfigVarsYaml(artifacts())).toBe(providerConfigVarsYaml(artifacts()));
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
describe('providerConfigVarsPath — lands in the generated inventory', () => {
|
|
95
|
+
it('writes into the auto-loaded group_vars directory', () => {
|
|
96
|
+
expect(providerConfigVarsPath('/tmp/generated')).toBe(
|
|
97
|
+
'/tmp/generated/ansible/inventory/group_vars/all/provider_config.yml',
|
|
98
|
+
);
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
describe('recordUnresolvedConsumers — the failure belongs to the consumer', () => {
|
|
103
|
+
let dir: string;
|
|
104
|
+
let db: DbClient;
|
|
105
|
+
|
|
106
|
+
beforeEach(async () => {
|
|
107
|
+
dir = mkdtempSync(join(tmpdir(), 'provider-converge-'));
|
|
108
|
+
const dbPath = join(dir, 'celilo.db');
|
|
109
|
+
process.env.CELILO_DB_PATH = dbPath;
|
|
110
|
+
db = await setupTestDatabaseAt(dbPath);
|
|
111
|
+
|
|
112
|
+
for (const id of ['caddy', 'byoi', 'peterbanka-org']) {
|
|
113
|
+
db.insert(modules)
|
|
114
|
+
.values({
|
|
115
|
+
id,
|
|
116
|
+
name: id,
|
|
117
|
+
version: '1.0.0',
|
|
118
|
+
manifestData: {},
|
|
119
|
+
sourcePath: join(dir, 'modules', id),
|
|
120
|
+
state: 'VERIFIED',
|
|
121
|
+
})
|
|
122
|
+
.onConflictDoNothing()
|
|
123
|
+
.run();
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
afterEach(() => {
|
|
128
|
+
db.$client.close();
|
|
129
|
+
resetTestDbPath();
|
|
130
|
+
try {
|
|
131
|
+
rmSync(dir, { recursive: true, force: true });
|
|
132
|
+
} catch {
|
|
133
|
+
/* ignore */
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
function stateOf(id: string) {
|
|
138
|
+
return db.select().from(modules).where(eq(modules.id, id)).get();
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
it('records the consumer in ERROR with the reason it could not be rendered', () => {
|
|
142
|
+
recordUnresolvedConsumers(db, 'caddy', [
|
|
143
|
+
{ moduleId: 'byoi', reason: 'no built site at /var/celilo/modules/byoi/site/dist' },
|
|
144
|
+
]);
|
|
145
|
+
|
|
146
|
+
const byoi = stateOf('byoi');
|
|
147
|
+
expect(byoi?.state).toBe('ERROR');
|
|
148
|
+
expect(byoi?.errorMessage).toContain('no built site');
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it('leaves the provider and every other consumer alone', () => {
|
|
152
|
+
recordUnresolvedConsumers(db, 'caddy', [{ moduleId: 'byoi', reason: 'no built site' }]);
|
|
153
|
+
|
|
154
|
+
expect(stateOf('caddy')?.state).toBe('VERIFIED');
|
|
155
|
+
expect(stateOf('peterbanka-org')?.state).toBe('VERIFIED');
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it('returns the modules it recorded, so the converge can report them', () => {
|
|
159
|
+
const recorded = recordUnresolvedConsumers(db, 'caddy', [
|
|
160
|
+
{ moduleId: 'byoi', reason: 'no built site' },
|
|
161
|
+
]);
|
|
162
|
+
|
|
163
|
+
expect(recorded).toEqual(['byoi']);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
it('records the consumer even when the converge cannot run at all', async () => {
|
|
167
|
+
const result = await convergeProviderConfig(db, 'not-installed-provider', {
|
|
168
|
+
files: [],
|
|
169
|
+
sites: [],
|
|
170
|
+
unresolved: [{ moduleId: 'byoi', reason: 'no built site' }],
|
|
171
|
+
retention: 5,
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
expect(result.success).toBe(false);
|
|
175
|
+
expect(result.error).toContain('not installed');
|
|
176
|
+
// The consumer's fault is recorded whatever happens to the provider run:
|
|
177
|
+
// the two failures are unrelated and the operator needs both.
|
|
178
|
+
expect(result.unresolved).toEqual(['byoi']);
|
|
179
|
+
expect(stateOf('byoi')?.state).toBe('ERROR');
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it('refuses a generated project that predates the converge, naming the redeploy', async () => {
|
|
183
|
+
const generated = join(dir, 'modules', 'caddy', 'generated', 'ansible', 'roles');
|
|
184
|
+
mkdirSync(generated, { recursive: true });
|
|
185
|
+
writeFileSync(join(generated, 'main.yml'), '- name: does not read the new vars\n', 'utf-8');
|
|
186
|
+
|
|
187
|
+
const result = await convergeProviderConfig(db, 'caddy', artifacts());
|
|
188
|
+
|
|
189
|
+
expect(result.success).toBe(false);
|
|
190
|
+
expect(result.error).toContain('predates the provider converge');
|
|
191
|
+
expect(result.error).toContain('celilo module deploy caddy');
|
|
192
|
+
// Nothing was written: the guard scans the tree for the variable name, and
|
|
193
|
+
// the vars file would land inside that tree carrying it. Writing first
|
|
194
|
+
// makes the guard find its own output and pass forever (celilo#1248).
|
|
195
|
+
expect(result.varsPath).toBe(undefined);
|
|
196
|
+
expect(existsSync(providerConfigVarsPath(join(dir, 'modules', 'caddy', 'generated')))).toBe(
|
|
197
|
+
false,
|
|
198
|
+
);
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it('ignores a module that is not installed rather than inventing a row', () => {
|
|
202
|
+
const recorded = recordUnresolvedConsumers(db, 'caddy', [
|
|
203
|
+
{ moduleId: 'never-installed', reason: 'no built site' },
|
|
204
|
+
]);
|
|
205
|
+
|
|
206
|
+
expect(recorded).toEqual([]);
|
|
207
|
+
expect(stateOf('never-installed')).toBe(undefined);
|
|
208
|
+
});
|
|
209
|
+
});
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The provider converge (openspec/changes/providers-converge-declared-state).
|
|
3
|
+
*
|
|
4
|
+
* A provider renders its own config from the rows it declares, and celilo makes
|
|
5
|
+
* the host match by running that provider's own Ansible role. One writer on the
|
|
6
|
+
* box, one path from declared state to a running config.
|
|
7
|
+
*
|
|
8
|
+
* Generalised from `static-content-converge.ts`, which already does this for
|
|
9
|
+
* static site content: desired state into the provider's generated inventory,
|
|
10
|
+
* then a tag-scoped `executeAnsible`. The difference is who decides what the
|
|
11
|
+
* desired state is. There, core walks every route row in the fleet and throws
|
|
12
|
+
* when any one module's site is missing (celilo#1383 — two out-of-tree modules
|
|
13
|
+
* with no `site/dist` made the public ingress undeployable). Here the provider
|
|
14
|
+
* hands over what it rendered, and a consumer it could not render is recorded
|
|
15
|
+
* against that consumer.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { existsSync, rmSync } from 'node:fs';
|
|
19
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
20
|
+
import { join } from 'node:path';
|
|
21
|
+
import { eq } from 'drizzle-orm';
|
|
22
|
+
import { stringify as stringifyYaml } from 'yaml';
|
|
23
|
+
import type { DbClient } from '../db/client';
|
|
24
|
+
import { modules } from '../db/schema';
|
|
25
|
+
import { ansibleTreeMentions } from './static-content-converge';
|
|
26
|
+
|
|
27
|
+
/** One file the provider wants on its host, rendered by the provider. */
|
|
28
|
+
export interface ProviderConfigFile {
|
|
29
|
+
/** Absolute path on the provider host. */
|
|
30
|
+
path: string;
|
|
31
|
+
/** The rendered contents. */
|
|
32
|
+
content: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** A consumer's site the provider host must carry. */
|
|
36
|
+
export interface ProviderSite {
|
|
37
|
+
slug: string;
|
|
38
|
+
moduleId: string;
|
|
39
|
+
contentHash: string;
|
|
40
|
+
hostnames: string[];
|
|
41
|
+
sourceDir: string;
|
|
42
|
+
overlayDir?: string;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A consumer the provider could not render, and why. */
|
|
46
|
+
export interface UnresolvedConsumer {
|
|
47
|
+
moduleId: string;
|
|
48
|
+
reason: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Everything the provider rendered for one converge. */
|
|
52
|
+
export interface ProviderArtifacts {
|
|
53
|
+
files: ProviderConfigFile[];
|
|
54
|
+
sites: ProviderSite[];
|
|
55
|
+
unresolved: UnresolvedConsumer[];
|
|
56
|
+
retention: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Where the provider's desired state lands inside its generated project.
|
|
61
|
+
*
|
|
62
|
+
* `group_vars/all/` is the existing auto-load directory — the same one
|
|
63
|
+
* `static_content.yml` uses — so no playbook change is needed to read it.
|
|
64
|
+
*/
|
|
65
|
+
export function providerConfigVarsPath(generatedPath: string): string {
|
|
66
|
+
return join(generatedPath, 'ansible', 'inventory', 'group_vars', 'all', 'provider_config.yml');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Render the vars file. Presentation function (Rule 10.1) — deterministic YAML,
|
|
71
|
+
* so an unchanged desired state writes a byte-identical file and the role's
|
|
72
|
+
* config task reports no change.
|
|
73
|
+
*
|
|
74
|
+
* `static_releases` keeps its name: the role tasks that converge `/srv/www`
|
|
75
|
+
* already read it, and renaming a working variable to match a new writer would
|
|
76
|
+
* be churn with a live-site failure mode.
|
|
77
|
+
*/
|
|
78
|
+
export function providerConfigVarsYaml(artifacts: ProviderArtifacts): string {
|
|
79
|
+
const vars = {
|
|
80
|
+
provider_config_files: artifacts.files.map((file) => ({
|
|
81
|
+
path: file.path,
|
|
82
|
+
content: file.content,
|
|
83
|
+
})),
|
|
84
|
+
static_release_retention: artifacts.retention,
|
|
85
|
+
static_releases: artifacts.sites.map((site) => ({
|
|
86
|
+
slug: site.slug,
|
|
87
|
+
content_hash: site.contentHash,
|
|
88
|
+
hostnames: site.hostnames,
|
|
89
|
+
source_dir: site.sourceDir,
|
|
90
|
+
// Absent when there is no overlay, so the role's `when:` guard reads a
|
|
91
|
+
// clean variable rather than an empty string a copy task would choke on.
|
|
92
|
+
...(site.overlayDir ? { overlay_dir: site.overlayDir } : {}),
|
|
93
|
+
})),
|
|
94
|
+
};
|
|
95
|
+
const header =
|
|
96
|
+
'# Desired provider state, generated by celilo (providers-converge-declared-state).\n' +
|
|
97
|
+
'# Hand edits are overwritten by the next converge.\n';
|
|
98
|
+
return `${header}${stringifyYaml(vars, { lineWidth: 0, sortMapEntries: true })}`;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Record each consumer the provider could not render, against that consumer.
|
|
103
|
+
*
|
|
104
|
+
* The provider is not at fault for a consumer that ships no built site, and the
|
|
105
|
+
* fleet is not at fault either: before this, one such module threw in core's
|
|
106
|
+
* plan and no site converged at all (celilo#1383). The state has to be honest
|
|
107
|
+
* about which module is broken (celilo#1363), so the row says ERROR and carries
|
|
108
|
+
* the reason the provider gave.
|
|
109
|
+
*
|
|
110
|
+
* A module that is not installed is skipped rather than inserted: celilo cannot
|
|
111
|
+
* record a state for something it does not have.
|
|
112
|
+
*/
|
|
113
|
+
export function recordUnresolvedConsumers(
|
|
114
|
+
db: DbClient,
|
|
115
|
+
providerModuleId: string,
|
|
116
|
+
unresolved: UnresolvedConsumer[],
|
|
117
|
+
): string[] {
|
|
118
|
+
const recorded: string[] = [];
|
|
119
|
+
|
|
120
|
+
for (const consumer of unresolved) {
|
|
121
|
+
const existing = db.select().from(modules).where(eq(modules.id, consumer.moduleId)).get();
|
|
122
|
+
if (!existing) continue;
|
|
123
|
+
|
|
124
|
+
db.update(modules)
|
|
125
|
+
.set({
|
|
126
|
+
state: 'ERROR',
|
|
127
|
+
errorMessage: `'${providerModuleId}' could not serve this module: ${consumer.reason}`,
|
|
128
|
+
updatedAt: new Date(),
|
|
129
|
+
})
|
|
130
|
+
.where(eq(modules.id, consumer.moduleId))
|
|
131
|
+
.run();
|
|
132
|
+
recorded.push(consumer.moduleId);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
return recorded;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** What one converge did, for the caller to report. */
|
|
139
|
+
export interface ProviderConvergeResult {
|
|
140
|
+
success: boolean;
|
|
141
|
+
error?: string;
|
|
142
|
+
/** The written vars file, absolute — for logs and tests. */
|
|
143
|
+
varsPath?: string;
|
|
144
|
+
/** Consumers recorded in ERROR because the provider could not render them. */
|
|
145
|
+
unresolved: string[];
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The tags a converge runs. The provider's whole playbook re-runs package
|
|
150
|
+
* installation and service enablement, which a route change has no business
|
|
151
|
+
* touching; these two cover the file placement and `/srv/www`.
|
|
152
|
+
*/
|
|
153
|
+
const CONVERGE_TAGS = ['provider_config', 'static_content'];
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Plan, write, converge: make the provider's host match what the provider
|
|
157
|
+
* rendered.
|
|
158
|
+
*
|
|
159
|
+
* `generatedPath` is rendered on demand when it is missing, because a
|
|
160
|
+
* successful deploy deletes the generated tree by design (D4 of
|
|
161
|
+
* control-plane-stops-building-modules) and generation is fast and local.
|
|
162
|
+
*/
|
|
163
|
+
export async function convergeProviderConfig(
|
|
164
|
+
db: DbClient,
|
|
165
|
+
providerModuleId: string,
|
|
166
|
+
artifacts: ProviderArtifacts,
|
|
167
|
+
options?: { tags?: string[] },
|
|
168
|
+
): Promise<ProviderConvergeResult> {
|
|
169
|
+
const unresolved = recordUnresolvedConsumers(db, providerModuleId, artifacts.unresolved);
|
|
170
|
+
|
|
171
|
+
const module = db.select().from(modules).where(eq(modules.id, providerModuleId)).get();
|
|
172
|
+
if (!module) {
|
|
173
|
+
return {
|
|
174
|
+
success: false,
|
|
175
|
+
error: `Provider module '${providerModuleId}' is not installed`,
|
|
176
|
+
unresolved,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const generatedPath = join(module.sourcePath, 'generated');
|
|
181
|
+
if (!existsSync(generatedPath)) {
|
|
182
|
+
const { generateTemplates } = await import('../templates/generator');
|
|
183
|
+
const rendered = await generateTemplates({
|
|
184
|
+
moduleId: providerModuleId,
|
|
185
|
+
modulePath: module.sourcePath,
|
|
186
|
+
outputPath: generatedPath,
|
|
187
|
+
db,
|
|
188
|
+
skipVariableValidation: false,
|
|
189
|
+
});
|
|
190
|
+
if (!rendered.success) {
|
|
191
|
+
return {
|
|
192
|
+
success: false,
|
|
193
|
+
error:
|
|
194
|
+
`Provider '${providerModuleId}' has no generated deploy project at ${generatedPath} ` +
|
|
195
|
+
`and rendering one failed: ${rendered.error ?? 'unknown error'}. ` +
|
|
196
|
+
`Run \`celilo module deploy ${providerModuleId}\` once, then retry.`,
|
|
197
|
+
unresolved,
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// A generated project that predates the converge role tasks would read the
|
|
203
|
+
// new vars file and place nothing, so the desired state would be written and
|
|
204
|
+
// silently never applied. The variable appearing nowhere in the tree is the
|
|
205
|
+
// tell. Generic string check — core names no module.
|
|
206
|
+
//
|
|
207
|
+
// CHECKED BEFORE THE VARS ARE WRITTEN, and the order is the whole point: the
|
|
208
|
+
// vars file lands inside the tree being scanned and contains the variable
|
|
209
|
+
// name, so a guard that runs after the write finds its own output and can
|
|
210
|
+
// never fail. That is celilo#1248's bug — a check that cannot reach its
|
|
211
|
+
// subject — reintroduced by write order rather than by path.
|
|
212
|
+
const ansiblePath = join(generatedPath, 'ansible');
|
|
213
|
+
if (existsSync(ansiblePath) && !ansibleTreeMentions(ansiblePath, 'provider_config_files')) {
|
|
214
|
+
return {
|
|
215
|
+
success: false,
|
|
216
|
+
error: `The generated project for '${providerModuleId}' predates the provider converge (its role never reads provider_config_files). Redeploy the provider (\`celilo module deploy ${providerModuleId}\`) to regenerate it, then retry.`,
|
|
217
|
+
unresolved,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
const varsPath = providerConfigVarsPath(generatedPath);
|
|
222
|
+
await mkdir(join(varsPath, '..'), { recursive: true });
|
|
223
|
+
await writeFile(varsPath, providerConfigVarsYaml(artifacts), 'utf-8');
|
|
224
|
+
|
|
225
|
+
const { executeAnsible } = await import('./deploy-ansible');
|
|
226
|
+
const result = await executeAnsible(generatedPath, { tags: options?.tags ?? CONVERGE_TAGS });
|
|
227
|
+
if (!result.success) {
|
|
228
|
+
return {
|
|
229
|
+
success: false,
|
|
230
|
+
error: `Provider converge failed on ${providerModuleId}: ${result.error ?? 'unknown error'}`,
|
|
231
|
+
varsPath,
|
|
232
|
+
unresolved,
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// The converge rendered the tree, used it, and is done with it: desired state
|
|
237
|
+
// is rebuilt from the declared rows every time and never kept as a record
|
|
238
|
+
// (design D5). On the failure paths above the tree stays for inspection.
|
|
239
|
+
rmSync(generatedPath, { recursive: true, force: true });
|
|
240
|
+
return { success: true, varsPath, unresolved };
|
|
241
|
+
}
|
|
@@ -152,6 +152,45 @@ describe('planStaticContent — the declared release set (D10 task 4.1)', () =>
|
|
|
152
152
|
|
|
153
153
|
expect(() => planStaticContent(db)).toThrow(/no built site/);
|
|
154
154
|
});
|
|
155
|
+
|
|
156
|
+
function pause(moduleId: string) {
|
|
157
|
+
db.update(modules).set({ state: 'PAUSED' }).where(eq(modules.id, moduleId)).run();
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
it("a paused module's slug is left alone, even one that would throw, and the rest converge (celilo#1383)", () => {
|
|
161
|
+
insertRoute({ slug: 'lunacycle', moduleId: 'lunacycle', hostname: 'lunacycle.net' });
|
|
162
|
+
insertRoute({
|
|
163
|
+
slug: 'lunacycle',
|
|
164
|
+
moduleId: 'lunacycle',
|
|
165
|
+
hostname: 'www.lunacycle.net',
|
|
166
|
+
contentHash: HASH2,
|
|
167
|
+
});
|
|
168
|
+
rmSync(join(dir, 'modules', 'lunacycle', 'site', 'dist'), { recursive: true, force: true });
|
|
169
|
+
insertRoute({
|
|
170
|
+
slug: 'hello-foo',
|
|
171
|
+
moduleId: 'hello-foo',
|
|
172
|
+
hostname: 'www.example.com',
|
|
173
|
+
path: '/foo',
|
|
174
|
+
});
|
|
175
|
+
pause('lunacycle');
|
|
176
|
+
|
|
177
|
+
expect(planStaticContent(db).map((r) => r.slug)).toEqual(['hello-foo']);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
it('a paused module that is publishing (an unpause redeploy) still converges its own slug', () => {
|
|
181
|
+
insertRoute({ slug: 'lunacycle', moduleId: 'lunacycle', hostname: 'lunacycle.net' });
|
|
182
|
+
pause('lunacycle');
|
|
183
|
+
|
|
184
|
+
expect(planStaticContent(db, 'lunacycle').map((r) => r.slug)).toEqual(['lunacycle']);
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it('a slug shared by a paused and an active module is still a contradiction', () => {
|
|
188
|
+
insertRoute({ slug: 'about', moduleId: 'lunacycle', hostname: 'a.example.com' });
|
|
189
|
+
insertRoute({ slug: 'about', moduleId: 'hello-foo', hostname: 'b.example.com' });
|
|
190
|
+
pause('lunacycle');
|
|
191
|
+
|
|
192
|
+
expect(() => planStaticContent(db)).toThrow(/claimed by 2 modules/);
|
|
193
|
+
});
|
|
155
194
|
});
|
|
156
195
|
|
|
157
196
|
describe('resolveStaticContentRetention — operator config with a default', () => {
|
|
@@ -32,6 +32,7 @@ import type { DbClient } from '../db/client';
|
|
|
32
32
|
import { moduleConfigs, modules, webRoutes } from '../db/schema';
|
|
33
33
|
import { resolveModuleStateWebRoot, resolveModuleWebRoot } from '../module/web-root';
|
|
34
34
|
import { parseStoredConfigValue } from './module-config';
|
|
35
|
+
import { listPausedModules } from './module-pause';
|
|
35
36
|
|
|
36
37
|
/** One content-hashed release the provider host must serve. */
|
|
37
38
|
export interface StaticRelease {
|
|
@@ -133,7 +134,7 @@ export function resolveStaticContentRetention(db: DbClient, providerModuleId: st
|
|
|
133
134
|
* cannot express, so it fails here with both rows named rather than silently
|
|
134
135
|
* converging one of them.
|
|
135
136
|
*/
|
|
136
|
-
export function planStaticContent(db: DbClient): StaticRelease[] {
|
|
137
|
+
export function planStaticContent(db: DbClient, publishingModuleId?: string): StaticRelease[] {
|
|
137
138
|
const rows = db
|
|
138
139
|
.select()
|
|
139
140
|
.from(webRoutes)
|
|
@@ -147,6 +148,14 @@ export function planStaticContent(db: DbClient): StaticRelease[] {
|
|
|
147
148
|
bySlug.set(row.slug, group);
|
|
148
149
|
}
|
|
149
150
|
|
|
151
|
+
// A paused module's site is left exactly as it is — pause preserves state
|
|
152
|
+
// (module-pause spec) — so its slug is neither converged nor validated. The
|
|
153
|
+
// exception is the module publishing right now: an unpause redeploy runs
|
|
154
|
+
// on_install while the module is still PAUSED (executeUnpause clears the
|
|
155
|
+
// pause only after the redeploy succeeds). celilo#1383.
|
|
156
|
+
const paused = new Set(listPausedModules(db).map((m) => m.id));
|
|
157
|
+
if (publishingModuleId) paused.delete(publishingModuleId);
|
|
158
|
+
|
|
150
159
|
const releases: StaticRelease[] = [];
|
|
151
160
|
for (const [slug, group] of bySlug) {
|
|
152
161
|
const moduleIds = [...new Set(group.map((r) => r.moduleId))];
|
|
@@ -157,6 +166,7 @@ export function planStaticContent(db: DbClient): StaticRelease[] {
|
|
|
157
166
|
}
|
|
158
167
|
const moduleId = moduleIds[0];
|
|
159
168
|
if (!moduleId) continue;
|
|
169
|
+
if (paused.has(moduleId)) continue;
|
|
160
170
|
|
|
161
171
|
const hashes = [...new Set(group.map((r) => r.contentHash))];
|
|
162
172
|
if (hashes.length > 1) {
|
|
@@ -201,9 +211,13 @@ export function planStaticContent(db: DbClient): StaticRelease[] {
|
|
|
201
211
|
* The full desired state: the release set plus the retention count resolved
|
|
202
212
|
* for THIS provider module. The unit the vars file and the tests speak in.
|
|
203
213
|
*/
|
|
204
|
-
export function buildStaticContentPlan(
|
|
214
|
+
export function buildStaticContentPlan(
|
|
215
|
+
db: DbClient,
|
|
216
|
+
providerModuleId: string,
|
|
217
|
+
publishingModuleId?: string,
|
|
218
|
+
): StaticContentPlan {
|
|
205
219
|
return {
|
|
206
|
-
releases: planStaticContent(db),
|
|
220
|
+
releases: planStaticContent(db, publishingModuleId),
|
|
207
221
|
retention: resolveStaticContentRetention(db, providerModuleId),
|
|
208
222
|
};
|
|
209
223
|
}
|
|
@@ -246,10 +260,11 @@ export async function writeStaticContentVars(
|
|
|
246
260
|
db: DbClient,
|
|
247
261
|
providerModuleId: string,
|
|
248
262
|
generatedPath: string,
|
|
263
|
+
publishingModuleId?: string,
|
|
249
264
|
): Promise<StaticContentConvergeResult> {
|
|
250
265
|
const varsPath = staticContentVarsPath(generatedPath);
|
|
251
266
|
|
|
252
|
-
const plan = buildStaticContentPlan(db, providerModuleId);
|
|
267
|
+
const plan = buildStaticContentPlan(db, providerModuleId, publishingModuleId);
|
|
253
268
|
|
|
254
269
|
await mkdir(join(varsPath, '..'), { recursive: true });
|
|
255
270
|
await writeFile(varsPath, staticContentVarsYaml(plan), 'utf-8');
|
|
@@ -303,6 +318,7 @@ export function ansibleTreeMentions(dir: string, needle: string): boolean {
|
|
|
303
318
|
export async function convergeStaticContent(
|
|
304
319
|
db: DbClient,
|
|
305
320
|
providerModuleId: string,
|
|
321
|
+
publishingModuleId?: string,
|
|
306
322
|
): Promise<StaticContentConvergeResult> {
|
|
307
323
|
const module = db.select().from(modules).where(eq(modules.id, providerModuleId)).get();
|
|
308
324
|
if (!module) {
|
|
@@ -336,7 +352,12 @@ export async function convergeStaticContent(
|
|
|
336
352
|
}
|
|
337
353
|
}
|
|
338
354
|
|
|
339
|
-
const written = await writeStaticContentVars(
|
|
355
|
+
const written = await writeStaticContentVars(
|
|
356
|
+
db,
|
|
357
|
+
providerModuleId,
|
|
358
|
+
generatedPath,
|
|
359
|
+
publishingModuleId,
|
|
360
|
+
);
|
|
340
361
|
|
|
341
362
|
// A generated project that predates the converge role tasks would read the
|
|
342
363
|
// new vars file and converge nothing. `static_releases` appearing nowhere in
|