@celilo/cli 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/drizzle/0031_module_config_source.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +2 -2
- package/schemas/system_config.json +2 -1
- package/src/capabilities/public-web-publish.test.ts +61 -0
- package/src/cli/commands/firewall-interface-list.test.ts +156 -7
- package/src/cli/commands/firewall-interface-list.ts +73 -7
- package/src/cli/commands/machine-add.ts +12 -55
- package/src/cli/commands/module-config.test.ts +20 -1
- package/src/cli/commands/module-import.ts +1 -1
- package/src/cli/commands/module-update.test.ts +82 -0
- package/src/cli/commands/module-update.ts +14 -4
- package/src/cli/commands/monitor.ts +2 -10
- package/src/cli/commands/restore.ts +16 -6
- package/src/cli/generate-zsh-completion.ts +1 -1
- package/src/cli/index.ts +4 -3
- package/src/cli/restore-migration-failure.test.ts +159 -0
- package/src/db/client.ts +5 -0
- package/src/db/migrate.test.ts +61 -135
- package/src/db/migrate.ts +7 -2
- package/src/db/schema.ts +10 -0
- package/src/hooks/broker.test.ts +106 -2
- package/src/hooks/broker.ts +91 -1
- package/src/hooks/capability-loader-firewall.test.ts +37 -0
- package/src/hooks/capability-loader.ts +15 -1
- package/src/hooks/define-hook.test.ts +4 -3
- package/src/hooks/executor.test.ts +19 -18
- package/src/hooks/executor.ts +82 -11
- package/src/hooks/hook-jail-toolchain-reach.test.ts +3 -2
- package/src/hooks/hook-jail-unreachability.test.ts +4 -3
- package/src/hooks/hook-protocol.ts +46 -1
- package/src/hooks/hook-runner.ts +36 -0
- package/src/hooks/hook-store-proxy.test.ts +109 -0
- package/src/hooks/hook-store-proxy.ts +85 -0
- package/src/hooks/hook-store.test.ts +162 -0
- package/src/hooks/hook-store.ts +290 -0
- package/src/hooks/hook-timeout.test.ts +3 -2
- package/src/hooks/hook-trespass.test.ts +29 -5
- package/src/hooks/jail.test.ts +1 -1
- package/src/hooks/jail.ts +14 -7
- package/src/hooks/mount-set.test.ts +208 -0
- package/src/hooks/mount-set.ts +62 -14
- package/src/hooks/run-named-hook.ts +2 -0
- package/src/hooks/test-fixtures/jail-probe-hook.ts +1 -1
- package/src/hooks/test-fixtures/on-restore-staging-hook.ts +26 -0
- package/src/hooks/test-fixtures/store-backed.ts +47 -0
- package/src/hooks/test-fixtures/store-writing-hook.ts +63 -0
- package/src/hooks/unjailed-lint.test.ts +22 -6
- package/src/manifest/schema.ts +1 -0
- package/src/module/packaging/build.ts +70 -2
- package/src/module/web-root.ts +17 -1
- package/src/policy/fixture-capability-coverage.test.ts +322 -0
- package/src/policy/module-script-scan.test.ts +42 -1
- package/src/policy/module-script-scan.ts +275 -5
- package/src/policy/no-hand-built-ssh.test.ts +34 -1
- package/src/policy/no-swallowed-refusal.test.ts +265 -0
- package/src/policy/no-tar-shell-out-in-services.test.ts +43 -0
- package/src/registry/client.test.ts +149 -0
- package/src/registry/client.ts +203 -11
- package/src/services/alerting/format.test.ts +57 -0
- package/src/services/alerting/format.ts +24 -0
- package/src/services/alerting/run-monitor.ts +2 -2
- package/src/services/backup-create.ts +7 -7
- package/src/services/backup-envelope-roundtrip.test.ts +45 -2
- package/src/services/backup-restore.ts +8 -4
- package/src/services/bus-interview.ts +37 -14
- package/src/services/config-provenance.ts +4 -0
- package/src/services/control-plane-bootstrap.test.ts +121 -1
- package/src/services/control-plane-bootstrap.ts +51 -4
- package/src/services/deploy-preflight.ts +8 -2
- package/src/services/deploy-validation.test.ts +22 -0
- package/src/services/deploy-validation.ts +8 -0
- package/src/services/dns-discovery.test.ts +54 -0
- package/src/services/dns-discovery.ts +47 -5
- package/src/services/fleet-key.test.ts +66 -2
- package/src/services/fleet-key.ts +54 -0
- package/src/services/health-runner.ts +2 -0
- package/src/services/module-config.ts +20 -2
- package/src/services/module-deploy.dns-repoint.test.ts +187 -0
- package/src/services/module-deploy.ts +163 -1
- package/src/services/module-validator/git-hygiene.test.ts +122 -3
- package/src/services/module-validator/git-hygiene.ts +83 -14
- package/src/services/restore-from-file.test.ts +20 -0
- package/src/services/restore-from-file.ts +21 -6
- package/src/services/static-content-converge.test.ts +140 -2
- package/src/services/static-content-converge.ts +55 -8
- package/src/services/system-config-schema-types.ts +1 -1
- package/src/services/system-config-validator.test.ts +36 -0
- package/src/services/system-config-validator.ts +11 -0
- package/src/services/trusted-sources.test.ts +30 -0
- package/src/services/trusted-sources.ts +47 -10
- package/src/templates/generator.ts +9 -2
- package/src/variables/context.ts +16 -5
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { spawnSync } from 'node:child_process';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import { classifyModulePath } from '../../module/packaging/package-rules';
|
|
3
4
|
import { collectGitInfo, makeRealGitRunner } from '../../module/packaging/release-metadata';
|
|
4
5
|
import type { Check } from './types';
|
|
5
6
|
|
|
@@ -19,10 +20,14 @@ export interface StalenessIssue {
|
|
|
19
20
|
* anywhere should still talk to lunacycle's git history).
|
|
20
21
|
*/
|
|
21
22
|
function lastCommitTouching(cwd: string, pathspec: string[]): string | null {
|
|
23
|
+
// timeout: a hung git degrades to the "can't determine — skip" path (status
|
|
24
|
+
// null) instead of hanging `module check` forever. 30s is enormous against
|
|
25
|
+
// the milliseconds a bounded `git log -1` takes on any real repo.
|
|
22
26
|
const r = spawnSync('git', ['log', '-1', '--format=%H', '--', ...pathspec], {
|
|
23
27
|
cwd,
|
|
24
28
|
stdio: ['ignore', 'pipe', 'ignore'],
|
|
25
29
|
encoding: 'utf-8',
|
|
30
|
+
timeout: 30_000,
|
|
26
31
|
});
|
|
27
32
|
if (r.status !== 0) return null;
|
|
28
33
|
const sha = r.stdout.trim();
|
|
@@ -33,17 +38,90 @@ function isAncestor(cwd: string, maybeAncestor: string, descendant: string): boo
|
|
|
33
38
|
const r = spawnSync('git', ['merge-base', '--is-ancestor', maybeAncestor, descendant], {
|
|
34
39
|
cwd,
|
|
35
40
|
stdio: 'ignore',
|
|
41
|
+
timeout: 30_000,
|
|
36
42
|
});
|
|
37
43
|
return r.status === 0;
|
|
38
44
|
}
|
|
39
45
|
|
|
46
|
+
/**
|
|
47
|
+
* One commit from `git log --name-only`: its SHA and the repo-root-relative
|
|
48
|
+
* paths it changed. A merge commit carries an empty file list (git log omits
|
|
49
|
+
* diffs for merges), so it never qualifies as a src commit — which is right,
|
|
50
|
+
* a merge changes no authored byte of its own.
|
|
51
|
+
*/
|
|
52
|
+
interface CommitWithPaths {
|
|
53
|
+
sha: string;
|
|
54
|
+
paths: string[];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The commits touching `pathspec`, newest first, with the repo-root-relative
|
|
59
|
+
* paths each one changed. `git log --name-only` prints repo-root-relative
|
|
60
|
+
* paths even when cwd is a subdirectory (measured: it does not honor cwd for
|
|
61
|
+
* output), so the caller strips the prefix itself via `show-prefix`.
|
|
62
|
+
*/
|
|
63
|
+
function listCommitsTouching(cwd: string, pathspec: string): CommitWithPaths[] {
|
|
64
|
+
const r = spawnSync(
|
|
65
|
+
'git',
|
|
66
|
+
['-c', 'core.quotePath=false', 'log', '--format=%H', '--name-only', '--', pathspec],
|
|
67
|
+
{ cwd, stdio: ['ignore', 'pipe', 'ignore'], encoding: 'utf-8' },
|
|
68
|
+
);
|
|
69
|
+
if (r.status !== 0) return [];
|
|
70
|
+
const commits: CommitWithPaths[] = [];
|
|
71
|
+
let current: CommitWithPaths | null = null;
|
|
72
|
+
for (const line of (r.stdout ?? '').split('\n')) {
|
|
73
|
+
const trimmed = line.trim();
|
|
74
|
+
if (!trimmed) continue;
|
|
75
|
+
if (/^[0-9a-f]{40}$/.test(trimmed)) {
|
|
76
|
+
current = { sha: trimmed, paths: [] };
|
|
77
|
+
commits.push(current);
|
|
78
|
+
} else if (current) {
|
|
79
|
+
current.paths.push(trimmed);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return commits;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* SHA of the last commit that touched SHIPPED module source, or null when no
|
|
87
|
+
* commit qualifies (no history, or only non-shipped paths ever committed).
|
|
88
|
+
*
|
|
89
|
+
* The staleness scan's reach is the PACKAGER's rule, not a second list beside
|
|
90
|
+
* it (celilo#1270). A path counts as source exactly when `classifyModulePath`
|
|
91
|
+
* calls it `package` — the same answer that decides whether the path lands in
|
|
92
|
+
* the .netapp. So a commit touching only `e2e/**`, `*.test.ts`, or
|
|
93
|
+
* `tsconfig.json` changes no installed byte and must not read as drift.
|
|
94
|
+
*
|
|
95
|
+
* Excluding manifest.yml from the "src" scan is the gate's own semantics, not
|
|
96
|
+
* packaging: we want to know if anything else shipped past it. node_modules,
|
|
97
|
+
* dist, and other build outputs are gitignored and so never appear here.
|
|
98
|
+
*/
|
|
99
|
+
function lastShippedSourceCommit(moduleDir: string): string | null {
|
|
100
|
+
const prefixR = spawnSync('git', ['rev-parse', '--show-prefix'], {
|
|
101
|
+
cwd: moduleDir,
|
|
102
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
103
|
+
encoding: 'utf-8',
|
|
104
|
+
});
|
|
105
|
+
if (prefixR.status !== 0) return null;
|
|
106
|
+
const repoPrefix = (prefixR.stdout ?? '').trim();
|
|
107
|
+
for (const commit of listCommitsTouching(moduleDir, moduleDir)) {
|
|
108
|
+
for (const p of commit.paths) {
|
|
109
|
+
if (repoPrefix && !p.startsWith(repoPrefix)) continue;
|
|
110
|
+
const rel = repoPrefix ? p.slice(repoPrefix.length) : p;
|
|
111
|
+
if (rel === 'manifest.yml') continue;
|
|
112
|
+
if (classifyModulePath(rel) === 'package') return commit.sha;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
|
|
40
118
|
/**
|
|
41
119
|
* Detect "I edited module src but forgot to bump (or touch) manifest.yml."
|
|
42
120
|
*
|
|
43
|
-
* Returns null when the manifest is the most recently-touched file in
|
|
44
|
-
* dir (or when neither side has any commit history — e.g. brand-new module
|
|
45
|
-
* not yet committed). Returns a StalenessIssue when src has commits
|
|
46
|
-
* the last manifest.yml change — the operator must bump the manifest
|
|
121
|
+
* Returns null when the manifest is the most recently-touched shipped file in
|
|
122
|
+
* the dir (or when neither side has any commit history — e.g. brand-new module
|
|
123
|
+
* not yet committed). Returns a StalenessIssue when shipped src has commits
|
|
124
|
+
* AFTER the last manifest.yml change — the operator must bump the manifest
|
|
47
125
|
* (semver change → reset +N to 1) or just touch it (release-only change →
|
|
48
126
|
* auto-bump +N), then re-publish.
|
|
49
127
|
*
|
|
@@ -54,16 +132,7 @@ function isAncestor(cwd: string, maybeAncestor: string, descendant: string): boo
|
|
|
54
132
|
export function checkModuleStale(moduleDir: string): StalenessIssue | null {
|
|
55
133
|
const manifestPath = join(moduleDir, 'manifest.yml');
|
|
56
134
|
const lastManifest = lastCommitTouching(moduleDir, [manifestPath]);
|
|
57
|
-
|
|
58
|
-
// want to know if anything ELSE in the dir moved past it. node_modules
|
|
59
|
-
// and common build outputs are gitignored already, but be explicit
|
|
60
|
-
// defensively.
|
|
61
|
-
const lastSrc = lastCommitTouching(moduleDir, [
|
|
62
|
-
moduleDir,
|
|
63
|
-
`:(exclude)${manifestPath}`,
|
|
64
|
-
`:(exclude)${moduleDir}/node_modules`,
|
|
65
|
-
`:(exclude)${moduleDir}/dist`,
|
|
66
|
-
]);
|
|
135
|
+
const lastSrc = lastShippedSourceCommit(moduleDir);
|
|
67
136
|
if (!lastManifest || !lastSrc) return null;
|
|
68
137
|
if (lastSrc === lastManifest) return null;
|
|
69
138
|
if (!isAncestor(moduleDir, lastManifest, lastSrc)) return null;
|
|
@@ -86,6 +86,26 @@ describe('applyStagedSystemFiles', () => {
|
|
|
86
86
|
expect(existsSync(`${livePath}-shm`)).toBe(false);
|
|
87
87
|
});
|
|
88
88
|
|
|
89
|
+
// celilo#1293 live repro 2026-09-06: the restore process's own pre-swap
|
|
90
|
+
// connection can outlive closeDb() at the fd level (measured: bun:sqlite's
|
|
91
|
+
// close() returned while the fds stayed open), and an IN-PLACE copy
|
|
92
|
+
// overwrites the inode that connection holds locks on — the swapped-in DB
|
|
93
|
+
// then inherits locks it must not have, and the next opener's DELETE→WAL
|
|
94
|
+
// conversion fails with "database is locked". The swap must therefore
|
|
95
|
+
// replace the INODE (rename), so any lingering lock lands on the unlinked
|
|
96
|
+
// old inode and is irrelevant to the new file.
|
|
97
|
+
it('replaces the live DB inode, not just its content', () => {
|
|
98
|
+
writeFileSync(livePath, 'old-db');
|
|
99
|
+
writeFileSync(join(stagingDir, 'celilo.db'), 'new-db');
|
|
100
|
+
const inodeBefore = statSync(livePath).ino;
|
|
101
|
+
|
|
102
|
+
applyStagedSystemFiles(stagingDir);
|
|
103
|
+
|
|
104
|
+
expect(readFileSync(livePath, 'utf-8')).toBe('new-db');
|
|
105
|
+
expect(statSync(livePath).ino).not.toBe(inodeBefore);
|
|
106
|
+
expect(existsSync(`${livePath}.restore-swap`)).toBe(false);
|
|
107
|
+
});
|
|
108
|
+
|
|
89
109
|
it('handles partial staging (only master.key, no DB)', () => {
|
|
90
110
|
writeFileSync(join(stagingDir, 'master.key'), 'just-the-key');
|
|
91
111
|
const result = applyStagedSystemFiles(stagingDir);
|
|
@@ -17,7 +17,6 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import { Database } from 'bun:sqlite';
|
|
20
|
-
import { execSync } from 'node:child_process';
|
|
21
20
|
import {
|
|
22
21
|
chmodSync,
|
|
23
22
|
closeSync,
|
|
@@ -29,20 +28,22 @@ import {
|
|
|
29
28
|
readFileSync,
|
|
30
29
|
readSync,
|
|
31
30
|
readdirSync,
|
|
31
|
+
renameSync,
|
|
32
32
|
rmSync,
|
|
33
33
|
} from 'node:fs';
|
|
34
34
|
import { tmpdir } from 'node:os';
|
|
35
35
|
import { join } from 'node:path';
|
|
36
36
|
import { eq } from 'drizzle-orm';
|
|
37
|
+
import { extract as tarExtract } from 'tar';
|
|
37
38
|
import { getDbPath, getMasterKeyPath, getModuleStoragePath } from '../config/paths';
|
|
38
39
|
import { closeDb, getDb } from '../db/client';
|
|
39
40
|
import { runMigrations } from '../db/migrate';
|
|
40
41
|
import { modules } from '../db/schema';
|
|
41
42
|
import { invokeHook } from '../hooks/executor';
|
|
43
|
+
import { createHookStores } from '../hooks/hook-store';
|
|
42
44
|
import { createConsoleLogger } from '../hooks/logger';
|
|
43
45
|
import type { ModuleManifest } from '../manifest/schema';
|
|
44
46
|
import { getOrCreateMasterKey } from '../secrets/master-key';
|
|
45
|
-
import { shellEscape } from '../utils/shell';
|
|
46
47
|
import { decryptFileToFile } from './backup-cipher';
|
|
47
48
|
import { assertCompatibleSchema, parseManifest } from './backup-manifest';
|
|
48
49
|
import { applyCrossModuleWriteRoot, moduleHasCrossModuleRead } from './cross-module-read';
|
|
@@ -128,7 +129,7 @@ export async function restoreFromArtifactFile(
|
|
|
128
129
|
}
|
|
129
130
|
|
|
130
131
|
// Extract the envelope tar.
|
|
131
|
-
|
|
132
|
+
await tarExtract({ file: tarPath, cwd: envelopeDir });
|
|
132
133
|
|
|
133
134
|
// 2. Read + validate the envelope manifest.
|
|
134
135
|
const manifestPath = join(envelopeDir, 'manifest.json');
|
|
@@ -221,6 +222,7 @@ export async function restoreFromArtifactFile(
|
|
|
221
222
|
debug: false,
|
|
222
223
|
systems: getModuleSystems(mod.id, db),
|
|
223
224
|
remoteAccess: remoteAccessPolicy(mod.id, db),
|
|
225
|
+
hookStores: () => createHookStores(db, mod.id),
|
|
224
226
|
},
|
|
225
227
|
);
|
|
226
228
|
if (!hookResult.success) {
|
|
@@ -428,9 +430,22 @@ export function applyStagedSystemFiles(systemStagingDir: string): StagedSystemAp
|
|
|
428
430
|
closeDb();
|
|
429
431
|
const livePath = getDbPath();
|
|
430
432
|
mkdirSync(join(livePath, '..'), { recursive: true });
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
//
|
|
433
|
+
// Swap by RENAME, not by in-place copy. copyFileSync overwrites the live
|
|
434
|
+
// inode in place, so any connection whose fd close was deferred (measured
|
|
435
|
+
// 2026-09-06, celilo#1293 live repro: bun:sqlite's close() returns before
|
|
436
|
+
// the fds are released) keeps its WAL shared locks on the NEW content, and
|
|
437
|
+
// the next opener's DELETE→WAL conversion in ensureWalMode then fails with
|
|
438
|
+
// "database is locked" for its whole retry budget. A rename gives the new
|
|
439
|
+
// file a fresh inode nobody has ever held. It also makes the swap atomic:
|
|
440
|
+
// a crash mid-copy left a truncated live DB; a crash mid-rename leaves the
|
|
441
|
+
// old one.
|
|
442
|
+
const stagedSwapPath = `${livePath}.restore-swap`;
|
|
443
|
+
copyFileSync(stagedDb, stagedSwapPath);
|
|
444
|
+
renameSync(stagedSwapPath, livePath);
|
|
445
|
+
|
|
446
|
+
// Stale WAL/SHM siblings belong to the replaced inode — remove them before
|
|
447
|
+
// any new connection opens this path, or SQLite would try to recover
|
|
448
|
+
// through a foreign WAL.
|
|
434
449
|
for (const suffix of ['-wal', '-shm']) {
|
|
435
450
|
try {
|
|
436
451
|
rmSync(`${livePath}${suffix}`, { force: true });
|
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
cpSync,
|
|
4
|
+
existsSync,
|
|
5
|
+
mkdirSync,
|
|
6
|
+
mkdtempSync,
|
|
7
|
+
readFileSync,
|
|
8
|
+
rmSync,
|
|
9
|
+
writeFileSync,
|
|
10
|
+
} from 'node:fs';
|
|
3
11
|
import { tmpdir } from 'node:os';
|
|
4
|
-
import { join } from 'node:path';
|
|
12
|
+
import { join, resolve } from 'node:path';
|
|
5
13
|
import { eq } from 'drizzle-orm';
|
|
6
14
|
import type { DbClient } from '../db/client';
|
|
7
15
|
import { moduleConfigs, modules, webRoutes } from '../db/schema';
|
|
8
16
|
import { setupTestDatabaseAt } from '../test-utils/database';
|
|
9
17
|
import {
|
|
18
|
+
ansibleTreeMentions,
|
|
10
19
|
buildStaticContentPlan,
|
|
11
20
|
planStaticContent,
|
|
12
21
|
resolveStaticContentRetention,
|
|
@@ -246,6 +255,82 @@ describe('staticContentVarsYaml — deterministic desired state', () => {
|
|
|
246
255
|
});
|
|
247
256
|
});
|
|
248
257
|
|
|
258
|
+
describe('the state web overlay (celilo#1265)', () => {
|
|
259
|
+
let dir: string;
|
|
260
|
+
let db: DbClient;
|
|
261
|
+
|
|
262
|
+
beforeEach(async () => {
|
|
263
|
+
dir = mkdtempSync(join(tmpdir(), 'static-converge-'));
|
|
264
|
+
const dbPath = join(dir, 'celilo.db');
|
|
265
|
+
process.env.CELILO_DB_PATH = dbPath;
|
|
266
|
+
db = await setupTestDatabaseAt(dbPath);
|
|
267
|
+
writeSite('lunacycle', '<html>luna</html>');
|
|
268
|
+
db.insert(modules)
|
|
269
|
+
.values({
|
|
270
|
+
id: 'lunacycle',
|
|
271
|
+
name: 'Lunacycle',
|
|
272
|
+
version: '1.0.0',
|
|
273
|
+
manifestData: {},
|
|
274
|
+
sourcePath: join(dir, 'modules', 'lunacycle'),
|
|
275
|
+
})
|
|
276
|
+
.onConflictDoNothing()
|
|
277
|
+
.run();
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
afterEach(() => {
|
|
281
|
+
db.$client.close();
|
|
282
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
283
|
+
try {
|
|
284
|
+
rmSync(dir, { recursive: true, force: true });
|
|
285
|
+
} catch {
|
|
286
|
+
/* ignore */
|
|
287
|
+
}
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
function writeSite(moduleId: string, html: string): string {
|
|
291
|
+
const webRoot = join(dir, 'modules', moduleId, 'site', 'dist');
|
|
292
|
+
mkdirSync(webRoot, { recursive: true });
|
|
293
|
+
writeFileSync(join(webRoot, 'index.html'), html, 'utf-8');
|
|
294
|
+
return webRoot;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function insertRoute(route: { slug: string; moduleId: string; hostname: string }) {
|
|
298
|
+
db.insert(webRoutes)
|
|
299
|
+
.values({
|
|
300
|
+
slug: route.slug,
|
|
301
|
+
moduleId: route.moduleId,
|
|
302
|
+
type: 'static',
|
|
303
|
+
path: '/',
|
|
304
|
+
hostname: route.hostname,
|
|
305
|
+
websocket: false,
|
|
306
|
+
contentHash: HASH1,
|
|
307
|
+
})
|
|
308
|
+
.run();
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
it('a release with a state overlay carries its directory; one without does not', () => {
|
|
312
|
+
insertRoute({ slug: 'lunacycle', moduleId: 'lunacycle', hostname: 'lunacycle.net' });
|
|
313
|
+
|
|
314
|
+
// No overlay in state: the release names no overlay_dir, and the converge
|
|
315
|
+
// role has nothing second to copy.
|
|
316
|
+
const bare = planStaticContent(db);
|
|
317
|
+
expect(bare).toHaveLength(1);
|
|
318
|
+
expect(bare[0]?.overlayDir).toBeUndefined();
|
|
319
|
+
expect(staticContentVarsYaml({ retention: 1, releases: bare })).not.toContain('overlay_dir');
|
|
320
|
+
|
|
321
|
+
// The hook wrote generated files into `<module>/state/site`: the converge
|
|
322
|
+
// must copy them over the release, or a rebuilt host loses the file.
|
|
323
|
+
const overlay = join(dir, 'modules', 'lunacycle', 'state', 'site');
|
|
324
|
+
mkdirSync(overlay, { recursive: true });
|
|
325
|
+
writeFileSync(join(overlay, 'ca.crt'), '---CERT---', 'utf-8');
|
|
326
|
+
|
|
327
|
+
const overlaid = planStaticContent(db);
|
|
328
|
+
expect(overlaid[0]?.overlayDir).toBe(overlay);
|
|
329
|
+
const yaml = staticContentVarsYaml({ retention: 1, releases: overlaid });
|
|
330
|
+
expect(yaml).toContain('overlay_dir:');
|
|
331
|
+
});
|
|
332
|
+
});
|
|
333
|
+
|
|
249
334
|
describe('writeStaticContentVars — lands in the generated inventory', () => {
|
|
250
335
|
let dir: string;
|
|
251
336
|
let db: DbClient;
|
|
@@ -336,3 +421,56 @@ describe('writeStaticContentVars — lands in the generated inventory', () => {
|
|
|
336
421
|
expect(plan.releases).toHaveLength(1);
|
|
337
422
|
});
|
|
338
423
|
});
|
|
424
|
+
|
|
425
|
+
describe('the stale-project guard reaches the whole generated tree (celilo#1248)', () => {
|
|
426
|
+
let dir: string;
|
|
427
|
+
|
|
428
|
+
beforeEach(() => {
|
|
429
|
+
dir = mkdtempSync(join(tmpdir(), 'static-guard-reach-'));
|
|
430
|
+
});
|
|
431
|
+
afterEach(() => {
|
|
432
|
+
rmSync(dir, { recursive: true, force: true });
|
|
433
|
+
});
|
|
434
|
+
|
|
435
|
+
it('sees a converge that lives in a role task file, not in playbook.yml', () => {
|
|
436
|
+
// Mirror DERIVED from the real module, never hand-written: a hand-built
|
|
437
|
+
// tree would give confident reach data about a world that does not exist,
|
|
438
|
+
// which is the same bug one level up.
|
|
439
|
+
const real = resolve(import.meta.dir, '../../../..', 'modules/caddy/ansible');
|
|
440
|
+
cpSync(real, join(dir, 'ansible'), { recursive: true });
|
|
441
|
+
|
|
442
|
+
// The premise this guards. caddy's playbook is a thin hosts/vars/roles
|
|
443
|
+
// stanza; if it ever DOES name static_releases, this test stops proving
|
|
444
|
+
// anything and should be re-derived rather than deleted.
|
|
445
|
+
const playbook = readFileSync(join(dir, 'ansible/playbook.yml.tpl'), 'utf-8');
|
|
446
|
+
expect(playbook).not.toContain('static_releases');
|
|
447
|
+
|
|
448
|
+
expect(ansibleTreeMentions(join(dir, 'ansible'), 'static_releases')).toBe(true);
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
it('still refuses a project that mentions the var nowhere', () => {
|
|
452
|
+
// A pre-converge generated project. Hand-built deliberately, and it has to
|
|
453
|
+
// be: no deploy on today's code can produce a tree without the role tasks,
|
|
454
|
+
// so this legacy shape is the fixture and not a shortcut.
|
|
455
|
+
mkdirSync(join(dir, 'ansible/roles/caddy/tasks'), { recursive: true });
|
|
456
|
+
writeFileSync(
|
|
457
|
+
join(dir, 'ansible/playbook.yml'),
|
|
458
|
+
'---\n- hosts: caddy\n roles:\n - caddy\n',
|
|
459
|
+
);
|
|
460
|
+
writeFileSync(
|
|
461
|
+
join(dir, 'ansible/roles/caddy/tasks/main.yml'),
|
|
462
|
+
'---\n- name: install\n debug: {}\n',
|
|
463
|
+
);
|
|
464
|
+
|
|
465
|
+
expect(ansibleTreeMentions(join(dir, 'ansible'), 'static_releases')).toBe(false);
|
|
466
|
+
});
|
|
467
|
+
|
|
468
|
+
it('ignores non-text files rather than reading assets as utf-8', () => {
|
|
469
|
+
mkdirSync(join(dir, 'ansible/files'), { recursive: true });
|
|
470
|
+
writeFileSync(join(dir, 'ansible/files/logo.png'), Buffer.from([0x89, 0x50, 0x4e, 0x47]));
|
|
471
|
+
writeFileSync(join(dir, 'ansible/site.yml'), 'vars:\n static_releases: []\n');
|
|
472
|
+
|
|
473
|
+
expect(ansibleTreeMentions(join(dir, 'ansible'), 'static_releases')).toBe(true);
|
|
474
|
+
expect(ansibleTreeMentions(join(dir, 'ansible'), 'PNG')).toBe(false);
|
|
475
|
+
});
|
|
476
|
+
});
|
|
@@ -23,14 +23,14 @@
|
|
|
23
23
|
* `module verify` (celilo#1018, deliberately left to that thread).
|
|
24
24
|
*/
|
|
25
25
|
|
|
26
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
26
|
+
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
27
27
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
28
28
|
import { join } from 'node:path';
|
|
29
29
|
import { and, eq, isNotNull } from 'drizzle-orm';
|
|
30
30
|
import { stringify as stringifyYaml } from 'yaml';
|
|
31
31
|
import type { DbClient } from '../db/client';
|
|
32
32
|
import { moduleConfigs, modules, webRoutes } from '../db/schema';
|
|
33
|
-
import { resolveModuleWebRoot } from '../module/web-root';
|
|
33
|
+
import { resolveModuleStateWebRoot, resolveModuleWebRoot } from '../module/web-root';
|
|
34
34
|
import { parseStoredConfigValue } from './module-config';
|
|
35
35
|
|
|
36
36
|
/** One content-hashed release the provider host must serve. */
|
|
@@ -43,6 +43,13 @@ export interface StaticRelease {
|
|
|
43
43
|
hostnames: string[];
|
|
44
44
|
/** Absolute celilo-mgr path core derived: `<module source_path>/site/dist`. */
|
|
45
45
|
sourceDir: string;
|
|
46
|
+
/**
|
|
47
|
+
* Absolute path to `<module source_path>/state/site` (celilo#1265), present
|
|
48
|
+
* ONLY when that directory exists. The role copies it over the release
|
|
49
|
+
* after the web root, so a hook-generated file wins over a built one. A
|
|
50
|
+
* module with no generated site content has no overlay and no row.
|
|
51
|
+
*/
|
|
52
|
+
overlayDir?: string;
|
|
46
53
|
}
|
|
47
54
|
|
|
48
55
|
export interface StaticContentPlan {
|
|
@@ -172,11 +179,17 @@ export function planStaticContent(db: DbClient): StaticRelease[] {
|
|
|
172
179
|
);
|
|
173
180
|
}
|
|
174
181
|
|
|
182
|
+
// Present only when the module's hook actually wrote an overlay —
|
|
183
|
+
// otherwise the vars file would name a directory that does not exist
|
|
184
|
+
// and the role's copy task would have nothing to copy from.
|
|
185
|
+
const stateWebRoot = resolveModuleStateWebRoot(moduleId, db);
|
|
186
|
+
const overlayDir = stateWebRoot && existsSync(stateWebRoot) ? stateWebRoot : undefined;
|
|
175
187
|
releases.push({
|
|
176
188
|
slug,
|
|
177
189
|
contentHash,
|
|
178
190
|
hostnames: [...new Set(group.map((r) => r.hostname))].sort(),
|
|
179
191
|
sourceDir,
|
|
192
|
+
...(overlayDir ? { overlayDir } : {}),
|
|
180
193
|
});
|
|
181
194
|
}
|
|
182
195
|
|
|
@@ -208,6 +221,9 @@ export function staticContentVarsYaml(plan: StaticContentPlan): string {
|
|
|
208
221
|
content_hash: r.contentHash,
|
|
209
222
|
hostnames: r.hostnames,
|
|
210
223
|
source_dir: r.sourceDir,
|
|
224
|
+
// Absent when there is no overlay, so the role's `when:` guard reads a
|
|
225
|
+
// clean variable — not an empty string a copy task would choke on.
|
|
226
|
+
...(r.overlayDir ? { overlay_dir: r.overlayDir } : {}),
|
|
211
227
|
})),
|
|
212
228
|
};
|
|
213
229
|
const header =
|
|
@@ -249,6 +265,40 @@ export async function writeStaticContentVars(
|
|
|
249
265
|
* never happen, so the playbook is checked for the converge's own variable
|
|
250
266
|
* first and a stale project fails loudly with the redeploy that fixes it.
|
|
251
267
|
*/
|
|
268
|
+
/**
|
|
269
|
+
* Whether any text file in a generated Ansible tree mentions `needle`.
|
|
270
|
+
*
|
|
271
|
+
* Scans the TREE rather than `playbook.yml`, because a provider's converge
|
|
272
|
+
* belongs in one of its role's task files and a generated `playbook.yml` is a
|
|
273
|
+
* thin hosts/vars/roles stanza that names no variable at all. caddy is the
|
|
274
|
+
* worked example: `static_releases` appears in
|
|
275
|
+
* `roles/caddy/tasks/static-content.yml`, `.../main.yml.tpl` and
|
|
276
|
+
* `.../converge-release.yml`, and zero times in its playbook.
|
|
277
|
+
*
|
|
278
|
+
* @psbanka - 2026-09: this used to read `ansible/playbook.yml` alone, which
|
|
279
|
+
* could not reach the string it was looking for. The guard below therefore
|
|
280
|
+
* refused EVERY provider that had the converge support, and its remedy told the
|
|
281
|
+
* operator to redeploy — which regenerates the same thin playbook and cannot
|
|
282
|
+
* help. Guard and role tasks landed in the same commit (1fd3595e), so it had
|
|
283
|
+
* never once passed, and it held three e2e suites red where nobody was looking.
|
|
284
|
+
* celilo#1248.
|
|
285
|
+
*/
|
|
286
|
+
export function ansibleTreeMentions(dir: string, needle: string): boolean {
|
|
287
|
+
if (!existsSync(dir)) return false;
|
|
288
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
289
|
+
const full = join(dir, entry.name);
|
|
290
|
+
if (entry.isDirectory()) {
|
|
291
|
+
if (ansibleTreeMentions(full, needle)) return true;
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
// Text only. A generated tree can carry static assets, and reading those
|
|
295
|
+
// as utf-8 to look for a variable name is waste at best.
|
|
296
|
+
if (!/\.(ya?ml|j2|tpl|cfg|ini|conf)$/.test(entry.name)) continue;
|
|
297
|
+
if (readFileSync(full, 'utf-8').includes(needle)) return true;
|
|
298
|
+
}
|
|
299
|
+
return false;
|
|
300
|
+
}
|
|
301
|
+
|
|
252
302
|
export async function convergeStaticContent(
|
|
253
303
|
db: DbClient,
|
|
254
304
|
providerModuleId: string,
|
|
@@ -271,12 +321,9 @@ export async function convergeStaticContent(
|
|
|
271
321
|
|
|
272
322
|
// A generated project that predates the converge role tasks would read the
|
|
273
323
|
// new vars file and converge nothing. `static_releases` appearing nowhere in
|
|
274
|
-
// the
|
|
275
|
-
const
|
|
276
|
-
if (
|
|
277
|
-
existsSync(playbookPath) &&
|
|
278
|
-
!readFileSync(playbookPath, 'utf-8').includes('static_releases')
|
|
279
|
-
) {
|
|
324
|
+
// the project is the tell. Generic string check — core names no module.
|
|
325
|
+
const ansiblePath = join(generatedPath, 'ansible');
|
|
326
|
+
if (existsSync(ansiblePath) && !ansibleTreeMentions(ansiblePath, 'static_releases')) {
|
|
280
327
|
return {
|
|
281
328
|
success: false,
|
|
282
329
|
error: `The generated project for '${providerModuleId}' predates the static-content converge (its playbook never reads static_releases). Redeploy the provider (\`celilo module deploy ${providerModuleId}\`) to regenerate it, then retry the publish.`,
|
|
@@ -119,6 +119,42 @@ describe('validateValue', () => {
|
|
|
119
119
|
expect(result.valid).toBe(false);
|
|
120
120
|
});
|
|
121
121
|
});
|
|
122
|
+
|
|
123
|
+
describe('cidr-list format validation', () => {
|
|
124
|
+
const property: SystemConfigProperty = {
|
|
125
|
+
type: 'string',
|
|
126
|
+
format: 'cidr-list',
|
|
127
|
+
description: 'Trusted source subnets',
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
it('accepts both stored forms: comma-separated CIDRs and a JSON array', () => {
|
|
131
|
+
expect(validateValue('firewall.trusted_subnets', '10.0.0.0/24', property).valid).toBe(true);
|
|
132
|
+
expect(
|
|
133
|
+
validateValue('firewall.trusted_subnets', '10.0.0.0/24, 10.1.0.0/24', property).valid,
|
|
134
|
+
).toBe(true);
|
|
135
|
+
expect(validateValue('firewall.trusted_subnets', '["10.226.120.0/24"]', property).valid).toBe(
|
|
136
|
+
true,
|
|
137
|
+
);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it('rejects a token that is not a CIDR, naming it, at write time', () => {
|
|
141
|
+
const result = validateValue(
|
|
142
|
+
'firewall.trusted_subnets',
|
|
143
|
+
'["10.226.120.0/24", oops]',
|
|
144
|
+
property,
|
|
145
|
+
);
|
|
146
|
+
expect(result.valid).toBe(false);
|
|
147
|
+
if (result.valid) return;
|
|
148
|
+
expect(result.error).toContain('looks like JSON but does not parse');
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it('rejects a JSON-looking value outright rather than mangling it into a subnet name', () => {
|
|
152
|
+
const result = validateValue('firewall.trusted_subnets', 'nonsense', property);
|
|
153
|
+
expect(result.valid).toBe(false);
|
|
154
|
+
if (result.valid) return;
|
|
155
|
+
expect(result.error).toContain('nonsense');
|
|
156
|
+
});
|
|
157
|
+
});
|
|
122
158
|
});
|
|
123
159
|
|
|
124
160
|
describe('validateKey', () => {
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import type { SystemConfigProperty, SystemConfigSchema } from './system-config-schema-types';
|
|
8
|
+
import { parseOperatorTrustedSubnets } from './trusted-sources';
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* Validation result (discriminated union)
|
|
@@ -55,6 +56,16 @@ export function validateValue(
|
|
|
55
56
|
}
|
|
56
57
|
}
|
|
57
58
|
|
|
59
|
+
// CIDR lists validate through the key's own parser, so the write-time check
|
|
60
|
+
// and the converge-time read can never disagree about what is valid.
|
|
61
|
+
if (property.format === 'cidr-list') {
|
|
62
|
+
try {
|
|
63
|
+
parseOperatorTrustedSubnets(value);
|
|
64
|
+
} catch (e) {
|
|
65
|
+
return { valid: false, error: e instanceof Error ? e.message : String(e) };
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
58
69
|
return { valid: true };
|
|
59
70
|
}
|
|
60
71
|
|
|
@@ -256,5 +256,35 @@ describe('parsing the operator override', () => {
|
|
|
256
256
|
it('an unset key contributes nothing', () => {
|
|
257
257
|
expect(parseOperatorTrustedSubnets(undefined)).toEqual([]);
|
|
258
258
|
expect(parseOperatorTrustedSubnets('')).toEqual([]);
|
|
259
|
+
expect(parseOperatorTrustedSubnets(' ')).toEqual([]);
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
it('parses the JSON-array form every sibling config key uses', () => {
|
|
263
|
+
expect(parseOperatorTrustedSubnets('["10.226.120.0/24"]')).toEqual(['10.226.120.0/24']);
|
|
264
|
+
expect(parseOperatorTrustedSubnets('[ "10.1.0.0/24" , "10.2.0.0/24" ]')).toEqual([
|
|
265
|
+
'10.1.0.0/24',
|
|
266
|
+
'10.2.0.0/24',
|
|
267
|
+
]);
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
it('refuses a JSON-looking value that does not parse to an array of strings', () => {
|
|
271
|
+
// The exact e2e failure (ce-yaqa): a JSON array read as ONE subnet named
|
|
272
|
+
// ["10.226.120.0/24"], rendered verbatim into rules.v4, and
|
|
273
|
+
// iptables-restore failed the converge with "Bad argument `]'".
|
|
274
|
+
expect(() => parseOperatorTrustedSubnets('{"a":1}')).toThrow(/JSON array of CIDR strings/);
|
|
275
|
+
expect(() => parseOperatorTrustedSubnets('["10.0.0.0/24", 5]')).toThrow(
|
|
276
|
+
/JSON array of CIDR strings/,
|
|
277
|
+
);
|
|
278
|
+
expect(() => parseOperatorTrustedSubnets('[10.0.0.0/24')).toThrow(
|
|
279
|
+
/looks like JSON but does not parse/,
|
|
280
|
+
);
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
it('refuses a token that is not an IPv4 CIDR, naming it', () => {
|
|
284
|
+
expect(() => parseOperatorTrustedSubnets('10.0.0.1')).toThrow(/10\.0\.0\.1/);
|
|
285
|
+
expect(() => parseOperatorTrustedSubnets('10.0.0.1')).toThrow(/not an IPv4 CIDR/);
|
|
286
|
+
expect(() => parseOperatorTrustedSubnets('10.0.0.0/24, hostname.local')).toThrow(
|
|
287
|
+
/hostname\.local/,
|
|
288
|
+
);
|
|
259
289
|
});
|
|
260
290
|
});
|