@celilo/cli 2.1.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/drizzle/0031_module_config_source.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/schemas/system_config.json +7 -1
- package/src/ansible/inventory.test.ts +2 -1
- package/src/api/sessions.test.ts +2 -1
- package/src/capabilities/public-web-publish.test.ts +61 -0
- package/src/cli/backup-rename.test.ts +2 -1
- package/src/cli/cli.test.ts +2 -1
- package/src/cli/commands/console-get-chain.test.ts +2 -1
- package/src/cli/commands/firewall-interface-list.test.ts +158 -8
- package/src/cli/commands/firewall-interface-list.ts +73 -7
- package/src/cli/commands/machine-add.ts +12 -55
- package/src/cli/commands/module-config.test.ts +22 -2
- package/src/cli/commands/module-deploy.ts +8 -2
- package/src/cli/commands/module-generate.test.ts +53 -0
- package/src/cli/commands/module-generate.ts +31 -26
- package/src/cli/commands/module-import-aspect.test.ts +2 -1
- package/src/cli/commands/module-import-registry.test.ts +2 -1
- package/src/cli/commands/module-import.ts +1 -1
- package/src/cli/commands/module-operations.test.ts +2 -1
- package/src/cli/commands/module-publish.test.ts +5 -12
- package/src/cli/commands/module-update.test.ts +87 -4
- package/src/cli/commands/module-update.ts +14 -4
- package/src/cli/commands/module-upgrade.test.ts +15 -0
- package/src/cli/commands/module-upgrade.ts +54 -2
- package/src/cli/commands/module-verify.test.ts +2 -3
- package/src/cli/commands/module-verify.ts +0 -1
- package/src/cli/commands/monitor.ts +2 -10
- package/src/cli/commands/notify-config.test.ts +5 -3
- package/src/cli/commands/registry-owner.test.ts +2 -1
- package/src/cli/commands/registry-token.test.ts +2 -1
- package/src/cli/commands/restore.ts +16 -6
- package/src/cli/commands/system-apply-config-equivalence.test.ts +5 -7
- package/src/cli/commands/system-config.test.ts +148 -0
- package/src/cli/commands/system-config.ts +26 -1
- package/src/cli/commands/system-doctor.test.ts +71 -0
- package/src/cli/commands/system-doctor.ts +110 -24
- package/src/cli/commands/system-init-deprecation.test.ts +6 -3
- package/src/cli/commands/system-migrate.test.ts +2 -1
- package/src/cli/generate-zsh-completion.ts +1 -1
- package/src/cli/index.ts +6 -4
- package/src/cli/restore-command.test.ts +2 -1
- package/src/cli/restore-migration-failure.test.ts +160 -0
- package/src/config/paths.test.ts +3 -3
- package/src/db/client.ts +5 -0
- package/src/db/migrate.test.ts +62 -135
- package/src/db/migrate.ts +17 -12
- package/src/db/schema.ts +10 -0
- package/src/hooks/broker.test.ts +106 -2
- package/src/hooks/broker.ts +91 -1
- package/src/hooks/capability-loader-firewall.test.ts +37 -0
- package/src/hooks/capability-loader.test.ts +78 -0
- package/src/hooks/capability-loader.ts +52 -3
- package/src/hooks/define-hook.test.ts +4 -3
- package/src/hooks/executor.test.ts +106 -19
- package/src/hooks/executor.ts +120 -13
- package/src/hooks/hook-jail-toolchain-reach.test.ts +3 -2
- package/src/hooks/hook-jail-unreachability.test.ts +4 -3
- package/src/hooks/hook-protocol.ts +46 -1
- package/src/hooks/hook-runner.ts +36 -0
- package/src/hooks/hook-store-proxy.test.ts +109 -0
- package/src/hooks/hook-store-proxy.ts +85 -0
- package/src/hooks/hook-store.test.ts +168 -0
- package/src/hooks/hook-store.ts +290 -0
- package/src/hooks/hook-timeout.test.ts +3 -2
- package/src/hooks/hook-trespass.test.ts +39 -5
- package/src/hooks/jail.test.ts +62 -3
- package/src/hooks/jail.ts +67 -8
- package/src/hooks/mount-set.test.ts +208 -0
- package/src/hooks/mount-set.ts +62 -14
- package/src/hooks/run-named-hook.ts +2 -0
- package/src/hooks/test-fixtures/jail-probe-hook.ts +1 -1
- package/src/hooks/test-fixtures/on-restore-staging-hook.ts +26 -0
- package/src/hooks/test-fixtures/store-backed.ts +47 -0
- package/src/hooks/test-fixtures/store-writing-hook.ts +63 -0
- package/src/hooks/unjailed-lint.test.ts +22 -6
- package/src/manifest/schema.ts +1 -0
- package/src/module/packaging/audit.ts +9 -26
- package/src/module/packaging/build-paths.test.ts +127 -0
- package/src/module/packaging/build-paths.ts +175 -0
- package/src/module/packaging/build.test.ts +71 -1
- package/src/module/packaging/build.ts +130 -2
- package/src/module/packaging/extract.ts +1 -5
- package/src/module/web-root.ts +17 -1
- package/src/policy/fixture-capability-coverage.test.ts +322 -0
- package/src/policy/module-script-scan.test.ts +42 -1
- package/src/policy/module-script-scan.ts +235 -5
- package/src/policy/no-hand-built-ssh.test.ts +34 -1
- package/src/policy/no-swallowed-refusal.test.ts +264 -0
- package/src/policy/no-tar-shell-out-in-services.test.ts +43 -0
- package/src/registry/client.test.ts +149 -0
- package/src/registry/client.ts +203 -11
- package/src/services/alerting/ack.test.ts +2 -1
- package/src/services/alerting/cadence-migration.test.ts +3 -2
- package/src/services/alerting/coverage-source.test.ts +2 -1
- package/src/services/alerting/deferral.test.ts +2 -1
- package/src/services/alerting/delivery-loop.test.ts +2 -1
- package/src/services/alerting/deploy-hooks.test.ts +2 -1
- package/src/services/alerting/format.test.ts +57 -0
- package/src/services/alerting/format.ts +24 -0
- package/src/services/alerting/inbound-poller.test.ts +2 -1
- package/src/services/alerting/inbound.test.ts +2 -1
- package/src/services/alerting/notification-responder.test.ts +2 -1
- package/src/services/alerting/run-monitor.test.ts +2 -1
- package/src/services/alerting/run-monitor.ts +2 -2
- package/src/services/alerting/store.test.ts +2 -1
- package/src/services/alerting/sweep-runner.test.ts +2 -1
- package/src/services/alerting/tokens.test.ts +2 -1
- package/src/services/aspect-approvals.test.ts +2 -1
- package/src/services/aspect-reconcile.test.ts +4 -3
- package/src/services/aspect-runner.test.ts +2 -1
- package/src/services/audit/module-integrity.test.ts +0 -21
- package/src/services/audit/module-integrity.ts +0 -14
- package/src/services/backup-age-agreement.test.ts +2 -1
- package/src/services/backup-create.ts +7 -7
- package/src/services/backup-envelope-roundtrip.test.ts +47 -3
- package/src/services/backup-in-flight-refusal.test.ts +2 -1
- package/src/services/backup-restore.ts +8 -4
- package/src/services/bus-ensure-flow.test.ts +2 -1
- package/src/services/bus-interview-park.test.ts +2 -1
- package/src/services/bus-interview.ts +37 -14
- package/src/services/bus-secret-flow.test.ts +2 -1
- package/src/services/capability-table-rows.test.ts +2 -1
- package/src/services/config-provenance.ts +4 -0
- package/src/services/consumer-cleanup.test.ts +3 -2
- package/src/services/container-service.test.ts +2 -1
- package/src/services/control-plane-bootstrap.test.ts +123 -2
- package/src/services/control-plane-bootstrap.ts +51 -4
- package/src/services/cross-module-read.test.ts +2 -1
- package/src/services/deploy-preflight.ts +8 -2
- package/src/services/deploy-validation.test.ts +25 -2
- package/src/services/deploy-validation.ts +8 -0
- package/src/services/dns-discovery.test.ts +54 -0
- package/src/services/dns-discovery.ts +47 -5
- package/src/services/dns-internal-records.test.ts +3 -2
- package/src/services/dns-provider-backfill.test.ts +2 -1
- package/src/services/dns-registrations.test.ts +2 -1
- package/src/services/ensure-interview.test.ts +3 -2
- package/src/services/fleet-checks.test.ts +3 -2
- package/src/services/fleet-key.test.ts +68 -3
- package/src/services/fleet-key.ts +54 -0
- package/src/services/health-runner.ts +2 -0
- package/src/services/infrastructure-selector.test.ts +2 -1
- package/src/services/infrastructure-variable-resolver.test.ts +2 -1
- package/src/services/machine-pool.test.ts +2 -1
- package/src/services/module-config.test.ts +2 -1
- package/src/services/module-config.ts +20 -2
- package/src/services/module-deploy.dns-repoint.test.ts +188 -0
- package/src/services/module-deploy.ts +199 -21
- package/src/services/module-operations.test.ts +2 -1
- package/src/services/module-subscriptions.test.ts +2 -1
- package/src/services/module-validator/git-hygiene.test.ts +122 -3
- package/src/services/module-validator/git-hygiene.ts +83 -14
- package/src/services/port-forwards.test.ts +2 -1
- package/src/services/programmatic-responder.aspect.test.ts +2 -1
- package/src/services/proxmox-reconcile.test.ts +2 -1
- package/src/services/restore-from-file.test.ts +23 -2
- package/src/services/restore-from-file.ts +21 -6
- package/src/services/restore-preflight.test.ts +2 -1
- package/src/services/secret-schema-loader.test.ts +2 -1
- package/src/services/ssh-key-manager.test.ts +2 -1
- package/src/services/static-content-converge.test.ts +144 -5
- package/src/services/static-content-converge.ts +87 -17
- package/src/services/system-config-schema-types.ts +1 -1
- package/src/services/system-config-validator.test.ts +36 -0
- package/src/services/system-config-validator.ts +11 -0
- package/src/services/system-state-stage.test.ts +2 -1
- package/src/services/trusted-sources.test.ts +33 -2
- package/src/services/trusted-sources.ts +47 -10
- package/src/services/zone-detector.test.ts +2 -1
- package/src/templates/generator.ts +9 -2
- package/src/test-utils/bus-responder.ts +5 -3
- package/src/test-utils/db-path.ts +25 -0
- package/src/test-utils/integration.ts +7 -0
- package/src/test-utils/module-fixtures.ts +5 -6
- package/src/variables/context.ts +16 -5
- package/src/module/packaging/generated-plane.test.ts +0 -79
- package/src/module/packaging/generated-plane.ts +0 -134
|
@@ -23,6 +23,7 @@ import { tmpdir } from 'node:os';
|
|
|
23
23
|
import { join } from 'node:path';
|
|
24
24
|
import { closeDb } from '../db/client';
|
|
25
25
|
import { runMigrations } from '../db/migrate';
|
|
26
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
26
27
|
import { applyStagedSystemFiles, restoreFromArtifactFile } from './restore-from-file';
|
|
27
28
|
|
|
28
29
|
describe('applyStagedSystemFiles', () => {
|
|
@@ -44,7 +45,7 @@ describe('applyStagedSystemFiles', () => {
|
|
|
44
45
|
|
|
45
46
|
afterEach(() => {
|
|
46
47
|
closeDb();
|
|
47
|
-
|
|
48
|
+
resetTestDbPath();
|
|
48
49
|
process.env.CELILO_MASTER_KEY_PATH = undefined;
|
|
49
50
|
process.env.CELILO_DATA_DIR = undefined;
|
|
50
51
|
try {
|
|
@@ -86,6 +87,26 @@ describe('applyStagedSystemFiles', () => {
|
|
|
86
87
|
expect(existsSync(`${livePath}-shm`)).toBe(false);
|
|
87
88
|
});
|
|
88
89
|
|
|
90
|
+
// celilo#1293 live repro 2026-09-06: the restore process's own pre-swap
|
|
91
|
+
// connection can outlive closeDb() at the fd level (measured: bun:sqlite's
|
|
92
|
+
// close() returned while the fds stayed open), and an IN-PLACE copy
|
|
93
|
+
// overwrites the inode that connection holds locks on — the swapped-in DB
|
|
94
|
+
// then inherits locks it must not have, and the next opener's DELETE→WAL
|
|
95
|
+
// conversion fails with "database is locked". The swap must therefore
|
|
96
|
+
// replace the INODE (rename), so any lingering lock lands on the unlinked
|
|
97
|
+
// old inode and is irrelevant to the new file.
|
|
98
|
+
it('replaces the live DB inode, not just its content', () => {
|
|
99
|
+
writeFileSync(livePath, 'old-db');
|
|
100
|
+
writeFileSync(join(stagingDir, 'celilo.db'), 'new-db');
|
|
101
|
+
const inodeBefore = statSync(livePath).ino;
|
|
102
|
+
|
|
103
|
+
applyStagedSystemFiles(stagingDir);
|
|
104
|
+
|
|
105
|
+
expect(readFileSync(livePath, 'utf-8')).toBe('new-db');
|
|
106
|
+
expect(statSync(livePath).ino).not.toBe(inodeBefore);
|
|
107
|
+
expect(existsSync(`${livePath}.restore-swap`)).toBe(false);
|
|
108
|
+
});
|
|
109
|
+
|
|
89
110
|
it('handles partial staging (only master.key, no DB)', () => {
|
|
90
111
|
writeFileSync(join(stagingDir, 'master.key'), 'just-the-key');
|
|
91
112
|
const result = applyStagedSystemFiles(stagingDir);
|
|
@@ -185,7 +206,7 @@ describe('restoreFromArtifactFile error paths', () => {
|
|
|
185
206
|
|
|
186
207
|
afterEach(() => {
|
|
187
208
|
closeDb();
|
|
188
|
-
|
|
209
|
+
resetTestDbPath();
|
|
189
210
|
process.env.CELILO_DATA_DIR = undefined;
|
|
190
211
|
process.env.CELILO_MASTER_KEY_PATH = undefined;
|
|
191
212
|
try {
|
|
@@ -17,7 +17,6 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import { Database } from 'bun:sqlite';
|
|
20
|
-
import { execSync } from 'node:child_process';
|
|
21
20
|
import {
|
|
22
21
|
chmodSync,
|
|
23
22
|
closeSync,
|
|
@@ -29,20 +28,22 @@ import {
|
|
|
29
28
|
readFileSync,
|
|
30
29
|
readSync,
|
|
31
30
|
readdirSync,
|
|
31
|
+
renameSync,
|
|
32
32
|
rmSync,
|
|
33
33
|
} from 'node:fs';
|
|
34
34
|
import { tmpdir } from 'node:os';
|
|
35
35
|
import { join } from 'node:path';
|
|
36
36
|
import { eq } from 'drizzle-orm';
|
|
37
|
+
import { extract as tarExtract } from 'tar';
|
|
37
38
|
import { getDbPath, getMasterKeyPath, getModuleStoragePath } from '../config/paths';
|
|
38
39
|
import { closeDb, getDb } from '../db/client';
|
|
39
40
|
import { runMigrations } from '../db/migrate';
|
|
40
41
|
import { modules } from '../db/schema';
|
|
41
42
|
import { invokeHook } from '../hooks/executor';
|
|
43
|
+
import { createHookStores } from '../hooks/hook-store';
|
|
42
44
|
import { createConsoleLogger } from '../hooks/logger';
|
|
43
45
|
import type { ModuleManifest } from '../manifest/schema';
|
|
44
46
|
import { getOrCreateMasterKey } from '../secrets/master-key';
|
|
45
|
-
import { shellEscape } from '../utils/shell';
|
|
46
47
|
import { decryptFileToFile } from './backup-cipher';
|
|
47
48
|
import { assertCompatibleSchema, parseManifest } from './backup-manifest';
|
|
48
49
|
import { applyCrossModuleWriteRoot, moduleHasCrossModuleRead } from './cross-module-read';
|
|
@@ -128,7 +129,7 @@ export async function restoreFromArtifactFile(
|
|
|
128
129
|
}
|
|
129
130
|
|
|
130
131
|
// Extract the envelope tar.
|
|
131
|
-
|
|
132
|
+
await tarExtract({ file: tarPath, cwd: envelopeDir });
|
|
132
133
|
|
|
133
134
|
// 2. Read + validate the envelope manifest.
|
|
134
135
|
const manifestPath = join(envelopeDir, 'manifest.json');
|
|
@@ -221,6 +222,7 @@ export async function restoreFromArtifactFile(
|
|
|
221
222
|
debug: false,
|
|
222
223
|
systems: getModuleSystems(mod.id, db),
|
|
223
224
|
remoteAccess: remoteAccessPolicy(mod.id, db),
|
|
225
|
+
hookStores: () => createHookStores(db, mod.id),
|
|
224
226
|
},
|
|
225
227
|
);
|
|
226
228
|
if (!hookResult.success) {
|
|
@@ -428,9 +430,22 @@ export function applyStagedSystemFiles(systemStagingDir: string): StagedSystemAp
|
|
|
428
430
|
closeDb();
|
|
429
431
|
const livePath = getDbPath();
|
|
430
432
|
mkdirSync(join(livePath, '..'), { recursive: true });
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
//
|
|
433
|
+
// Swap by RENAME, not by in-place copy. copyFileSync overwrites the live
|
|
434
|
+
// inode in place, so any connection whose fd close was deferred (measured
|
|
435
|
+
// 2026-09-06, celilo#1293 live repro: bun:sqlite's close() returns before
|
|
436
|
+
// the fds are released) keeps its WAL shared locks on the NEW content, and
|
|
437
|
+
// the next opener's DELETE→WAL conversion in ensureWalMode then fails with
|
|
438
|
+
// "database is locked" for its whole retry budget. A rename gives the new
|
|
439
|
+
// file a fresh inode nobody has ever held. It also makes the swap atomic:
|
|
440
|
+
// a crash mid-copy left a truncated live DB; a crash mid-rename leaves the
|
|
441
|
+
// old one.
|
|
442
|
+
const stagedSwapPath = `${livePath}.restore-swap`;
|
|
443
|
+
copyFileSync(stagedDb, stagedSwapPath);
|
|
444
|
+
renameSync(stagedSwapPath, livePath);
|
|
445
|
+
|
|
446
|
+
// Stale WAL/SHM siblings belong to the replaced inode — remove them before
|
|
447
|
+
// any new connection opens this path, or SQLite would try to recover
|
|
448
|
+
// through a foreign WAL.
|
|
434
449
|
for (const suffix of ['-wal', '-shm']) {
|
|
435
450
|
try {
|
|
436
451
|
rmSync(`${livePath}${suffix}`, { force: true });
|
|
@@ -5,6 +5,7 @@ import { join } from 'node:path';
|
|
|
5
5
|
import { closeDb, getDb } from '../db/client';
|
|
6
6
|
import { runMigrations } from '../db/migrate';
|
|
7
7
|
import { modules } from '../db/schema';
|
|
8
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
8
9
|
import {
|
|
9
10
|
NonEmptyRestoreTargetError,
|
|
10
11
|
assertRestoreTargetEmpty,
|
|
@@ -23,7 +24,7 @@ describe('restore pre-flight', () => {
|
|
|
23
24
|
|
|
24
25
|
afterEach(() => {
|
|
25
26
|
closeDb();
|
|
26
|
-
|
|
27
|
+
resetTestDbPath();
|
|
27
28
|
process.env.CELILO_DATA_DIR = undefined;
|
|
28
29
|
try {
|
|
29
30
|
rmSync(dir, { recursive: true, force: true });
|
|
@@ -4,6 +4,7 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { type DbClient, getDb } from '../db/client';
|
|
6
6
|
import { modules } from '../db/schema';
|
|
7
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
7
8
|
import {
|
|
8
9
|
type SecretsSchema,
|
|
9
10
|
getAllSecretMetadata,
|
|
@@ -40,7 +41,7 @@ describe('secret-schema-loader', () => {
|
|
|
40
41
|
afterEach(() => {
|
|
41
42
|
// Clean up temp directory
|
|
42
43
|
rmSync(tempDir, { recursive: true, force: true });
|
|
43
|
-
|
|
44
|
+
resetTestDbPath();
|
|
44
45
|
});
|
|
45
46
|
|
|
46
47
|
describe('loadSecretsSchema', () => {
|
|
@@ -8,6 +8,7 @@ import { tmpdir } from 'node:os';
|
|
|
8
8
|
import { join } from 'node:path';
|
|
9
9
|
import { closeDb } from '../db/client';
|
|
10
10
|
import { runMigrations } from '../db/migrate';
|
|
11
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
11
12
|
import { addMachine } from './machine-pool';
|
|
12
13
|
import {
|
|
13
14
|
LOCAL_MACHINE_IP,
|
|
@@ -97,7 +98,7 @@ describe('ssh-key-manager', () => {
|
|
|
97
98
|
}
|
|
98
99
|
|
|
99
100
|
// Clear environment variables
|
|
100
|
-
|
|
101
|
+
resetTestDbPath();
|
|
101
102
|
process.env.CELILO_DATA_DIR = undefined;
|
|
102
103
|
process.env.CELILO_MASTER_KEY_PATH = undefined;
|
|
103
104
|
});
|
|
@@ -1,12 +1,22 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
cpSync,
|
|
4
|
+
existsSync,
|
|
5
|
+
mkdirSync,
|
|
6
|
+
mkdtempSync,
|
|
7
|
+
readFileSync,
|
|
8
|
+
rmSync,
|
|
9
|
+
writeFileSync,
|
|
10
|
+
} from 'node:fs';
|
|
3
11
|
import { tmpdir } from 'node:os';
|
|
4
|
-
import { join } from 'node:path';
|
|
12
|
+
import { join, resolve } from 'node:path';
|
|
5
13
|
import { eq } from 'drizzle-orm';
|
|
6
14
|
import type { DbClient } from '../db/client';
|
|
7
15
|
import { moduleConfigs, modules, webRoutes } from '../db/schema';
|
|
8
16
|
import { setupTestDatabaseAt } from '../test-utils/database';
|
|
17
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
9
18
|
import {
|
|
19
|
+
ansibleTreeMentions,
|
|
10
20
|
buildStaticContentPlan,
|
|
11
21
|
planStaticContent,
|
|
12
22
|
resolveStaticContentRetention,
|
|
@@ -55,7 +65,7 @@ describe('planStaticContent — the declared release set (D10 task 4.1)', () =>
|
|
|
55
65
|
|
|
56
66
|
afterEach(() => {
|
|
57
67
|
db.$client.close();
|
|
58
|
-
|
|
68
|
+
resetTestDbPath();
|
|
59
69
|
try {
|
|
60
70
|
rmSync(dir, { recursive: true, force: true });
|
|
61
71
|
} catch {
|
|
@@ -168,7 +178,7 @@ describe('resolveStaticContentRetention — operator config with a default', ()
|
|
|
168
178
|
|
|
169
179
|
afterEach(() => {
|
|
170
180
|
db.$client.close();
|
|
171
|
-
|
|
181
|
+
resetTestDbPath();
|
|
172
182
|
try {
|
|
173
183
|
rmSync(dir, { recursive: true, force: true });
|
|
174
184
|
} catch {
|
|
@@ -246,6 +256,82 @@ describe('staticContentVarsYaml — deterministic desired state', () => {
|
|
|
246
256
|
});
|
|
247
257
|
});
|
|
248
258
|
|
|
259
|
+
describe('the state web overlay (celilo#1265)', () => {
|
|
260
|
+
let dir: string;
|
|
261
|
+
let db: DbClient;
|
|
262
|
+
|
|
263
|
+
beforeEach(async () => {
|
|
264
|
+
dir = mkdtempSync(join(tmpdir(), 'static-converge-'));
|
|
265
|
+
const dbPath = join(dir, 'celilo.db');
|
|
266
|
+
process.env.CELILO_DB_PATH = dbPath;
|
|
267
|
+
db = await setupTestDatabaseAt(dbPath);
|
|
268
|
+
writeSite('lunacycle', '<html>luna</html>');
|
|
269
|
+
db.insert(modules)
|
|
270
|
+
.values({
|
|
271
|
+
id: 'lunacycle',
|
|
272
|
+
name: 'Lunacycle',
|
|
273
|
+
version: '1.0.0',
|
|
274
|
+
manifestData: {},
|
|
275
|
+
sourcePath: join(dir, 'modules', 'lunacycle'),
|
|
276
|
+
})
|
|
277
|
+
.onConflictDoNothing()
|
|
278
|
+
.run();
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
afterEach(() => {
|
|
282
|
+
db.$client.close();
|
|
283
|
+
resetTestDbPath();
|
|
284
|
+
try {
|
|
285
|
+
rmSync(dir, { recursive: true, force: true });
|
|
286
|
+
} catch {
|
|
287
|
+
/* ignore */
|
|
288
|
+
}
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
function writeSite(moduleId: string, html: string): string {
|
|
292
|
+
const webRoot = join(dir, 'modules', moduleId, 'site', 'dist');
|
|
293
|
+
mkdirSync(webRoot, { recursive: true });
|
|
294
|
+
writeFileSync(join(webRoot, 'index.html'), html, 'utf-8');
|
|
295
|
+
return webRoot;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function insertRoute(route: { slug: string; moduleId: string; hostname: string }) {
|
|
299
|
+
db.insert(webRoutes)
|
|
300
|
+
.values({
|
|
301
|
+
slug: route.slug,
|
|
302
|
+
moduleId: route.moduleId,
|
|
303
|
+
type: 'static',
|
|
304
|
+
path: '/',
|
|
305
|
+
hostname: route.hostname,
|
|
306
|
+
websocket: false,
|
|
307
|
+
contentHash: HASH1,
|
|
308
|
+
})
|
|
309
|
+
.run();
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
it('a release with a state overlay carries its directory; one without does not', () => {
|
|
313
|
+
insertRoute({ slug: 'lunacycle', moduleId: 'lunacycle', hostname: 'lunacycle.net' });
|
|
314
|
+
|
|
315
|
+
// No overlay in state: the release names no overlay_dir, and the converge
|
|
316
|
+
// role has nothing second to copy.
|
|
317
|
+
const bare = planStaticContent(db);
|
|
318
|
+
expect(bare).toHaveLength(1);
|
|
319
|
+
expect(bare[0]?.overlayDir).toBeUndefined();
|
|
320
|
+
expect(staticContentVarsYaml({ retention: 1, releases: bare })).not.toContain('overlay_dir');
|
|
321
|
+
|
|
322
|
+
// The hook wrote generated files into `<module>/state/site`: the converge
|
|
323
|
+
// must copy them over the release, or a rebuilt host loses the file.
|
|
324
|
+
const overlay = join(dir, 'modules', 'lunacycle', 'state', 'site');
|
|
325
|
+
mkdirSync(overlay, { recursive: true });
|
|
326
|
+
writeFileSync(join(overlay, 'ca.crt'), '---CERT---', 'utf-8');
|
|
327
|
+
|
|
328
|
+
const overlaid = planStaticContent(db);
|
|
329
|
+
expect(overlaid[0]?.overlayDir).toBe(overlay);
|
|
330
|
+
const yaml = staticContentVarsYaml({ retention: 1, releases: overlaid });
|
|
331
|
+
expect(yaml).toContain('overlay_dir:');
|
|
332
|
+
});
|
|
333
|
+
});
|
|
334
|
+
|
|
249
335
|
describe('writeStaticContentVars — lands in the generated inventory', () => {
|
|
250
336
|
let dir: string;
|
|
251
337
|
let db: DbClient;
|
|
@@ -274,7 +360,7 @@ describe('writeStaticContentVars — lands in the generated inventory', () => {
|
|
|
274
360
|
|
|
275
361
|
afterEach(() => {
|
|
276
362
|
db.$client.close();
|
|
277
|
-
|
|
363
|
+
resetTestDbPath();
|
|
278
364
|
try {
|
|
279
365
|
rmSync(dir, { recursive: true, force: true });
|
|
280
366
|
} catch {
|
|
@@ -336,3 +422,56 @@ describe('writeStaticContentVars — lands in the generated inventory', () => {
|
|
|
336
422
|
expect(plan.releases).toHaveLength(1);
|
|
337
423
|
});
|
|
338
424
|
});
|
|
425
|
+
|
|
426
|
+
describe('the stale-project guard reaches the whole generated tree (celilo#1248)', () => {
|
|
427
|
+
let dir: string;
|
|
428
|
+
|
|
429
|
+
beforeEach(() => {
|
|
430
|
+
dir = mkdtempSync(join(tmpdir(), 'static-guard-reach-'));
|
|
431
|
+
});
|
|
432
|
+
afterEach(() => {
|
|
433
|
+
rmSync(dir, { recursive: true, force: true });
|
|
434
|
+
});
|
|
435
|
+
|
|
436
|
+
it('sees a converge that lives in a role task file, not in playbook.yml', () => {
|
|
437
|
+
// Mirror DERIVED from the real module, never hand-written: a hand-built
|
|
438
|
+
// tree would give confident reach data about a world that does not exist,
|
|
439
|
+
// which is the same bug one level up.
|
|
440
|
+
const real = resolve(import.meta.dir, '../../../..', 'modules/caddy/ansible');
|
|
441
|
+
cpSync(real, join(dir, 'ansible'), { recursive: true });
|
|
442
|
+
|
|
443
|
+
// The premise this guards. caddy's playbook is a thin hosts/vars/roles
|
|
444
|
+
// stanza; if it ever DOES name static_releases, this test stops proving
|
|
445
|
+
// anything and should be re-derived rather than deleted.
|
|
446
|
+
const playbook = readFileSync(join(dir, 'ansible/playbook.yml.tpl'), 'utf-8');
|
|
447
|
+
expect(playbook).not.toContain('static_releases');
|
|
448
|
+
|
|
449
|
+
expect(ansibleTreeMentions(join(dir, 'ansible'), 'static_releases')).toBe(true);
|
|
450
|
+
});
|
|
451
|
+
|
|
452
|
+
it('still refuses a project that mentions the var nowhere', () => {
|
|
453
|
+
// A pre-converge generated project. Hand-built deliberately, and it has to
|
|
454
|
+
// be: no deploy on today's code can produce a tree without the role tasks,
|
|
455
|
+
// so this legacy shape is the fixture and not a shortcut.
|
|
456
|
+
mkdirSync(join(dir, 'ansible/roles/caddy/tasks'), { recursive: true });
|
|
457
|
+
writeFileSync(
|
|
458
|
+
join(dir, 'ansible/playbook.yml'),
|
|
459
|
+
'---\n- hosts: caddy\n roles:\n - caddy\n',
|
|
460
|
+
);
|
|
461
|
+
writeFileSync(
|
|
462
|
+
join(dir, 'ansible/roles/caddy/tasks/main.yml'),
|
|
463
|
+
'---\n- name: install\n debug: {}\n',
|
|
464
|
+
);
|
|
465
|
+
|
|
466
|
+
expect(ansibleTreeMentions(join(dir, 'ansible'), 'static_releases')).toBe(false);
|
|
467
|
+
});
|
|
468
|
+
|
|
469
|
+
it('ignores non-text files rather than reading assets as utf-8', () => {
|
|
470
|
+
mkdirSync(join(dir, 'ansible/files'), { recursive: true });
|
|
471
|
+
writeFileSync(join(dir, 'ansible/files/logo.png'), Buffer.from([0x89, 0x50, 0x4e, 0x47]));
|
|
472
|
+
writeFileSync(join(dir, 'ansible/site.yml'), 'vars:\n static_releases: []\n');
|
|
473
|
+
|
|
474
|
+
expect(ansibleTreeMentions(join(dir, 'ansible'), 'static_releases')).toBe(true);
|
|
475
|
+
expect(ansibleTreeMentions(join(dir, 'ansible'), 'PNG')).toBe(false);
|
|
476
|
+
});
|
|
477
|
+
});
|
|
@@ -23,14 +23,14 @@
|
|
|
23
23
|
* `module verify` (celilo#1018, deliberately left to that thread).
|
|
24
24
|
*/
|
|
25
25
|
|
|
26
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
26
|
+
import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs';
|
|
27
27
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
28
28
|
import { join } from 'node:path';
|
|
29
29
|
import { and, eq, isNotNull } from 'drizzle-orm';
|
|
30
30
|
import { stringify as stringifyYaml } from 'yaml';
|
|
31
31
|
import type { DbClient } from '../db/client';
|
|
32
32
|
import { moduleConfigs, modules, webRoutes } from '../db/schema';
|
|
33
|
-
import { resolveModuleWebRoot } from '../module/web-root';
|
|
33
|
+
import { resolveModuleStateWebRoot, resolveModuleWebRoot } from '../module/web-root';
|
|
34
34
|
import { parseStoredConfigValue } from './module-config';
|
|
35
35
|
|
|
36
36
|
/** One content-hashed release the provider host must serve. */
|
|
@@ -43,6 +43,13 @@ export interface StaticRelease {
|
|
|
43
43
|
hostnames: string[];
|
|
44
44
|
/** Absolute celilo-mgr path core derived: `<module source_path>/site/dist`. */
|
|
45
45
|
sourceDir: string;
|
|
46
|
+
/**
|
|
47
|
+
* Absolute path to `<module source_path>/state/site` (celilo#1265), present
|
|
48
|
+
* ONLY when that directory exists. The role copies it over the release
|
|
49
|
+
* after the web root, so a hook-generated file wins over a built one. A
|
|
50
|
+
* module with no generated site content has no overlay and no row.
|
|
51
|
+
*/
|
|
52
|
+
overlayDir?: string;
|
|
46
53
|
}
|
|
47
54
|
|
|
48
55
|
export interface StaticContentPlan {
|
|
@@ -172,11 +179,17 @@ export function planStaticContent(db: DbClient): StaticRelease[] {
|
|
|
172
179
|
);
|
|
173
180
|
}
|
|
174
181
|
|
|
182
|
+
// Present only when the module's hook actually wrote an overlay —
|
|
183
|
+
// otherwise the vars file would name a directory that does not exist
|
|
184
|
+
// and the role's copy task would have nothing to copy from.
|
|
185
|
+
const stateWebRoot = resolveModuleStateWebRoot(moduleId, db);
|
|
186
|
+
const overlayDir = stateWebRoot && existsSync(stateWebRoot) ? stateWebRoot : undefined;
|
|
175
187
|
releases.push({
|
|
176
188
|
slug,
|
|
177
189
|
contentHash,
|
|
178
190
|
hostnames: [...new Set(group.map((r) => r.hostname))].sort(),
|
|
179
191
|
sourceDir,
|
|
192
|
+
...(overlayDir ? { overlayDir } : {}),
|
|
180
193
|
});
|
|
181
194
|
}
|
|
182
195
|
|
|
@@ -208,6 +221,9 @@ export function staticContentVarsYaml(plan: StaticContentPlan): string {
|
|
|
208
221
|
content_hash: r.contentHash,
|
|
209
222
|
hostnames: r.hostnames,
|
|
210
223
|
source_dir: r.sourceDir,
|
|
224
|
+
// Absent when there is no overlay, so the role's `when:` guard reads a
|
|
225
|
+
// clean variable — not an empty string a copy task would choke on.
|
|
226
|
+
...(r.overlayDir ? { overlay_dir: r.overlayDir } : {}),
|
|
211
227
|
})),
|
|
212
228
|
};
|
|
213
229
|
const header =
|
|
@@ -221,9 +237,10 @@ export function staticContentVarsYaml(plan: StaticContentPlan): string {
|
|
|
221
237
|
*
|
|
222
238
|
* The shared half of both converge triggers: the capability's callback adds
|
|
223
239
|
* `executeAnsible` after this, and the provider's own deploy already runs it.
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
*
|
|
240
|
+
* Nothing inspects verbatim role assets any more — the staleness pre-flight
|
|
241
|
+
* retired with the ephemeral generated tree (D5 of
|
|
242
|
+
* control-plane-stops-building-modules), so an injected vars file cannot read
|
|
243
|
+
* as a stale generation because nothing reads staleness.
|
|
227
244
|
*/
|
|
228
245
|
export async function writeStaticContentVars(
|
|
229
246
|
db: DbClient,
|
|
@@ -249,6 +266,40 @@ export async function writeStaticContentVars(
|
|
|
249
266
|
* never happen, so the playbook is checked for the converge's own variable
|
|
250
267
|
* first and a stale project fails loudly with the redeploy that fixes it.
|
|
251
268
|
*/
|
|
269
|
+
/**
|
|
270
|
+
* Whether any text file in a generated Ansible tree mentions `needle`.
|
|
271
|
+
*
|
|
272
|
+
* Scans the TREE rather than `playbook.yml`, because a provider's converge
|
|
273
|
+
* belongs in one of its role's task files and a generated `playbook.yml` is a
|
|
274
|
+
* thin hosts/vars/roles stanza that names no variable at all. caddy is the
|
|
275
|
+
* worked example: `static_releases` appears in
|
|
276
|
+
* `roles/caddy/tasks/static-content.yml`, `.../main.yml.tpl` and
|
|
277
|
+
* `.../converge-release.yml`, and zero times in its playbook.
|
|
278
|
+
*
|
|
279
|
+
* @psbanka - 2026-09: this used to read `ansible/playbook.yml` alone, which
|
|
280
|
+
* could not reach the string it was looking for. The guard below therefore
|
|
281
|
+
* refused EVERY provider that had the converge support, and its remedy told the
|
|
282
|
+
* operator to redeploy — which regenerates the same thin playbook and cannot
|
|
283
|
+
* help. Guard and role tasks landed in the same commit (1fd3595e), so it had
|
|
284
|
+
* never once passed, and it held three e2e suites red where nobody was looking.
|
|
285
|
+
* celilo#1248.
|
|
286
|
+
*/
|
|
287
|
+
export function ansibleTreeMentions(dir: string, needle: string): boolean {
|
|
288
|
+
if (!existsSync(dir)) return false;
|
|
289
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
290
|
+
const full = join(dir, entry.name);
|
|
291
|
+
if (entry.isDirectory()) {
|
|
292
|
+
if (ansibleTreeMentions(full, needle)) return true;
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
// Text only. A generated tree can carry static assets, and reading those
|
|
296
|
+
// as utf-8 to look for a variable name is waste at best.
|
|
297
|
+
if (!/\.(ya?ml|j2|tpl|cfg|ini|conf)$/.test(entry.name)) continue;
|
|
298
|
+
if (readFileSync(full, 'utf-8').includes(needle)) return true;
|
|
299
|
+
}
|
|
300
|
+
return false;
|
|
301
|
+
}
|
|
302
|
+
|
|
252
303
|
export async function convergeStaticContent(
|
|
253
304
|
db: DbClient,
|
|
254
305
|
providerModuleId: string,
|
|
@@ -258,25 +309,40 @@ export async function convergeStaticContent(
|
|
|
258
309
|
return { success: false, error: `Provider module '${providerModuleId}' is not installed` };
|
|
259
310
|
}
|
|
260
311
|
const generatedPath = join(module.sourcePath, 'generated');
|
|
312
|
+
// D4 (control-plane-stops-building-modules): the generated project is
|
|
313
|
+
// ephemeral, so a successful provider deploy deletes it and nothing is
|
|
314
|
+
// persisted for this converge to read. Render it here, on demand, from the
|
|
315
|
+
// signed payload plus resolved config — generation is fast and local. A
|
|
316
|
+
// render failure fails loudly with the deploy-once remedy, same as the old
|
|
317
|
+
// missing-project refusal did; a tree that predates the converge support is
|
|
318
|
+
// still refused by the static_releases check below.
|
|
261
319
|
if (!existsSync(generatedPath)) {
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
320
|
+
const { generateTemplates } = await import('../templates/generator');
|
|
321
|
+
const rendered = await generateTemplates({
|
|
322
|
+
moduleId: providerModuleId,
|
|
323
|
+
modulePath: module.sourcePath,
|
|
324
|
+
outputPath: generatedPath,
|
|
325
|
+
db,
|
|
326
|
+
skipVariableValidation: false,
|
|
327
|
+
});
|
|
328
|
+
if (!rendered.success) {
|
|
329
|
+
return {
|
|
330
|
+
success: false,
|
|
331
|
+
error:
|
|
332
|
+
`public_web provider '${providerModuleId}' has no generated deploy project at ${generatedPath} ` +
|
|
333
|
+
`and rendering one failed: ${rendered.error ?? 'unknown error'}. ` +
|
|
334
|
+
`Run \`celilo module deploy ${providerModuleId}\` once, then retry the publish.`,
|
|
335
|
+
};
|
|
336
|
+
}
|
|
268
337
|
}
|
|
269
338
|
|
|
270
339
|
const written = await writeStaticContentVars(db, providerModuleId, generatedPath);
|
|
271
340
|
|
|
272
341
|
// A generated project that predates the converge role tasks would read the
|
|
273
342
|
// new vars file and converge nothing. `static_releases` appearing nowhere in
|
|
274
|
-
// the
|
|
275
|
-
const
|
|
276
|
-
if (
|
|
277
|
-
existsSync(playbookPath) &&
|
|
278
|
-
!readFileSync(playbookPath, 'utf-8').includes('static_releases')
|
|
279
|
-
) {
|
|
343
|
+
// the project is the tell. Generic string check — core names no module.
|
|
344
|
+
const ansiblePath = join(generatedPath, 'ansible');
|
|
345
|
+
if (existsSync(ansiblePath) && !ansibleTreeMentions(ansiblePath, 'static_releases')) {
|
|
280
346
|
return {
|
|
281
347
|
success: false,
|
|
282
348
|
error: `The generated project for '${providerModuleId}' predates the static-content converge (its playbook never reads static_releases). Redeploy the provider (\`celilo module deploy ${providerModuleId}\`) to regenerate it, then retry the publish.`,
|
|
@@ -295,5 +361,9 @@ export async function convergeStaticContent(
|
|
|
295
361
|
varsPath: written.varsPath,
|
|
296
362
|
};
|
|
297
363
|
}
|
|
364
|
+
// D4: the converge rendered the tree, used it, and is done with it. The same
|
|
365
|
+
// no-secrets-at-rest rule the deploy follows applies here; on the failure
|
|
366
|
+
// paths above the tree stays for inspection.
|
|
367
|
+
rmSync(generatedPath, { recursive: true, force: true });
|
|
298
368
|
return { success: true, varsPath: written.varsPath };
|
|
299
369
|
}
|
|
@@ -119,6 +119,42 @@ describe('validateValue', () => {
|
|
|
119
119
|
expect(result.valid).toBe(false);
|
|
120
120
|
});
|
|
121
121
|
});
|
|
122
|
+
|
|
123
|
+
describe('cidr-list format validation', () => {
|
|
124
|
+
const property: SystemConfigProperty = {
|
|
125
|
+
type: 'string',
|
|
126
|
+
format: 'cidr-list',
|
|
127
|
+
description: 'Trusted source subnets',
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
it('accepts both stored forms: comma-separated CIDRs and a JSON array', () => {
|
|
131
|
+
expect(validateValue('firewall.trusted_subnets', '10.0.0.0/24', property).valid).toBe(true);
|
|
132
|
+
expect(
|
|
133
|
+
validateValue('firewall.trusted_subnets', '10.0.0.0/24, 10.1.0.0/24', property).valid,
|
|
134
|
+
).toBe(true);
|
|
135
|
+
expect(validateValue('firewall.trusted_subnets', '["10.226.120.0/24"]', property).valid).toBe(
|
|
136
|
+
true,
|
|
137
|
+
);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it('rejects a token that is not a CIDR, naming it, at write time', () => {
|
|
141
|
+
const result = validateValue(
|
|
142
|
+
'firewall.trusted_subnets',
|
|
143
|
+
'["10.226.120.0/24", oops]',
|
|
144
|
+
property,
|
|
145
|
+
);
|
|
146
|
+
expect(result.valid).toBe(false);
|
|
147
|
+
if (result.valid) return;
|
|
148
|
+
expect(result.error).toContain('looks like JSON but does not parse');
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it('rejects a JSON-looking value outright rather than mangling it into a subnet name', () => {
|
|
152
|
+
const result = validateValue('firewall.trusted_subnets', 'nonsense', property);
|
|
153
|
+
expect(result.valid).toBe(false);
|
|
154
|
+
if (result.valid) return;
|
|
155
|
+
expect(result.error).toContain('nonsense');
|
|
156
|
+
});
|
|
157
|
+
});
|
|
122
158
|
});
|
|
123
159
|
|
|
124
160
|
describe('validateKey', () => {
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import type { SystemConfigProperty, SystemConfigSchema } from './system-config-schema-types';
|
|
8
|
+
import { parseOperatorTrustedSubnets } from './trusted-sources';
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* Validation result (discriminated union)
|
|
@@ -55,6 +56,16 @@ export function validateValue(
|
|
|
55
56
|
}
|
|
56
57
|
}
|
|
57
58
|
|
|
59
|
+
// CIDR lists validate through the key's own parser, so the write-time check
|
|
60
|
+
// and the converge-time read can never disagree about what is valid.
|
|
61
|
+
if (property.format === 'cidr-list') {
|
|
62
|
+
try {
|
|
63
|
+
parseOperatorTrustedSubnets(value);
|
|
64
|
+
} catch (e) {
|
|
65
|
+
return { valid: false, error: e instanceof Error ? e.message : String(e) };
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
58
69
|
return { valid: true };
|
|
59
70
|
}
|
|
60
71
|
|
|
@@ -13,6 +13,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
|
13
13
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
14
14
|
import { tmpdir } from 'node:os';
|
|
15
15
|
import { join } from 'node:path';
|
|
16
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
16
17
|
import { snapshotDatabase, stageSystemState } from './system-state-stage';
|
|
17
18
|
|
|
18
19
|
describe('stageSystemState', () => {
|
|
@@ -38,7 +39,7 @@ describe('stageSystemState', () => {
|
|
|
38
39
|
|
|
39
40
|
afterEach(() => {
|
|
40
41
|
process.env.CELILO_DATA_DIR = undefined;
|
|
41
|
-
|
|
42
|
+
resetTestDbPath();
|
|
42
43
|
process.env.CELILO_MASTER_KEY_PATH = undefined;
|
|
43
44
|
rmSync(dataDir, { recursive: true, force: true });
|
|
44
45
|
});
|