@celilo/cli 2.1.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/drizzle/0031_module_config_source.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/schemas/system_config.json +7 -1
- package/src/ansible/inventory.test.ts +2 -1
- package/src/api/sessions.test.ts +2 -1
- package/src/capabilities/public-web-publish.test.ts +61 -0
- package/src/cli/backup-rename.test.ts +2 -1
- package/src/cli/cli.test.ts +2 -1
- package/src/cli/commands/console-get-chain.test.ts +2 -1
- package/src/cli/commands/firewall-interface-list.test.ts +158 -8
- package/src/cli/commands/firewall-interface-list.ts +73 -7
- package/src/cli/commands/machine-add.ts +12 -55
- package/src/cli/commands/module-config.test.ts +22 -2
- package/src/cli/commands/module-deploy.ts +8 -2
- package/src/cli/commands/module-generate.test.ts +53 -0
- package/src/cli/commands/module-generate.ts +31 -26
- package/src/cli/commands/module-import-aspect.test.ts +2 -1
- package/src/cli/commands/module-import-registry.test.ts +2 -1
- package/src/cli/commands/module-import.ts +1 -1
- package/src/cli/commands/module-operations.test.ts +2 -1
- package/src/cli/commands/module-publish.test.ts +5 -12
- package/src/cli/commands/module-update.test.ts +87 -4
- package/src/cli/commands/module-update.ts +14 -4
- package/src/cli/commands/module-upgrade.test.ts +15 -0
- package/src/cli/commands/module-upgrade.ts +54 -2
- package/src/cli/commands/module-verify.test.ts +2 -3
- package/src/cli/commands/module-verify.ts +0 -1
- package/src/cli/commands/monitor.ts +2 -10
- package/src/cli/commands/notify-config.test.ts +5 -3
- package/src/cli/commands/registry-owner.test.ts +2 -1
- package/src/cli/commands/registry-token.test.ts +2 -1
- package/src/cli/commands/restore.ts +16 -6
- package/src/cli/commands/system-apply-config-equivalence.test.ts +5 -7
- package/src/cli/commands/system-config.test.ts +148 -0
- package/src/cli/commands/system-config.ts +26 -1
- package/src/cli/commands/system-doctor.test.ts +71 -0
- package/src/cli/commands/system-doctor.ts +110 -24
- package/src/cli/commands/system-init-deprecation.test.ts +6 -3
- package/src/cli/commands/system-migrate.test.ts +2 -1
- package/src/cli/generate-zsh-completion.ts +1 -1
- package/src/cli/index.ts +6 -4
- package/src/cli/restore-command.test.ts +2 -1
- package/src/cli/restore-migration-failure.test.ts +160 -0
- package/src/config/paths.test.ts +3 -3
- package/src/db/client.ts +5 -0
- package/src/db/migrate.test.ts +62 -135
- package/src/db/migrate.ts +17 -12
- package/src/db/schema.ts +10 -0
- package/src/hooks/broker.test.ts +106 -2
- package/src/hooks/broker.ts +91 -1
- package/src/hooks/capability-loader-firewall.test.ts +37 -0
- package/src/hooks/capability-loader.test.ts +78 -0
- package/src/hooks/capability-loader.ts +52 -3
- package/src/hooks/define-hook.test.ts +4 -3
- package/src/hooks/executor.test.ts +106 -19
- package/src/hooks/executor.ts +120 -13
- package/src/hooks/hook-jail-toolchain-reach.test.ts +3 -2
- package/src/hooks/hook-jail-unreachability.test.ts +4 -3
- package/src/hooks/hook-protocol.ts +46 -1
- package/src/hooks/hook-runner.ts +36 -0
- package/src/hooks/hook-store-proxy.test.ts +109 -0
- package/src/hooks/hook-store-proxy.ts +85 -0
- package/src/hooks/hook-store.test.ts +168 -0
- package/src/hooks/hook-store.ts +290 -0
- package/src/hooks/hook-timeout.test.ts +3 -2
- package/src/hooks/hook-trespass.test.ts +39 -5
- package/src/hooks/jail.test.ts +62 -3
- package/src/hooks/jail.ts +67 -8
- package/src/hooks/mount-set.test.ts +208 -0
- package/src/hooks/mount-set.ts +62 -14
- package/src/hooks/run-named-hook.ts +2 -0
- package/src/hooks/test-fixtures/jail-probe-hook.ts +1 -1
- package/src/hooks/test-fixtures/on-restore-staging-hook.ts +26 -0
- package/src/hooks/test-fixtures/store-backed.ts +47 -0
- package/src/hooks/test-fixtures/store-writing-hook.ts +63 -0
- package/src/hooks/unjailed-lint.test.ts +22 -6
- package/src/manifest/schema.ts +1 -0
- package/src/module/packaging/audit.ts +9 -26
- package/src/module/packaging/build-paths.test.ts +127 -0
- package/src/module/packaging/build-paths.ts +175 -0
- package/src/module/packaging/build.test.ts +71 -1
- package/src/module/packaging/build.ts +130 -2
- package/src/module/packaging/extract.ts +1 -5
- package/src/module/web-root.ts +17 -1
- package/src/policy/fixture-capability-coverage.test.ts +322 -0
- package/src/policy/module-script-scan.test.ts +42 -1
- package/src/policy/module-script-scan.ts +235 -5
- package/src/policy/no-hand-built-ssh.test.ts +34 -1
- package/src/policy/no-swallowed-refusal.test.ts +264 -0
- package/src/policy/no-tar-shell-out-in-services.test.ts +43 -0
- package/src/registry/client.test.ts +149 -0
- package/src/registry/client.ts +203 -11
- package/src/services/alerting/ack.test.ts +2 -1
- package/src/services/alerting/cadence-migration.test.ts +3 -2
- package/src/services/alerting/coverage-source.test.ts +2 -1
- package/src/services/alerting/deferral.test.ts +2 -1
- package/src/services/alerting/delivery-loop.test.ts +2 -1
- package/src/services/alerting/deploy-hooks.test.ts +2 -1
- package/src/services/alerting/format.test.ts +57 -0
- package/src/services/alerting/format.ts +24 -0
- package/src/services/alerting/inbound-poller.test.ts +2 -1
- package/src/services/alerting/inbound.test.ts +2 -1
- package/src/services/alerting/notification-responder.test.ts +2 -1
- package/src/services/alerting/run-monitor.test.ts +2 -1
- package/src/services/alerting/run-monitor.ts +2 -2
- package/src/services/alerting/store.test.ts +2 -1
- package/src/services/alerting/sweep-runner.test.ts +2 -1
- package/src/services/alerting/tokens.test.ts +2 -1
- package/src/services/aspect-approvals.test.ts +2 -1
- package/src/services/aspect-reconcile.test.ts +4 -3
- package/src/services/aspect-runner.test.ts +2 -1
- package/src/services/audit/module-integrity.test.ts +0 -21
- package/src/services/audit/module-integrity.ts +0 -14
- package/src/services/backup-age-agreement.test.ts +2 -1
- package/src/services/backup-create.ts +7 -7
- package/src/services/backup-envelope-roundtrip.test.ts +47 -3
- package/src/services/backup-in-flight-refusal.test.ts +2 -1
- package/src/services/backup-restore.ts +8 -4
- package/src/services/bus-ensure-flow.test.ts +2 -1
- package/src/services/bus-interview-park.test.ts +2 -1
- package/src/services/bus-interview.ts +37 -14
- package/src/services/bus-secret-flow.test.ts +2 -1
- package/src/services/capability-table-rows.test.ts +2 -1
- package/src/services/config-provenance.ts +4 -0
- package/src/services/consumer-cleanup.test.ts +3 -2
- package/src/services/container-service.test.ts +2 -1
- package/src/services/control-plane-bootstrap.test.ts +123 -2
- package/src/services/control-plane-bootstrap.ts +51 -4
- package/src/services/cross-module-read.test.ts +2 -1
- package/src/services/deploy-preflight.ts +8 -2
- package/src/services/deploy-validation.test.ts +25 -2
- package/src/services/deploy-validation.ts +8 -0
- package/src/services/dns-discovery.test.ts +54 -0
- package/src/services/dns-discovery.ts +47 -5
- package/src/services/dns-internal-records.test.ts +3 -2
- package/src/services/dns-provider-backfill.test.ts +2 -1
- package/src/services/dns-registrations.test.ts +2 -1
- package/src/services/ensure-interview.test.ts +3 -2
- package/src/services/fleet-checks.test.ts +3 -2
- package/src/services/fleet-key.test.ts +68 -3
- package/src/services/fleet-key.ts +54 -0
- package/src/services/health-runner.ts +2 -0
- package/src/services/infrastructure-selector.test.ts +2 -1
- package/src/services/infrastructure-variable-resolver.test.ts +2 -1
- package/src/services/machine-pool.test.ts +2 -1
- package/src/services/module-config.test.ts +2 -1
- package/src/services/module-config.ts +20 -2
- package/src/services/module-deploy.dns-repoint.test.ts +188 -0
- package/src/services/module-deploy.ts +199 -21
- package/src/services/module-operations.test.ts +2 -1
- package/src/services/module-subscriptions.test.ts +2 -1
- package/src/services/module-validator/git-hygiene.test.ts +122 -3
- package/src/services/module-validator/git-hygiene.ts +83 -14
- package/src/services/port-forwards.test.ts +2 -1
- package/src/services/programmatic-responder.aspect.test.ts +2 -1
- package/src/services/proxmox-reconcile.test.ts +2 -1
- package/src/services/restore-from-file.test.ts +23 -2
- package/src/services/restore-from-file.ts +21 -6
- package/src/services/restore-preflight.test.ts +2 -1
- package/src/services/secret-schema-loader.test.ts +2 -1
- package/src/services/ssh-key-manager.test.ts +2 -1
- package/src/services/static-content-converge.test.ts +144 -5
- package/src/services/static-content-converge.ts +87 -17
- package/src/services/system-config-schema-types.ts +1 -1
- package/src/services/system-config-validator.test.ts +36 -0
- package/src/services/system-config-validator.ts +11 -0
- package/src/services/system-state-stage.test.ts +2 -1
- package/src/services/trusted-sources.test.ts +33 -2
- package/src/services/trusted-sources.ts +47 -10
- package/src/services/zone-detector.test.ts +2 -1
- package/src/templates/generator.ts +9 -2
- package/src/test-utils/bus-responder.ts +5 -3
- package/src/test-utils/db-path.ts +25 -0
- package/src/test-utils/integration.ts +7 -0
- package/src/test-utils/module-fixtures.ts +5 -6
- package/src/variables/context.ts +16 -5
- package/src/module/packaging/generated-plane.test.ts +0 -79
- package/src/module/packaging/generated-plane.ts +0 -134
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { execFileSync, execSync } from 'node:child_process';
|
|
2
|
-
import { cpSync, existsSync, mkdtempSync, rmSync } from 'node:fs';
|
|
2
|
+
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
3
3
|
import { readFile, readdir, writeFile } from 'node:fs/promises';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { basename, join, relative } from 'node:path';
|
|
6
|
+
import { gunzipSync } from 'node:zlib';
|
|
6
7
|
import { create as tarCreate } from 'tar';
|
|
7
8
|
import { parse as parseYaml } from 'yaml';
|
|
8
9
|
import { log } from '../../cli/prompts';
|
|
9
10
|
import { formatViolations, scanModuleDirectory } from '../../policy/module-script-scan';
|
|
10
11
|
import { validateModuleDirectory } from '../import';
|
|
12
|
+
import { resolveBuildCommandPaths } from './build-paths';
|
|
11
13
|
import { computeFileChecksum } from './checksum';
|
|
12
14
|
import { classifyModulePath, includeNodeModulesPath } from './package-rules';
|
|
13
15
|
import { signChecksums } from './signature';
|
|
@@ -22,6 +24,37 @@ export interface ChecksumsData {
|
|
|
22
24
|
files: Record<string, string>; // filepath -> xxhash
|
|
23
25
|
}
|
|
24
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Put `scripts/node_modules` back after a `bun pm pack` staging.
|
|
29
|
+
*
|
|
30
|
+
* `bun pm pack` drops every `node_modules` unconditionally — that is npm-pack
|
|
31
|
+
* semantics, and a `files` entry naming the path does not override it. But the
|
|
32
|
+
* packager is explicit downstream (`includeNodeModulesPath`) that
|
|
33
|
+
* `scripts/node_modules` is the module's HOOK RUNTIME and ships in full, so
|
|
34
|
+
* hooks resolve their third-party deps on a target with no reachable registry.
|
|
35
|
+
*
|
|
36
|
+
* Those two rules disagreed, and which one won was decided by whether a module
|
|
37
|
+
* happened to have a root `package.json`. Every module in `modules/` has only
|
|
38
|
+
* `scripts/package.json`, so all of them take the `cpSync` branch below and
|
|
39
|
+
* keep their runtime. A module that is ALSO a bun project — a build script, a
|
|
40
|
+
* data pipeline, its own tests — takes this branch instead and shipped without
|
|
41
|
+
* one. Nothing caught it: package, publish, import and deploy all succeed, and
|
|
42
|
+
* the first symptom is the hook dying on the target with
|
|
43
|
+
* `Cannot find package '@celilo/capabilities'` (celilo#1310).
|
|
44
|
+
*
|
|
45
|
+
* The hook runtime is not build input, so `files` has no business filtering it.
|
|
46
|
+
* Symlinks are materialised for the same reason the `cpSync` branch does it: a
|
|
47
|
+
* `file:` workspace link would point at a path the target does not have.
|
|
48
|
+
*/
|
|
49
|
+
function stageHookRuntime(sourceDir: string, buildDir: string): void {
|
|
50
|
+
const from = join(sourceDir, 'scripts', 'node_modules');
|
|
51
|
+
if (!existsSync(from)) return;
|
|
52
|
+
cpSync(from, join(buildDir, 'scripts', 'node_modules'), {
|
|
53
|
+
recursive: true,
|
|
54
|
+
dereference: true,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
|
|
25
58
|
/**
|
|
26
59
|
* Module build options
|
|
27
60
|
*/
|
|
@@ -120,6 +153,33 @@ export async function computeChecksums(sourceDir: string): Promise<ChecksumsData
|
|
|
120
153
|
};
|
|
121
154
|
}
|
|
122
155
|
|
|
156
|
+
/**
|
|
157
|
+
* Ceiling on the final tar+gzip write of the .netapp. Every step before it
|
|
158
|
+
* (staging, the module's own build command) already carries a timeout, but the
|
|
159
|
+
* tar stream did not — and 2026-09-04 it stalled mid-write (all samples in
|
|
160
|
+
* kevent64, artifact truncated at 59 percent) and sat there for two hours with
|
|
161
|
+
* no error and no exit. A bounded wait turns that into a nameable failure.
|
|
162
|
+
*/
|
|
163
|
+
const PACKAGE_STREAM_TIMEOUT_MS = 300_000;
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Validate that a produced .netapp is a complete gzip stream.
|
|
167
|
+
*
|
|
168
|
+
* A .netapp is a gzipped tar (tarCreate above runs with gzip: true), so a
|
|
169
|
+
* full gunzip pass is the whole check: a stream truncated mid-write fails
|
|
170
|
+
* with 'unexpected end of file' instead of surviving to the consumer, where
|
|
171
|
+
* the same zlib error reads as a truncated DOWNLOAD from an unrelated cause.
|
|
172
|
+
* Returns null when valid, otherwise the reason.
|
|
173
|
+
*/
|
|
174
|
+
export function verifyNetappIntegrity(packagePath: string): string | null {
|
|
175
|
+
try {
|
|
176
|
+
gunzipSync(readFileSync(packagePath));
|
|
177
|
+
return null;
|
|
178
|
+
} catch (err) {
|
|
179
|
+
return err instanceof Error ? err.message : String(err);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
123
183
|
/**
|
|
124
184
|
* Build a module package (.netapp file)
|
|
125
185
|
*
|
|
@@ -187,6 +247,7 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
|
|
|
187
247
|
timeout: 60_000,
|
|
188
248
|
});
|
|
189
249
|
rmSync(tarballs[0], { force: true });
|
|
250
|
+
stageHookRuntime(sourceDir, buildDir);
|
|
190
251
|
} catch (err) {
|
|
191
252
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
192
253
|
return {
|
|
@@ -230,6 +291,33 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
|
|
|
230
291
|
if (manifest.build?.command || manifest.build?.script) {
|
|
231
292
|
log.info(`Building module (${manifest.build.command ? 'command' : 'script'})...`);
|
|
232
293
|
|
|
294
|
+
// Publish-time gate on the build command's paths (D3 of
|
|
295
|
+
// control-plane-stops-building-modules). A `cd` whose target does not
|
|
296
|
+
// resolve where the command runs means the module depends on inputs it
|
|
297
|
+
// does not carry. celilo-registry did exactly this for four months
|
|
298
|
+
// (`cd ../../packages/registry-server`) and the failure surfaced on a
|
|
299
|
+
// production control plane, months after publish, as a /bin/sh error.
|
|
300
|
+
// Refuse here, at the only point where a human is watching, and name the
|
|
301
|
+
// path. Targets the gate cannot resolve (untracked shell variables,
|
|
302
|
+
// command substitution) are skipped, never guessed at.
|
|
303
|
+
if (manifest.build.command) {
|
|
304
|
+
const pathViolations = resolveBuildCommandPaths(manifest.build.command, {
|
|
305
|
+
moduleSourceDir: sourceDir,
|
|
306
|
+
buildDir,
|
|
307
|
+
});
|
|
308
|
+
if (pathViolations.length > 0) {
|
|
309
|
+
return {
|
|
310
|
+
success: false,
|
|
311
|
+
error: [
|
|
312
|
+
`Refusing to build ${moduleId}: the build command references a path that does not exist where the command runs.`,
|
|
313
|
+
...pathViolations.map((v) => ` ${v.rawPath}\n resolves to: ${v.resolvedPath}`),
|
|
314
|
+
'',
|
|
315
|
+
'A build input that does not resolve at publish time makes the module unbuildable wherever it is installed. Fix the path in manifest.yml (keep it module-local, or reach the monorepo through $CELILO_MODULE_SOURCE_DIR) and re-publish.',
|
|
316
|
+
].join('\n'),
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
233
321
|
// Resolve `workspace:` build deps (ISS-0147 / celilo#216). The staged
|
|
234
322
|
// buildDir has no workspace root, so a sibling monorepo package pulled in
|
|
235
323
|
// via `workspace:^` won't resolve during the build's `bun install`.
|
|
@@ -321,7 +409,7 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
|
|
|
321
409
|
|
|
322
410
|
// Create tarball
|
|
323
411
|
const finalOutputPath = outputPath || join(process.cwd(), `${moduleId}.netapp`);
|
|
324
|
-
|
|
412
|
+
const tarPromise = tarCreate(
|
|
325
413
|
{
|
|
326
414
|
file: finalOutputPath,
|
|
327
415
|
cwd: buildDir,
|
|
@@ -329,6 +417,46 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
|
|
|
329
417
|
},
|
|
330
418
|
['checksums.json', 'signature.sig', ...Object.keys(checksumsData.files)],
|
|
331
419
|
);
|
|
420
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
421
|
+
const timeout = new Promise<never>((_, reject) => {
|
|
422
|
+
timer = setTimeout(
|
|
423
|
+
() =>
|
|
424
|
+
reject(
|
|
425
|
+
new Error(
|
|
426
|
+
`packaging tar stream exceeded ${PACKAGE_STREAM_TIMEOUT_MS / 1000}s and was abandoned mid-write`,
|
|
427
|
+
),
|
|
428
|
+
),
|
|
429
|
+
PACKAGE_STREAM_TIMEOUT_MS,
|
|
430
|
+
);
|
|
431
|
+
});
|
|
432
|
+
timer?.unref?.();
|
|
433
|
+
try {
|
|
434
|
+
await Promise.race([tarPromise, timeout]);
|
|
435
|
+
} catch (tarError) {
|
|
436
|
+
// The losing side may still hold the output file open: swallow its
|
|
437
|
+
// eventual rejection and remove the partial artifact so nothing
|
|
438
|
+
// downstream mistakes a truncated write for a package.
|
|
439
|
+
tarPromise.catch(() => {});
|
|
440
|
+
rmSync(finalOutputPath, { force: true });
|
|
441
|
+
return {
|
|
442
|
+
success: false,
|
|
443
|
+
error: `Failed to build module: ${
|
|
444
|
+
tarError instanceof Error ? tarError.message : 'Unknown error'
|
|
445
|
+
}`,
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// Refuse to declare success on an artifact we cannot read back. The tar
|
|
450
|
+
// step resolved, so this only trips if the stream left the file truncated
|
|
451
|
+
// anyway — the exact corruption build-infra shipped on 2026-09-04.
|
|
452
|
+
const integrityError = verifyNetappIntegrity(finalOutputPath);
|
|
453
|
+
if (integrityError) {
|
|
454
|
+
rmSync(finalOutputPath, { force: true });
|
|
455
|
+
return {
|
|
456
|
+
success: false,
|
|
457
|
+
error: `Produced package failed gzip integrity check (${finalOutputPath}): ${integrityError}`,
|
|
458
|
+
};
|
|
459
|
+
}
|
|
332
460
|
|
|
333
461
|
return {
|
|
334
462
|
success: true,
|
|
@@ -19,12 +19,8 @@ export interface IntegrityViolation {
|
|
|
19
19
|
* being old rather than by the files having changed. Only `auditModule`
|
|
20
20
|
* produces it; package verification compares a package to its own manifest
|
|
21
21
|
* and cannot be stale in this sense.
|
|
22
|
-
*
|
|
23
|
-
* `stale-generated` is the second plane (D3): a verbatim role asset in
|
|
24
|
-
* `generated/` whose bytes are not the installed module's. That is what
|
|
25
|
-
* celilo#925 was, and what would ship on the next deploy.
|
|
26
22
|
*/
|
|
27
|
-
type: 'missing' | 'modified' | 'extra' | 'stale-baseline'
|
|
23
|
+
type: 'missing' | 'modified' | 'extra' | 'stale-baseline';
|
|
28
24
|
/**
|
|
29
25
|
* What the baseline says the file should hash to, and what it actually
|
|
30
26
|
* hashes to. Both optional because not every violation is about a digest —
|
package/src/module/web-root.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import { join } from 'node:path';
|
|
17
|
-
import { MODULE_WEB_ROOT } from '@celilo/capabilities';
|
|
17
|
+
import { MODULE_STATE_WEB_ROOT, MODULE_WEB_ROOT } from '@celilo/capabilities';
|
|
18
18
|
import { eq } from 'drizzle-orm';
|
|
19
19
|
import type { DbClient } from '../db/client';
|
|
20
20
|
import { modules } from '../db/schema';
|
|
@@ -33,3 +33,19 @@ export function resolveModuleWebRoot(moduleId: string, db: DbClient): string | u
|
|
|
33
33
|
if (!module) return undefined;
|
|
34
34
|
return join(module.sourcePath, MODULE_WEB_ROOT);
|
|
35
35
|
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Absolute path to `<module root>/state/site` — the state web overlay
|
|
39
|
+
* (celilo#1265) — or undefined when no such module is installed.
|
|
40
|
+
*
|
|
41
|
+
* Beside `resolveModuleWebRoot` for the same reason that function lives here:
|
|
42
|
+
* ONE application of the convention, by the one party that can resolve it for
|
|
43
|
+
* a module that is not running. Does NOT check that the directory exists — a
|
|
44
|
+
* module with no generated site content has no overlay, and deciding what an
|
|
45
|
+
* absent directory means belongs to the caller (see `requireWebRoot`).
|
|
46
|
+
*/
|
|
47
|
+
export function resolveModuleStateWebRoot(moduleId: string, db: DbClient): string | undefined {
|
|
48
|
+
const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
|
|
49
|
+
if (!module) return undefined;
|
|
50
|
+
return join(module.sourcePath, MODULE_STATE_WEB_ROOT);
|
|
51
|
+
}
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A module's e2e fixtures must deploy (at least import) a provider for every
|
|
3
|
+
* capability its manifest requires.
|
|
4
|
+
*
|
|
5
|
+
* The failure this exists to catch is silent in the worst way: the manifest
|
|
6
|
+
* requirement makes stage 1 of the module's own suite fail at
|
|
7
|
+
* `module import <id>` with "Required capability 'X' not found", and every
|
|
8
|
+
* later stage cascade-skips on requireStage. Nothing else goes red — the
|
|
9
|
+
* suite is quarantined, or CI never runs it, so a fixture can sit
|
|
10
|
+
* structurally incapable of passing for six weeks (celilo#1260:
|
|
11
|
+
* registry-pipeline required idp@1.2.0 from 7e2d27b9 onward and never
|
|
12
|
+
* deployed an idp provider).
|
|
13
|
+
*
|
|
14
|
+
* Both sides of the comparison are computed sets. The required set comes
|
|
15
|
+
* from each module's manifest.yml parsed by the production validator; the
|
|
16
|
+
* provided set comes from the manifests of whatever modules the module's
|
|
17
|
+
* fixtures actually import (`module import <ref>` statements, scanned in
|
|
18
|
+
* source). A hand-written list of "modules that need an idp" would rot the
|
|
19
|
+
* moment the next module declares the requirement.
|
|
20
|
+
*
|
|
21
|
+
* The check runs the real `validateCapabilityAccess` against an in-memory
|
|
22
|
+
* database seeded with the fixtures' providers — the same code path the CLI
|
|
23
|
+
* executes at import time, including the secret-allowlist gate — so the
|
|
24
|
+
* assertion is over the actual contract stage 1 exercises, not a re-derivation
|
|
25
|
+
* of it.
|
|
26
|
+
*
|
|
27
|
+
* ## What this gate does NOT prove. Read before trusting green.
|
|
28
|
+
*
|
|
29
|
+
* - Import is necessary, not sufficient. A fixture can import the provider
|
|
30
|
+
* and still deploy the consumer before it, or never deploy the provider
|
|
31
|
+
* at all (the consumer's on_install would then fail against a dead idp).
|
|
32
|
+
* Only running the suite proves deploy order.
|
|
33
|
+
* - The import scan is a source scan. An import built dynamically in a way
|
|
34
|
+
* this scanner cannot parse is invisible here (the scanner requires a
|
|
35
|
+
* literal module id, a `${CONST}` reference to a string literal in the
|
|
36
|
+
* same file, or a trailing path segment).
|
|
37
|
+
* - Fixtures living outside `modules/<id>/e2e/*.test.ts` (shared harness
|
|
38
|
+
* files that deploy on the suite's behalf) are not scanned.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
import { Database, type Database as DatabaseType } from 'bun:sqlite';
|
|
42
|
+
import { describe, expect, test } from 'bun:test';
|
|
43
|
+
import {
|
|
44
|
+
existsSync,
|
|
45
|
+
mkdirSync,
|
|
46
|
+
mkdtempSync,
|
|
47
|
+
readFileSync,
|
|
48
|
+
readdirSync,
|
|
49
|
+
rmSync,
|
|
50
|
+
writeFileSync,
|
|
51
|
+
} from 'node:fs';
|
|
52
|
+
import { tmpdir } from 'node:os';
|
|
53
|
+
import { join } from 'node:path';
|
|
54
|
+
import { validateCapabilityAccess } from '../capabilities/validation';
|
|
55
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
56
|
+
import { validateManifest } from '../manifest/validate';
|
|
57
|
+
import { repoRoot } from './capability-shape';
|
|
58
|
+
|
|
59
|
+
/** Every module manifest under `modulesDir`, parsed by the production validator. */
|
|
60
|
+
function loadModuleManifests(modulesDir: string): Map<string, ModuleManifest> {
|
|
61
|
+
const manifests = new Map<string, ModuleManifest>();
|
|
62
|
+
for (const id of readdirSync(modulesDir)) {
|
|
63
|
+
const path = join(modulesDir, id, 'manifest.yml');
|
|
64
|
+
if (!existsSync(path)) continue;
|
|
65
|
+
const result = validateManifest(readFileSync(path, 'utf8'));
|
|
66
|
+
if (result.success) manifests.set(id, result.data);
|
|
67
|
+
// Manifests that fail validation are another gate's job (validate.test.ts
|
|
68
|
+
// and scripts/check-modules.sh); scanning them here would double-report.
|
|
69
|
+
}
|
|
70
|
+
return manifests;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Module ids a fixture source imports, via `module import <ref>` statements.
|
|
75
|
+
*
|
|
76
|
+
* Handles the three shapes fixtures actually use: a bare literal
|
|
77
|
+
* (`'module import authentik'`), a path (`'module import /celilo/modules/authentik'`),
|
|
78
|
+
* and a `${CONST}` reference to a string literal defined in the same file
|
|
79
|
+
* (`module import ${PRIMARY}`). Trailing CLI flags (`--accept-aspects`) and
|
|
80
|
+
* timeouts are stripped. Anything that does not resolve to a plausible
|
|
81
|
+
* module id is skipped rather than guessed.
|
|
82
|
+
*/
|
|
83
|
+
function scanFixtureImports(fixtureSource: string): string[] {
|
|
84
|
+
const constants = new Map<string, string>();
|
|
85
|
+
for (const match of fixtureSource.matchAll(/const\s+(\w+)\s*=\s*'([^']+)'/g)) {
|
|
86
|
+
constants.set(match[1], match[2]);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const imports: string[] = [];
|
|
90
|
+
for (const match of fixtureSource.matchAll(/module import ([^'"\n]+?)['"`]/g)) {
|
|
91
|
+
let ref = match[1].trim();
|
|
92
|
+
const constRef = ref.match(/^\$\{(\w+)\}/);
|
|
93
|
+
if (constRef) ref = constants.get(constRef[1]) ?? '';
|
|
94
|
+
const id = ref.split(/\s+/)[0]?.split('/').pop() ?? '';
|
|
95
|
+
if (id && /^[a-z0-9-]+$/.test(id)) imports.push(id);
|
|
96
|
+
}
|
|
97
|
+
return imports;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** The union of modules any `*.test.ts` fixture in `moduleId`'s e2e/ imports. */
|
|
101
|
+
function fixtureImportedModules(modulesDir: string, moduleId: string): string[] {
|
|
102
|
+
const e2eDir = join(modulesDir, moduleId, 'e2e');
|
|
103
|
+
if (!existsSync(e2eDir)) return [];
|
|
104
|
+
const imported = new Set<string>();
|
|
105
|
+
for (const file of readdirSync(e2eDir)) {
|
|
106
|
+
if (!file.endsWith('.test.ts')) continue;
|
|
107
|
+
for (const id of scanFixtureImports(readFileSync(join(e2eDir, file), 'utf8'))) {
|
|
108
|
+
imported.add(id);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return [...imported];
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* An in-memory database shaped like the CLI's module registry, seeded with
|
|
116
|
+
* every capability `modules` provide — what `getProviderManifest` queries.
|
|
117
|
+
*/
|
|
118
|
+
function providerDatabase(manifests: Map<string, ModuleManifest>): DatabaseType {
|
|
119
|
+
const db = new Database(':memory:');
|
|
120
|
+
db.exec('CREATE TABLE modules (id TEXT PRIMARY KEY, manifest_data TEXT)');
|
|
121
|
+
db.exec('CREATE TABLE capabilities (module_id TEXT, capability_name TEXT)');
|
|
122
|
+
const insertModule = db.prepare('INSERT INTO modules (id, manifest_data) VALUES (?, ?)');
|
|
123
|
+
const insertCapability = db.prepare(
|
|
124
|
+
'INSERT INTO capabilities (module_id, capability_name) VALUES (?, ?)',
|
|
125
|
+
);
|
|
126
|
+
for (const [id, manifest] of manifests) {
|
|
127
|
+
insertModule.run(id, JSON.stringify(manifest));
|
|
128
|
+
for (const capability of manifest.provides?.capabilities ?? []) {
|
|
129
|
+
insertCapability.run(id, capability.name);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return db;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Modules with e2e fixtures whose required capabilities no imported module
|
|
137
|
+
* provides. The provider union is seeded into a real database and each
|
|
138
|
+
* consumer is checked with the production `validateCapabilityAccess`.
|
|
139
|
+
*/
|
|
140
|
+
async function capabilityCoverageViolations(modulesDir: string): Promise<string[]> {
|
|
141
|
+
const manifests = loadModuleManifests(modulesDir);
|
|
142
|
+
const violations: string[] = [];
|
|
143
|
+
for (const [id, manifest] of manifests) {
|
|
144
|
+
if (fixtureImportedModules(modulesDir, id).length === 0) continue;
|
|
145
|
+
const imported = fixtureImportedModules(modulesDir, id)
|
|
146
|
+
.map((dep) => manifests.get(dep))
|
|
147
|
+
.filter((m): m is ModuleManifest => m !== undefined);
|
|
148
|
+
const db = providerDatabase(new Map(imported.map((m) => [m.id, m])));
|
|
149
|
+
try {
|
|
150
|
+
const result = await validateCapabilityAccess(manifest, db);
|
|
151
|
+
if (!result.success) {
|
|
152
|
+
violations.push(`${id}: ${result.error ?? 'unknown error'}`);
|
|
153
|
+
}
|
|
154
|
+
} finally {
|
|
155
|
+
db.close();
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return violations;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
describe('fixture capability coverage', () => {
|
|
162
|
+
test('every module with e2e fixtures imports a provider for each required capability', async () => {
|
|
163
|
+
const violations = await capabilityCoverageViolations(join(repoRoot(), 'modules'));
|
|
164
|
+
expect(violations).toEqual([]);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
// Reach measurement (Rule 2 of the lane plan): the real scanner and checker
|
|
168
|
+
// run against a mirrored module tree with planted markers, so the pass and
|
|
169
|
+
// fail cases are both proven to fire — not reasoned about. This is the
|
|
170
|
+
// mutation test for the gate itself: the real-tree test above can only go
|
|
171
|
+
// green, so the mirror is where the gate is shown to fail.
|
|
172
|
+
describe('reach: the gate fires on a mirrored tree (synthetic modules)', () => {
|
|
173
|
+
function mirrorTree(
|
|
174
|
+
dir: string,
|
|
175
|
+
modules: Record<string, { manifest: string; fixtures?: string[] }>,
|
|
176
|
+
): string {
|
|
177
|
+
mkdirSync(dir, { recursive: true });
|
|
178
|
+
for (const [id, spec] of Object.entries(modules)) {
|
|
179
|
+
const moduleDir = join(dir, id);
|
|
180
|
+
mkdirSync(join(moduleDir, 'e2e'), { recursive: true });
|
|
181
|
+
writeFileSync(join(moduleDir, 'manifest.yml'), spec.manifest);
|
|
182
|
+
for (const [i, fixture] of (spec.fixtures ?? []).entries()) {
|
|
183
|
+
writeFileSync(join(moduleDir, 'e2e', `fixture-${i}.test.ts`), fixture);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return dir;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const providerManifest = [
|
|
190
|
+
'celilo_contract: "1.0"',
|
|
191
|
+
'id: marker-provider',
|
|
192
|
+
'name: Marker Provider',
|
|
193
|
+
'version: 1.0.0',
|
|
194
|
+
'description: synthetic provider',
|
|
195
|
+
'provides:',
|
|
196
|
+
' capabilities:',
|
|
197
|
+
' - name: marker_cap',
|
|
198
|
+
' version: 1.0.0',
|
|
199
|
+
'',
|
|
200
|
+
].join('\n');
|
|
201
|
+
|
|
202
|
+
const consumerFixture = "await net.celilo('module import marker-provider');\n";
|
|
203
|
+
|
|
204
|
+
test('a fixture importing the provider satisfies the requirement', async () => {
|
|
205
|
+
const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-ok-'));
|
|
206
|
+
try {
|
|
207
|
+
mirrorTree(dir, {
|
|
208
|
+
'marker-provider': { manifest: providerManifest },
|
|
209
|
+
'marker-consumer': {
|
|
210
|
+
manifest: [
|
|
211
|
+
'celilo_contract: "1.0"',
|
|
212
|
+
'id: marker-consumer',
|
|
213
|
+
'name: Marker Consumer',
|
|
214
|
+
'version: 1.0.0',
|
|
215
|
+
'description: synthetic consumer',
|
|
216
|
+
'requires:',
|
|
217
|
+
' capabilities:',
|
|
218
|
+
' - name: marker_cap',
|
|
219
|
+
' version: 1.0.0',
|
|
220
|
+
'',
|
|
221
|
+
].join('\n'),
|
|
222
|
+
fixtures: [consumerFixture],
|
|
223
|
+
},
|
|
224
|
+
});
|
|
225
|
+
expect(await capabilityCoverageViolations(dir)).toEqual([]);
|
|
226
|
+
} finally {
|
|
227
|
+
rmSync(dir, { recursive: true, force: true });
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
test('a fixture missing the provider fails, naming the module and capability', async () => {
|
|
232
|
+
const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-fail-'));
|
|
233
|
+
try {
|
|
234
|
+
mirrorTree(dir, {
|
|
235
|
+
'marker-provider': { manifest: providerManifest },
|
|
236
|
+
'marker-consumer': {
|
|
237
|
+
manifest: [
|
|
238
|
+
'celilo_contract: "1.0"',
|
|
239
|
+
'id: marker-consumer',
|
|
240
|
+
'name: Marker Consumer',
|
|
241
|
+
'version: 1.0.0',
|
|
242
|
+
'description: synthetic consumer',
|
|
243
|
+
'requires:',
|
|
244
|
+
' capabilities:',
|
|
245
|
+
' - name: marker_cap',
|
|
246
|
+
' version: 1.0.0',
|
|
247
|
+
' - name: marker_other',
|
|
248
|
+
' version: 1.0.0',
|
|
249
|
+
'',
|
|
250
|
+
].join('\n'),
|
|
251
|
+
// provider-b covers marker_cap; nothing covers marker_other
|
|
252
|
+
fixtures: [consumerFixture],
|
|
253
|
+
},
|
|
254
|
+
'marker-nonprovider': {
|
|
255
|
+
manifest: providerManifest.replace('marker_cap', 'marker_other'),
|
|
256
|
+
},
|
|
257
|
+
});
|
|
258
|
+
const violations = await capabilityCoverageViolations(dir);
|
|
259
|
+
expect(violations).toEqual([
|
|
260
|
+
expect.stringContaining("marker-consumer: Required capability 'marker_other' not found"),
|
|
261
|
+
]);
|
|
262
|
+
} finally {
|
|
263
|
+
rmSync(dir, { recursive: true, force: true });
|
|
264
|
+
}
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
test('a privilege the framework grants needs no provider', async () => {
|
|
268
|
+
const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-priv-'));
|
|
269
|
+
try {
|
|
270
|
+
mirrorTree(dir, {
|
|
271
|
+
'marker-consumer': {
|
|
272
|
+
manifest: [
|
|
273
|
+
'celilo_contract: "1.0"',
|
|
274
|
+
'id: marker-consumer',
|
|
275
|
+
'name: Marker Consumer',
|
|
276
|
+
'version: 1.0.0',
|
|
277
|
+
'description: synthetic consumer',
|
|
278
|
+
'requires:',
|
|
279
|
+
' capabilities:',
|
|
280
|
+
' - name: cross_module_read',
|
|
281
|
+
' version: 1.0.0',
|
|
282
|
+
'',
|
|
283
|
+
].join('\n'),
|
|
284
|
+
fixtures: ["await net.celilo('module import nothing-here');\n"],
|
|
285
|
+
},
|
|
286
|
+
});
|
|
287
|
+
expect(await capabilityCoverageViolations(dir)).toEqual([]);
|
|
288
|
+
} finally {
|
|
289
|
+
rmSync(dir, { recursive: true, force: true });
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
test('a ${CONST} import resolves through the same-file string literal', async () => {
|
|
294
|
+
const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-const-'));
|
|
295
|
+
try {
|
|
296
|
+
mirrorTree(dir, {
|
|
297
|
+
'marker-provider': { manifest: providerManifest },
|
|
298
|
+
'marker-consumer': {
|
|
299
|
+
manifest: [
|
|
300
|
+
'celilo_contract: "1.0"',
|
|
301
|
+
'id: marker-consumer',
|
|
302
|
+
'name: Marker Consumer',
|
|
303
|
+
'version: 1.0.0',
|
|
304
|
+
'description: synthetic consumer',
|
|
305
|
+
'requires:',
|
|
306
|
+
' capabilities:',
|
|
307
|
+
' - name: marker_cap',
|
|
308
|
+
' version: 1.0.0',
|
|
309
|
+
'',
|
|
310
|
+
].join('\n'),
|
|
311
|
+
fixtures: [
|
|
312
|
+
"const PROVIDER = 'marker-provider';\nawait net.celilo(`module import ${PROVIDER} --accept-aspects`);\n",
|
|
313
|
+
],
|
|
314
|
+
},
|
|
315
|
+
});
|
|
316
|
+
expect(await capabilityCoverageViolations(dir)).toEqual([]);
|
|
317
|
+
} finally {
|
|
318
|
+
rmSync(dir, { recursive: true, force: true });
|
|
319
|
+
}
|
|
320
|
+
});
|
|
321
|
+
});
|
|
322
|
+
});
|
|
@@ -12,7 +12,11 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
|
12
12
|
import { tmpdir } from 'node:os';
|
|
13
13
|
import { join } from 'node:path';
|
|
14
14
|
import { buildModule } from '../module/packaging/build';
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
JAILED_CLI_SPAWN_RULE,
|
|
17
|
+
scanModuleDirectory,
|
|
18
|
+
scanModuleScriptSource,
|
|
19
|
+
} from './module-script-scan';
|
|
16
20
|
|
|
17
21
|
const rules = (src: string) => scanModuleScriptSource('f.ts', src).map((v) => v.rule);
|
|
18
22
|
|
|
@@ -60,6 +64,43 @@ describe('module script scan — namespace tools (hygiene, not a boundary)', ()
|
|
|
60
64
|
});
|
|
61
65
|
});
|
|
62
66
|
|
|
67
|
+
describe('module script scan — jailed CLI spawn rule', () => {
|
|
68
|
+
// Each spawn spelling asserted separately. An alternation that loses a leg
|
|
69
|
+
// still passes a test that only exercises the first.
|
|
70
|
+
it.each([
|
|
71
|
+
[
|
|
72
|
+
"execFileSync('celilo', argv)",
|
|
73
|
+
"execFileSync('celilo', ['module', 'config', 'set', 'x', 'y']);",
|
|
74
|
+
],
|
|
75
|
+
['execFileSync("celilo", argv)', 'execFileSync("celilo", args);'],
|
|
76
|
+
['execSync(`celilo …`)', 'execSync(`celilo module secret set x y`);'],
|
|
77
|
+
[
|
|
78
|
+
'exec(`celilo …`) after a shell prefix',
|
|
79
|
+
'exec(`cd /opt && sudo celilo system config get k`);',
|
|
80
|
+
],
|
|
81
|
+
["spawn('celilo', argv)", "const child = spawn('celilo', ['events', 'status']);"],
|
|
82
|
+
])('catches %s', (_spelling, src) => {
|
|
83
|
+
expect(rules(src)).toContain(JAILED_CLI_SPAWN_RULE);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('does not fire on operator prose that merely names the CLI', () => {
|
|
87
|
+
expect(rules('throw new Error("re-run `celilo module deploy x` to regenerate it");')).toEqual(
|
|
88
|
+
[],
|
|
89
|
+
);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
// The `\w` after `celilo ` is what separates a CLI invocation from an
|
|
93
|
+
// argument that merely begins with the string, so the negative case has to
|
|
94
|
+
// be a hyphenated name, which would match a bare `\bcelilo\b`.
|
|
95
|
+
it('does not fire on a container argument named celilo-*', () => {
|
|
96
|
+
expect(rules('execSync(`docker exec celilo-mgr status`);')).toEqual([]);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it('does not fire on the celilo.db file name', () => {
|
|
100
|
+
expect(rules('execSync(`cp /var/lib/celilo/celilo.db /tmp/x`);')).toEqual([]);
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
63
104
|
describe('module script scan — raw-exec escape hatch', () => {
|
|
64
105
|
it('flags a runAppCommand call with no justification', () => {
|
|
65
106
|
expect(rules('const r = runAppCommand(system, "rm -f /tmp/x", run);')).toContain(
|