@celilo/cli 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_SUBSYSTEMS.md +16 -2
- package/MODULE_PRIMITIVES.md +19 -6
- package/drizzle/0026_module_integrity_version.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +2 -2
- package/src/cli/commands/module-audit.ts +5 -2
- package/src/cli/commands/module-update.test.ts +90 -2
- package/src/cli/commands/module-update.ts +112 -6
- package/src/cli/commands/module-verify.ts +77 -13
- package/src/cli/commands/system-audit.ts +17 -0
- package/src/cli/commands/system-doctor.ts +78 -2
- package/src/cli/commands/system-update.ts +33 -3
- package/src/cli/index.ts +2 -2
- package/src/cli/tui/audit-state.ts +11 -3
- package/src/cli/tui/audit-tui.tsx +10 -4
- package/src/cli/tui/icons.ts +9 -2
- package/src/cli/tui/modals/analyzing.tsx +3 -0
- package/src/db/schema.ts +5 -0
- package/src/manifest/json-schema-roundtrip.test.ts +12 -4
- package/src/manifest/schema.ts +23 -0
- package/src/module/import.ts +36 -35
- package/src/module/packaging/audit.ts +103 -28
- package/src/module/packaging/build.ts +12 -53
- package/src/module/packaging/classify-module-path.test.ts +104 -0
- package/src/module/packaging/extract.ts +31 -3
- package/src/module/packaging/generated-plane.test.ts +79 -0
- package/src/module/packaging/generated-plane.ts +134 -0
- package/src/module/packaging/host-plane.test.ts +132 -0
- package/src/module/packaging/host-plane.ts +135 -0
- package/src/module/packaging/package-rules.ts +62 -0
- package/src/services/audit/cli-version.test.ts +6 -2
- package/src/services/audit/cli-version.ts +20 -6
- package/src/services/audit/detect-without-converge.test.ts +91 -0
- package/src/services/audit/detect-without-converge.ts +81 -0
- package/src/services/audit/disk-space.test.ts +5 -2
- package/src/services/audit/disk-space.ts +5 -3
- package/src/services/audit/health.test.ts +39 -0
- package/src/services/audit/index.test.ts +7 -1
- package/src/services/audit/index.ts +12 -0
- package/src/services/audit/module-integrity.test.ts +146 -0
- package/src/services/audit/module-integrity.ts +113 -0
- package/src/services/audit/module-versions.ts +4 -1
- package/src/services/audit/schema.test.ts +7 -2
- package/src/services/audit/schema.ts +19 -1
- package/src/services/audit/terraform-plan.ts +17 -2
- package/src/services/audit/types.test.ts +29 -0
- package/src/services/audit/types.ts +30 -4
- package/src/services/module-deploy.ts +21 -0
- package/src/services/restore-from-file.ts +4 -0
- package/src/services/update/orchestrator.test.ts +2 -0
|
@@ -4,8 +4,12 @@ import { join, relative } from 'node:path';
|
|
|
4
4
|
import { eq } from 'drizzle-orm';
|
|
5
5
|
import { getDb } from '../../db/client';
|
|
6
6
|
import { moduleIntegrity, modules } from '../../db/schema';
|
|
7
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
7
8
|
import { computeFileChecksum } from './checksum';
|
|
8
9
|
import type { IntegrityViolation } from './extract';
|
|
10
|
+
import { compareVerbatimRoleAssets, readVerbatimRoleAssets } from './generated-plane';
|
|
11
|
+
import { type HostPlaneResult, verifyModuleOnHosts } from './host-plane';
|
|
12
|
+
import { classifyModulePath } from './package-rules';
|
|
9
13
|
|
|
10
14
|
/**
|
|
11
15
|
* Audit result for a module
|
|
@@ -15,24 +19,27 @@ export interface AuditResult {
|
|
|
15
19
|
moduleId: string;
|
|
16
20
|
violations: IntegrityViolation[];
|
|
17
21
|
error?: string;
|
|
22
|
+
/** What the module records, and what version the baseline describes. */
|
|
23
|
+
moduleVersion?: string;
|
|
24
|
+
baselineVersion?: string | null;
|
|
25
|
+
/** Present only under `deep`. See `host-plane.ts`. */
|
|
26
|
+
hostPlane?: HostPlaneResult;
|
|
18
27
|
}
|
|
19
28
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
* Phase 2 belt-and-suspenders). A fresh copy appears on disk after
|
|
28
|
-
* extracting the package, but it isn't in `checksums.json` because the
|
|
29
|
-
* package may have been built before the type was generated.
|
|
30
|
-
*/
|
|
31
|
-
const FRAMEWORK_OWNED_PATHS = new Set(['celilo/types.d.ts']);
|
|
29
|
+
export interface AuditOptions {
|
|
30
|
+
/**
|
|
31
|
+
* Also ask each of the module's systems whether what is running is what
|
|
32
|
+
* celilo generated. One SSH per system, so it is off by default.
|
|
33
|
+
*/
|
|
34
|
+
deep?: boolean;
|
|
35
|
+
}
|
|
32
36
|
|
|
33
37
|
/**
|
|
34
|
-
* Recursively scan
|
|
35
|
-
*
|
|
38
|
+
* Recursively scan the installed tree, keeping only paths whose content is a
|
|
39
|
+
* stable integrity claim (`package`) or whose presence is a finding in itself
|
|
40
|
+
* (`unknown`). `derived` paths — `generated/**`, the hook runtime closure,
|
|
41
|
+
* `checksums.json` — are celilo's own and are dropped here, because a check
|
|
42
|
+
* that reports them can only ever be wrong.
|
|
36
43
|
*/
|
|
37
44
|
async function scanDirectory(dir: string, baseDir: string): Promise<string[]> {
|
|
38
45
|
const files: string[] = [];
|
|
@@ -42,21 +49,14 @@ async function scanDirectory(dir: string, baseDir: string): Promise<string[]> {
|
|
|
42
49
|
const fullPath = join(dir, entry.name);
|
|
43
50
|
const relativePath = relative(baseDir, fullPath);
|
|
44
51
|
|
|
45
|
-
if (FRAMEWORK_OWNED_PATHS.has(relativePath)) {
|
|
46
|
-
continue;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// Skip node_modules anywhere in the tree — these are installed by
|
|
50
|
-
// the framework during `module import` (NPM_PACKAGE_RESOLUTION
|
|
51
|
-
// Option B), not part of the module's signed checksums.
|
|
52
|
-
if (entry.isDirectory() && entry.name === 'node_modules') {
|
|
53
|
-
continue;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
52
|
if (entry.isDirectory()) {
|
|
53
|
+
// Prune whole derived subtrees rather than walking them. `generated/`
|
|
54
|
+
// alone carries terraform provider binaries.
|
|
55
|
+
if (classifyModulePath(relativePath) === 'derived') continue;
|
|
57
56
|
const subFiles = await scanDirectory(fullPath, baseDir);
|
|
58
57
|
files.push(...subFiles);
|
|
59
58
|
} else if (entry.isFile()) {
|
|
59
|
+
if (classifyModulePath(relativePath) === 'derived') continue;
|
|
60
60
|
files.push(relativePath);
|
|
61
61
|
}
|
|
62
62
|
}
|
|
@@ -71,7 +71,11 @@ async function scanDirectory(dir: string, baseDir: string): Promise<string[]> {
|
|
|
71
71
|
* @param db - Database client (optional, for testing)
|
|
72
72
|
* @returns Audit result with any violations found
|
|
73
73
|
*/
|
|
74
|
-
export async function auditModule(
|
|
74
|
+
export async function auditModule(
|
|
75
|
+
moduleId: string,
|
|
76
|
+
db = getDb(),
|
|
77
|
+
options: AuditOptions = {},
|
|
78
|
+
): Promise<AuditResult> {
|
|
75
79
|
const violations: IntegrityViolation[] = [];
|
|
76
80
|
|
|
77
81
|
try {
|
|
@@ -106,6 +110,26 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
106
110
|
const expectedChecksums: Record<string, string> = integrity.checksums;
|
|
107
111
|
const moduleDir = module.sourcePath;
|
|
108
112
|
|
|
113
|
+
// Which version do these checksums describe? Ahead of every file finding,
|
|
114
|
+
// because when the answer is "not the installed one" the file findings are
|
|
115
|
+
// a consequence of it and not independent evidence. Before D1 this was
|
|
116
|
+
// unanswerable: the row was written once at first import and `module
|
|
117
|
+
// update` never touched it, so verify reported the same violations whether
|
|
118
|
+
// the files were old or the checksums were old.
|
|
119
|
+
if (integrity.version === null) {
|
|
120
|
+
violations.push({
|
|
121
|
+
type: 'stale-baseline',
|
|
122
|
+
path: 'checksums.json',
|
|
123
|
+
message: `Baseline records no version — it was written before celilo stamped them, so it cannot be compared to the installed ${module.version}. Re-run 'celilo module update' for this module to refresh it.`,
|
|
124
|
+
});
|
|
125
|
+
} else if (integrity.version !== module.version) {
|
|
126
|
+
violations.push({
|
|
127
|
+
type: 'stale-baseline',
|
|
128
|
+
path: 'checksums.json',
|
|
129
|
+
message: `Baseline describes ${integrity.version}, module records ${module.version}. The checksums are old, not the files. Re-run 'celilo module update' for this module to refresh it.`,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
|
|
109
133
|
// Check if module directory exists
|
|
110
134
|
if (!existsSync(moduleDir)) {
|
|
111
135
|
return {
|
|
@@ -116,8 +140,12 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
116
140
|
};
|
|
117
141
|
}
|
|
118
142
|
|
|
119
|
-
// Validate all expected files exist and have correct checksums
|
|
143
|
+
// Validate all expected files exist and have correct checksums. A baseline
|
|
144
|
+
// entry for a derived path is not checkable: celilo rewrites those bytes
|
|
145
|
+
// after install (`bun install` over the hook runtime closure), so comparing
|
|
146
|
+
// them to what the package shipped can only ever produce a false positive.
|
|
120
147
|
for (const [filePath, expectedChecksum] of Object.entries(expectedChecksums)) {
|
|
148
|
+
if (classifyModulePath(filePath) === 'derived') continue;
|
|
121
149
|
const fullPath = join(moduleDir, filePath);
|
|
122
150
|
|
|
123
151
|
if (!existsSync(fullPath)) {
|
|
@@ -125,6 +153,8 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
125
153
|
type: 'missing',
|
|
126
154
|
path: filePath,
|
|
127
155
|
message: `Missing file: ${filePath}`,
|
|
156
|
+
expectedDigest: expectedChecksum,
|
|
157
|
+
actualDigest: null,
|
|
128
158
|
});
|
|
129
159
|
continue;
|
|
130
160
|
}
|
|
@@ -135,6 +165,33 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
135
165
|
type: 'modified',
|
|
136
166
|
path: filePath,
|
|
137
167
|
message: `Checksum mismatch: ${filePath}`,
|
|
168
|
+
expectedDigest: expectedChecksum,
|
|
169
|
+
actualDigest: actualChecksum,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Plane two: is what we would deploy built from what we installed? Only
|
|
175
|
+
// verbatim role assets have a meaningful expected digest — Ansible
|
|
176
|
+
// templates the rest, and a `.j2` in `generated/` is SUPPOSED to differ.
|
|
177
|
+
// A module that has never been generated has nothing to compare and is not
|
|
178
|
+
// a finding; it is simply pre-deploy.
|
|
179
|
+
const generatedDir = join(moduleDir, 'generated');
|
|
180
|
+
if (existsSync(generatedDir)) {
|
|
181
|
+
const differences = compareVerbatimRoleAssets(
|
|
182
|
+
await readVerbatimRoleAssets(moduleDir),
|
|
183
|
+
await readVerbatimRoleAssets(generatedDir),
|
|
184
|
+
);
|
|
185
|
+
for (const difference of differences) {
|
|
186
|
+
violations.push({
|
|
187
|
+
type: 'stale-generated',
|
|
188
|
+
path: difference.relPath,
|
|
189
|
+
expectedDigest: difference.installedDigest,
|
|
190
|
+
actualDigest: difference.generatedDigest,
|
|
191
|
+
message:
|
|
192
|
+
difference.reason === 'missing'
|
|
193
|
+
? `Generated project is missing ${difference.relPath} — the next deploy would ship nothing for it. Run 'celilo module generate ${moduleId}'.`
|
|
194
|
+
: `Generated project holds different bytes for ${difference.relPath} (generated ${difference.generatedDigest}, installed ${difference.installedDigest}) — the next deploy would ship the wrong ones. Run 'celilo module generate ${moduleId}'.`,
|
|
138
195
|
});
|
|
139
196
|
}
|
|
140
197
|
}
|
|
@@ -153,10 +210,28 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
153
210
|
}
|
|
154
211
|
}
|
|
155
212
|
|
|
213
|
+
// Plane three: is what is running what we generated? One SSH per system,
|
|
214
|
+
// so it is asked only when the caller says so.
|
|
215
|
+
let hostPlane: HostPlaneResult | undefined;
|
|
216
|
+
if (options.deep) {
|
|
217
|
+
hostPlane = await verifyModuleOnHosts({
|
|
218
|
+
moduleId,
|
|
219
|
+
manifest: module.manifestData as unknown as ModuleManifest,
|
|
220
|
+
generatedPath: generatedDir,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// An `unmeasured` host is not a pass. A check that could not reach its
|
|
225
|
+
// subject says so, and does not count as green.
|
|
226
|
+
const hostPlaneClean = (hostPlane?.findings ?? []).every((f) => f.state === 'converged');
|
|
227
|
+
|
|
156
228
|
return {
|
|
157
|
-
success: violations.length === 0,
|
|
229
|
+
success: violations.length === 0 && hostPlaneClean,
|
|
158
230
|
moduleId,
|
|
159
231
|
violations,
|
|
232
|
+
moduleVersion: module.version,
|
|
233
|
+
baselineVersion: integrity.version,
|
|
234
|
+
hostPlane,
|
|
160
235
|
};
|
|
161
236
|
} catch (error) {
|
|
162
237
|
return {
|
|
@@ -9,7 +9,7 @@ import { log } from '../../cli/prompts';
|
|
|
9
9
|
import { formatViolations, scanModuleDirectory } from '../../policy/module-script-scan';
|
|
10
10
|
import { validateModuleDirectory } from '../import';
|
|
11
11
|
import { computeFileChecksum } from './checksum';
|
|
12
|
-
import { includeNodeModulesPath } from './package-rules';
|
|
12
|
+
import { classifyModulePath, includeNodeModulesPath } from './package-rules';
|
|
13
13
|
import { signChecksums } from './signature';
|
|
14
14
|
import { rewriteWorkspaceDeps } from './workspace-deps';
|
|
15
15
|
|
|
@@ -49,63 +49,22 @@ export interface ModuleBuildResult {
|
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
/**
|
|
52
|
-
*
|
|
52
|
+
* Check if a path inside the source dir should be excluded from the package.
|
|
53
53
|
*
|
|
54
|
-
* `
|
|
55
|
-
*
|
|
56
|
-
* closure
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
'.DS_Store',
|
|
61
|
-
'*.netapp',
|
|
62
|
-
'*.test.ts',
|
|
63
|
-
// Dev-only, same bucket as the tests: scripts/tsconfig.json exists so tsc can
|
|
64
|
-
// check hooks in CI. Nothing on a target ever runs tsc, and shipping it would
|
|
65
|
-
// make every module's packaged content change whenever the shared base moves.
|
|
66
|
-
'tsconfig.json',
|
|
67
|
-
'checksums.json',
|
|
68
|
-
'signature.sig',
|
|
69
|
-
];
|
|
70
|
-
|
|
71
|
-
/**
|
|
72
|
-
* Path-relative files the framework manages and that should never be
|
|
73
|
-
* bundled into a `.netapp` package. `celilo/types.d.ts` is generated
|
|
74
|
-
* post-import by `module import` (HOOK_API_V2 Phase 2 belt-and-suspenders),
|
|
75
|
-
* so shipping the local copy would just stamp the package with whatever
|
|
76
|
-
* was on the author's disk at build time — including potentially stale
|
|
77
|
-
* versions if the manifest changed since.
|
|
78
|
-
*/
|
|
79
|
-
const FRAMEWORK_OWNED_PATHS = new Set(['celilo/types.d.ts']);
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Check if a path inside the source dir should be excluded from the
|
|
83
|
-
* package.
|
|
84
|
-
*
|
|
85
|
-
* The `node_modules` decision is delegated to the canonical
|
|
86
|
-
* `includeNodeModulesPath` rule (package-rules.ts) — the single source of truth
|
|
87
|
-
* the registry-server's bootstrap packager is held to as well (ISS-0046).
|
|
54
|
+
* `classifyModulePath` (package-rules.ts) is the one answer to what belongs to
|
|
55
|
+
* a module. Packaging differs from it in exactly one place: the hook runtime
|
|
56
|
+
* closure under `scripts/node_modules/` is `derived` (celilo's `bun install`
|
|
57
|
+
* owns the on-disk copy) and still SHIPS, because a target may have no
|
|
58
|
+
* reachable registry (ISS-0046). Everything else `derived` is celilo's own
|
|
59
|
+
* output, or the checksum manifest that cannot list itself.
|
|
88
60
|
*/
|
|
89
61
|
function shouldExclude(filePath: string): boolean {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
// Module's e2e/ directory at the source root is tests + their deps —
|
|
95
|
-
// not part of the deployed module.
|
|
96
|
-
if (segments[0] === 'e2e') return true;
|
|
97
|
-
|
|
98
|
-
if (segments.includes('node_modules')) {
|
|
62
|
+
const cls = classifyModulePath(filePath);
|
|
63
|
+
if (cls === 'package') return false;
|
|
64
|
+
if (cls === 'derived' && filePath.split('/').includes('node_modules')) {
|
|
99
65
|
return !includeNodeModulesPath(filePath);
|
|
100
66
|
}
|
|
101
|
-
|
|
102
|
-
const name = basename(filePath);
|
|
103
|
-
return EXCLUDE_PATTERNS.some((pattern) => {
|
|
104
|
-
if (pattern.startsWith('*')) {
|
|
105
|
-
return name.endsWith(pattern.slice(1));
|
|
106
|
-
}
|
|
107
|
-
return name === pattern;
|
|
108
|
-
});
|
|
67
|
+
return true;
|
|
109
68
|
}
|
|
110
69
|
|
|
111
70
|
/**
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { classifyModulePath } from './package-rules';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Real input, not input built by the same helper as the expectation.
|
|
8
|
+
*
|
|
9
|
+
* These fixtures are verbatim `celilo module verify` output captured from the
|
|
10
|
+
* live fleet on 2026-08-19 (see the fixture README). Every path below is a path
|
|
11
|
+
* that actually exists in an installed module tree on celilo-mgr. celilo#951
|
|
12
|
+
* shipped 18 false positives because its comparator's test synthesised both
|
|
13
|
+
* sides of the comparison and so never saw real input.
|
|
14
|
+
*/
|
|
15
|
+
function fixturePaths(name: string): string[] {
|
|
16
|
+
const raw = readFileSync(
|
|
17
|
+
join(process.cwd(), 'test-fixtures', 'module-integrity', `${name}-verify.txt`),
|
|
18
|
+
'utf-8',
|
|
19
|
+
);
|
|
20
|
+
const paths: string[] = [];
|
|
21
|
+
for (const line of raw.split('\n')) {
|
|
22
|
+
const match = /(?:Checksum mismatch|Unexpected file): (.+)$/.exec(line.trim());
|
|
23
|
+
if (match?.[1]) paths.push(match[1]);
|
|
24
|
+
}
|
|
25
|
+
return paths;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
describe('classifyModulePath against the live fleet listings', () => {
|
|
29
|
+
for (const [moduleName, expectedCount] of [
|
|
30
|
+
['wireguard-manager', 47],
|
|
31
|
+
['wireguard', 25],
|
|
32
|
+
] as const) {
|
|
33
|
+
describe(moduleName, () => {
|
|
34
|
+
const paths = fixturePaths(moduleName);
|
|
35
|
+
|
|
36
|
+
test(`fixture carries all ${expectedCount} reported paths`, () => {
|
|
37
|
+
expect(paths.length).toBe(expectedCount);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test('no path in a real installed tree classifies unknown', () => {
|
|
41
|
+
const unknown = paths.filter((p) => classifyModulePath(p) === 'unknown');
|
|
42
|
+
expect(unknown).toEqual([]);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test('celilo-owned paths classify derived', () => {
|
|
46
|
+
const derived = paths.filter(
|
|
47
|
+
(p) =>
|
|
48
|
+
p.startsWith('generated/') ||
|
|
49
|
+
p.includes('node_modules/') ||
|
|
50
|
+
p === 'checksums.json' ||
|
|
51
|
+
p === 'signature.sig',
|
|
52
|
+
);
|
|
53
|
+
// Guard the guard: if this is empty the assertion below is vacuous.
|
|
54
|
+
expect(derived.length).toBeGreaterThan(0);
|
|
55
|
+
for (const p of derived) {
|
|
56
|
+
expect(`${p} => ${classifyModulePath(p)}`).toBe(`${p} => derived`);
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("the module's own content classifies package", () => {
|
|
61
|
+
const own = paths.filter(
|
|
62
|
+
(p) =>
|
|
63
|
+
!p.startsWith('generated/') &&
|
|
64
|
+
!p.includes('node_modules/') &&
|
|
65
|
+
p !== 'checksums.json' &&
|
|
66
|
+
p !== 'signature.sig',
|
|
67
|
+
);
|
|
68
|
+
expect(own.length).toBeGreaterThan(0);
|
|
69
|
+
for (const p of own) {
|
|
70
|
+
expect(`${p} => ${classifyModulePath(p)}`).toBe(`${p} => package`);
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
describe('classifyModulePath: source-tree paths that must never be installed', () => {
|
|
78
|
+
test.each([
|
|
79
|
+
['e2e/deploy.test.ts'],
|
|
80
|
+
['.git/config'],
|
|
81
|
+
['scripts/tsconfig.json'],
|
|
82
|
+
['.DS_Store'],
|
|
83
|
+
['server/src/api.test.ts'],
|
|
84
|
+
['wireguard.netapp'],
|
|
85
|
+
['node_modules/tldts/package.json'],
|
|
86
|
+
])('%s is unknown', (relPath) => {
|
|
87
|
+
expect(classifyModulePath(relPath)).toBe('unknown');
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
describe('classifyModulePath: composes with includeNodeModulesPath', () => {
|
|
92
|
+
test('the hook runtime closure is derived, not unknown', () => {
|
|
93
|
+
expect(classifyModulePath('scripts/node_modules/tldts/index.js')).toBe('derived');
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test('a .bin shim is still excluded', () => {
|
|
97
|
+
expect(classifyModulePath('scripts/node_modules/.bin/tsc')).toBe('unknown');
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test('non-scripts node_modules ships only @celilo/capabilities', () => {
|
|
101
|
+
expect(classifyModulePath('node_modules/@celilo/capabilities/src/index.ts')).toBe('derived');
|
|
102
|
+
expect(classifyModulePath('node_modules/@celilo/cli/index.js')).toBe('unknown');
|
|
103
|
+
});
|
|
104
|
+
});
|
|
@@ -5,13 +5,40 @@ import { extract as tarExtract } from 'tar';
|
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
import { parseJsonWithValidation } from '../../validation/schemas';
|
|
7
7
|
import { computeFileChecksum } from './checksum';
|
|
8
|
+
import { classifyModulePath } from './package-rules';
|
|
8
9
|
import { verifySignature } from './signature';
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* Integrity violation types
|
|
12
13
|
*/
|
|
13
14
|
export interface IntegrityViolation {
|
|
14
|
-
|
|
15
|
+
/**
|
|
16
|
+
* `stale-baseline` is not a file finding. It says the recorded checksums
|
|
17
|
+
* describe a DIFFERENT version of the module than the one celilo has
|
|
18
|
+
* installed, so every file finding beneath it is explained by the baseline
|
|
19
|
+
* being old rather than by the files having changed. Only `auditModule`
|
|
20
|
+
* produces it; package verification compares a package to its own manifest
|
|
21
|
+
* and cannot be stale in this sense.
|
|
22
|
+
*
|
|
23
|
+
* `stale-generated` is the second plane (D3): a verbatim role asset in
|
|
24
|
+
* `generated/` whose bytes are not the installed module's. That is what
|
|
25
|
+
* celilo#925 was, and what would ship on the next deploy.
|
|
26
|
+
*/
|
|
27
|
+
type: 'missing' | 'modified' | 'extra' | 'stale-baseline' | 'stale-generated';
|
|
28
|
+
/**
|
|
29
|
+
* What the baseline says the file should hash to, and what it actually
|
|
30
|
+
* hashes to. Both optional because not every violation is about a digest —
|
|
31
|
+
* a `stale-baseline` finding is about the row, not a file.
|
|
32
|
+
*
|
|
33
|
+
* These exist so `module verify --json` can answer "is the installed tree
|
|
34
|
+
* the 0.3.2 package" in one call. celilo#925 stalled for days on that
|
|
35
|
+
* question because nothing could read a file on celilo-mgr, and the
|
|
36
|
+
* tempting fix — a remote read primitive — is a real security surface (every
|
|
37
|
+
* module's secrets and vault material live under the same tree) for a
|
|
38
|
+
* question file hashes answer directly (D9).
|
|
39
|
+
*/
|
|
40
|
+
expectedDigest?: string;
|
|
41
|
+
actualDigest?: string | null;
|
|
15
42
|
path: string;
|
|
16
43
|
message: string;
|
|
17
44
|
}
|
|
@@ -164,8 +191,9 @@ export async function verifyPackageIntegrity(
|
|
|
164
191
|
const expectedFiles = new Set(Object.keys(expectedChecksums));
|
|
165
192
|
|
|
166
193
|
for (const file of actualFiles) {
|
|
167
|
-
//
|
|
168
|
-
|
|
194
|
+
// `checksums.json` / `signature.sig` and the rest of the derived set are
|
|
195
|
+
// never listed by the manifest they accompany.
|
|
196
|
+
if (classifyModulePath(file) === 'derived') {
|
|
169
197
|
continue;
|
|
170
198
|
}
|
|
171
199
|
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import {
|
|
6
|
+
compareVerbatimRoleAssets,
|
|
7
|
+
describeVerbatimDifferences,
|
|
8
|
+
readVerbatimRoleAssets,
|
|
9
|
+
} from './generated-plane';
|
|
10
|
+
|
|
11
|
+
describe('compareVerbatimRoleAssets', () => {
|
|
12
|
+
const asset = 'ansible/roles/vpn/files/vpn-linux-x86_64';
|
|
13
|
+
|
|
14
|
+
test('identical trees produce no differences', () => {
|
|
15
|
+
const both = new Map([[asset, 'aaa']]);
|
|
16
|
+
expect(compareVerbatimRoleAssets(both, new Map(both))).toEqual([]);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test('the celilo#925 shape: generated holds the previous version bytes', () => {
|
|
20
|
+
const differences = compareVerbatimRoleAssets(
|
|
21
|
+
new Map([[asset, 'new-digest']]),
|
|
22
|
+
new Map([[asset, 'old-digest']]),
|
|
23
|
+
);
|
|
24
|
+
expect(differences).toEqual([
|
|
25
|
+
{
|
|
26
|
+
relPath: asset,
|
|
27
|
+
reason: 'stale',
|
|
28
|
+
installedDigest: 'new-digest',
|
|
29
|
+
generatedDigest: 'old-digest',
|
|
30
|
+
},
|
|
31
|
+
]);
|
|
32
|
+
expect(describeVerbatimDifferences(differences)[0]).toContain(asset);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test('an asset generation never produced is missing, not stale', () => {
|
|
36
|
+
expect(compareVerbatimRoleAssets(new Map([[asset, 'aaa']]), new Map())).toEqual([
|
|
37
|
+
{ relPath: asset, reason: 'missing', installedDigest: 'aaa', generatedDigest: null },
|
|
38
|
+
]);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test('output for a role the module no longer has is not a difference', () => {
|
|
42
|
+
// Nothing includes it, so refusing a deploy over it would refuse a correct
|
|
43
|
+
// module for a file Ansible never reads.
|
|
44
|
+
expect(
|
|
45
|
+
compareVerbatimRoleAssets(new Map(), new Map([['ansible/roles/gone/files/x', 'aaa']])),
|
|
46
|
+
).toEqual([]);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
describe('readVerbatimRoleAssets', () => {
|
|
51
|
+
test('digests role files/ assets and nothing else', async () => {
|
|
52
|
+
const root = mkdtempSync(join(tmpdir(), 'celilo-plane-'));
|
|
53
|
+
try {
|
|
54
|
+
mkdirSync(join(root, 'ansible', 'roles', 'vpn', 'files', 'nested'), { recursive: true });
|
|
55
|
+
mkdirSync(join(root, 'ansible', 'roles', 'vpn', 'templates'), { recursive: true });
|
|
56
|
+
writeFileSync(join(root, 'ansible', 'roles', 'vpn', 'files', 'bin'), 'NEW');
|
|
57
|
+
writeFileSync(join(root, 'ansible', 'roles', 'vpn', 'files', 'nested', 'blob'), 'B');
|
|
58
|
+
// Templated, and expected to differ once generated. Must not be compared.
|
|
59
|
+
writeFileSync(join(root, 'ansible', 'roles', 'vpn', 'templates', 'x.j2'), '{{ v }}');
|
|
60
|
+
|
|
61
|
+
const assets = await readVerbatimRoleAssets(root);
|
|
62
|
+
expect([...assets.keys()].sort()).toEqual([
|
|
63
|
+
'ansible/roles/vpn/files/bin',
|
|
64
|
+
'ansible/roles/vpn/files/nested/blob',
|
|
65
|
+
]);
|
|
66
|
+
} finally {
|
|
67
|
+
rmSync(root, { recursive: true, force: true });
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test('a module with no ansible/ at all yields nothing rather than throwing', async () => {
|
|
72
|
+
const root = mkdtempSync(join(tmpdir(), 'celilo-plane-'));
|
|
73
|
+
try {
|
|
74
|
+
expect((await readVerbatimRoleAssets(root)).size).toBe(0);
|
|
75
|
+
} finally {
|
|
76
|
+
rmSync(root, { recursive: true, force: true });
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
});
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The second plane: is what we would deploy built from what we installed?
|
|
3
|
+
*
|
|
4
|
+
* A module exists in four places — the published package, the installed tree,
|
|
5
|
+
* the generated project, and the host — and celilo reported a version for the
|
|
6
|
+
* first as though it described the last. Between them sit three copies and two
|
|
7
|
+
* transformations, and nothing verified any of them against the one before it
|
|
8
|
+
* (openspec/changes/module-integrity-rigor, D3).
|
|
9
|
+
*
|
|
10
|
+
* Only VERBATIM assets have a meaningful expected digest. Ansible templates
|
|
11
|
+
* most of what it writes, and a `.j2` in `generated/` is supposed to differ
|
|
12
|
+
* from its source. `ansible/roles/<role>/files/` is the exception: it holds
|
|
13
|
+
* static assets — built binaries, certs, blobs — copied byte for byte because
|
|
14
|
+
* they need no variable resolution and may not survive utf-8 round-tripping.
|
|
15
|
+
*
|
|
16
|
+
* That is exactly where celilo#925 lived. `copyAnsibleRoleFilesDirs` skipped an
|
|
17
|
+
* existing destination on bun, so a module's built binary landed in
|
|
18
|
+
* `generated/` once, at first generate, and no later version replaced it.
|
|
19
|
+
* `cp` reported no error, Ansible copied that first binary forever and reported
|
|
20
|
+
* `ok`, and the module's version field advanced past code that was never
|
|
21
|
+
* shipped. This comparison is local, takes milliseconds, needs no SSH, and is
|
|
22
|
+
* the check that turns that silence into a refusal.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { existsSync } from 'node:fs';
|
|
26
|
+
import { readdir } from 'node:fs/promises';
|
|
27
|
+
import { join, relative } from 'node:path';
|
|
28
|
+
import { computeFileChecksum } from './checksum';
|
|
29
|
+
|
|
30
|
+
export interface VerbatimAssetDifference {
|
|
31
|
+
/** Path relative to the module root, identical in both trees. */
|
|
32
|
+
relPath: string;
|
|
33
|
+
/** `missing`: generation never produced it. `stale`: it holds other bytes. */
|
|
34
|
+
reason: 'missing' | 'stale';
|
|
35
|
+
installedDigest: string;
|
|
36
|
+
generatedDigest: string | null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Pure. Both sides are digest maps keyed by the same module-relative path.
|
|
41
|
+
*
|
|
42
|
+
* An asset present in `generated` but absent from `installed` is NOT reported.
|
|
43
|
+
* That is a role the module no longer has, whose old output nothing includes,
|
|
44
|
+
* and failing a deploy for it would refuse a correct module for a stale file
|
|
45
|
+
* Ansible never reads.
|
|
46
|
+
*/
|
|
47
|
+
export function compareVerbatimRoleAssets(
|
|
48
|
+
installed: ReadonlyMap<string, string>,
|
|
49
|
+
generated: ReadonlyMap<string, string>,
|
|
50
|
+
): VerbatimAssetDifference[] {
|
|
51
|
+
const differences: VerbatimAssetDifference[] = [];
|
|
52
|
+
for (const [relPath, installedDigest] of installed) {
|
|
53
|
+
const generatedDigest = generated.get(relPath) ?? null;
|
|
54
|
+
if (generatedDigest === null) {
|
|
55
|
+
differences.push({ relPath, reason: 'missing', installedDigest, generatedDigest: null });
|
|
56
|
+
} else if (generatedDigest !== installedDigest) {
|
|
57
|
+
differences.push({ relPath, reason: 'stale', installedDigest, generatedDigest });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return differences;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Digest every `ansible/roles/<role>/files/**` asset under `root`, keyed by its
|
|
65
|
+
* path relative to `root`. The generated project mirrors that layout, so the
|
|
66
|
+
* two maps this produces are directly comparable.
|
|
67
|
+
*/
|
|
68
|
+
export async function readVerbatimRoleAssets(root: string): Promise<Map<string, string>> {
|
|
69
|
+
const assets = new Map<string, string>();
|
|
70
|
+
const rolesDir = join(root, 'ansible', 'roles');
|
|
71
|
+
if (!existsSync(rolesDir)) return assets;
|
|
72
|
+
|
|
73
|
+
for (const role of await readdir(rolesDir, { withFileTypes: true })) {
|
|
74
|
+
if (!role.isDirectory()) continue;
|
|
75
|
+
const filesDir = join(rolesDir, role.name, 'files');
|
|
76
|
+
if (!existsSync(filesDir)) continue;
|
|
77
|
+
for (const filePath of await listFiles(filesDir)) {
|
|
78
|
+
assets.set(relative(root, filePath), await computeFileChecksum(filePath));
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return assets;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async function listFiles(dir: string): Promise<string[]> {
|
|
85
|
+
const found: string[] = [];
|
|
86
|
+
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
|
87
|
+
const full = join(dir, entry.name);
|
|
88
|
+
if (entry.isDirectory()) {
|
|
89
|
+
found.push(...(await listFiles(full)));
|
|
90
|
+
} else if (entry.isFile()) {
|
|
91
|
+
found.push(full);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return found;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* One line per difference, naming the file. celilo#925 took days partly because
|
|
99
|
+
* nothing anywhere named the file that had gone stale.
|
|
100
|
+
*/
|
|
101
|
+
export function describeVerbatimDifferences(differences: VerbatimAssetDifference[]): string[] {
|
|
102
|
+
return differences.map((d) =>
|
|
103
|
+
d.reason === 'missing'
|
|
104
|
+
? `${d.relPath}: generation never produced it (installed ${d.installedDigest})`
|
|
105
|
+
: `${d.relPath}: generated holds ${d.generatedDigest}, installed is ${d.installedDigest}`,
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The deploy pre-flight (D6). Returns the refusal message, or `null` to proceed.
|
|
111
|
+
*
|
|
112
|
+
* Local, milliseconds, no SSH, and it runs before anything contacts a system.
|
|
113
|
+
* It deliberately does NOT verify after the deploy: Ansible's `changed` is
|
|
114
|
+
* truthful about what Ansible did, and the lie in celilo#925 was upstream of
|
|
115
|
+
* Ansible.
|
|
116
|
+
*/
|
|
117
|
+
export async function refuseIfGeneratedIsStale(
|
|
118
|
+
moduleId: string,
|
|
119
|
+
modulePath: string,
|
|
120
|
+
generatedPath: string,
|
|
121
|
+
): Promise<string | null> {
|
|
122
|
+
const differences = compareVerbatimRoleAssets(
|
|
123
|
+
await readVerbatimRoleAssets(modulePath),
|
|
124
|
+
await readVerbatimRoleAssets(generatedPath),
|
|
125
|
+
);
|
|
126
|
+
if (differences.length === 0) return null;
|
|
127
|
+
return [
|
|
128
|
+
`Refusing to deploy ${moduleId}: the generated project does not match the installed module.`,
|
|
129
|
+
` ${differences.length} verbatim asset(s) differ:`,
|
|
130
|
+
...describeVerbatimDifferences(differences).map((line) => ` ${line}`),
|
|
131
|
+
'',
|
|
132
|
+
`Deploying would ship these bytes and report success. Run 'celilo module generate ${moduleId}' and try again.`,
|
|
133
|
+
].join('\n');
|
|
134
|
+
}
|