@celilo/cli 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -1
- package/CELILO_SUBSYSTEMS.md +18 -2
- package/MODULE_PRIMITIVES.md +25 -7
- package/drizzle/0026_module_integrity_version.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/src/capabilities/lookup.ts +39 -29
- package/src/capabilities/secret-ref.test.ts +24 -0
- package/src/capabilities/secret-validation.ts +50 -0
- package/src/capabilities/validation.test.ts +187 -2
- package/src/capabilities/validation.ts +53 -1
- package/src/cli/commands/alerts-sweep.ts +18 -0
- package/src/cli/commands/module-audit.ts +5 -2
- package/src/cli/commands/module-remove.ts +34 -2
- package/src/cli/commands/module-update.test.ts +238 -3
- package/src/cli/commands/module-update.ts +206 -12
- package/src/cli/commands/module-verify.ts +77 -13
- package/src/cli/commands/service-set-credentials.test.ts +108 -0
- package/src/cli/commands/service-set-credentials.ts +115 -0
- package/src/cli/commands/system-audit.ts +17 -0
- package/src/cli/commands/system-doctor.ts +78 -2
- package/src/cli/commands/system-migrate.ts +6 -4
- package/src/cli/commands/system-update.ts +33 -3
- package/src/cli/completion.ts +16 -1
- package/src/cli/index.ts +11 -2
- package/src/cli/tui/audit-state.ts +11 -3
- package/src/cli/tui/audit-tui.tsx +10 -4
- package/src/cli/tui/icons.ts +9 -2
- package/src/cli/tui/modals/analyzing.tsx +3 -0
- package/src/db/client.ts +10 -8
- package/src/db/migrate.test.ts +147 -0
- package/src/db/migrate.ts +69 -1
- package/src/db/schema.ts +5 -0
- package/src/hooks/capability-loader.test.ts +55 -0
- package/src/hooks/capability-loader.ts +16 -1
- package/src/manifest/json-schema-roundtrip.test.ts +12 -4
- package/src/manifest/schema.ts +23 -0
- package/src/module/import.ts +56 -40
- package/src/module/packaging/audit.ts +103 -28
- package/src/module/packaging/build.ts +12 -53
- package/src/module/packaging/classify-module-path.test.ts +104 -0
- package/src/module/packaging/extract.ts +31 -3
- package/src/module/packaging/generated-plane.test.ts +79 -0
- package/src/module/packaging/generated-plane.ts +134 -0
- package/src/module/packaging/host-plane.test.ts +132 -0
- package/src/module/packaging/host-plane.ts +135 -0
- package/src/module/packaging/package-rules.ts +62 -0
- package/src/policy/module-business-baseline.ts +0 -11
- package/src/services/alerting/monitors.ts +54 -2
- package/src/services/alerting/sweep-runner.ts +38 -1
- package/src/services/audit/cli-version.test.ts +6 -2
- package/src/services/audit/cli-version.ts +20 -6
- package/src/services/audit/detect-without-converge.test.ts +91 -0
- package/src/services/audit/detect-without-converge.ts +81 -0
- package/src/services/audit/disk-space.test.ts +5 -2
- package/src/services/audit/disk-space.ts +5 -3
- package/src/services/audit/health.test.ts +39 -0
- package/src/services/audit/index.test.ts +7 -1
- package/src/services/audit/index.ts +12 -0
- package/src/services/audit/module-integrity.test.ts +146 -0
- package/src/services/audit/module-integrity.ts +113 -0
- package/src/services/audit/module-versions.ts +4 -1
- package/src/services/audit/schema.test.ts +7 -2
- package/src/services/audit/schema.ts +19 -1
- package/src/services/audit/terraform-plan.ts +17 -2
- package/src/services/audit/types.test.ts +29 -0
- package/src/services/audit/types.ts +30 -4
- package/src/services/consumer-cleanup.ts +5 -3
- package/src/services/container-service.test.ts +34 -0
- package/src/services/container-service.ts +44 -0
- package/src/services/deployed-systems.test.ts +101 -0
- package/src/services/deployed-systems.ts +43 -11
- package/src/services/dns-provider-backfill.ts +30 -0
- package/src/services/fleet-checks.test.ts +26 -0
- package/src/services/fleet-checks.ts +11 -1
- package/src/services/module-deploy.ts +109 -41
- package/src/services/provider-arrival.test.ts +241 -0
- package/src/services/provider-arrival.ts +213 -0
- package/src/services/restore-from-file.ts +4 -0
- package/src/services/update/orchestrator.test.ts +2 -0
- package/src/templates/generator.test.ts +35 -0
- package/src/templates/generator.ts +29 -1
- package/src/variables/context.test.ts +63 -0
- package/src/variables/context.ts +10 -2
- package/src/variables/declarative-derivation.test.ts +47 -8
- package/src/variables/declarative-derivation.ts +6 -4
- package/src/services/public-web-republish.test.ts +0 -189
- package/src/services/public-web-republish.ts +0 -84
package/src/db/schema.ts
CHANGED
|
@@ -241,6 +241,11 @@ export const moduleIntegrity = sqliteTable('module_integrity', {
|
|
|
241
241
|
.unique()
|
|
242
242
|
.references(() => modules.id, { onDelete: 'cascade' }),
|
|
243
243
|
checksums: text('checksums', { mode: 'json' }).$type<Record<string, string>>().notNull(), // { "path/to/file": "xxhash", ... }
|
|
244
|
+
// Which version these checksums describe. NULL means the row was written
|
|
245
|
+
// before celilo stamped versions — a fact `module verify` reports rather
|
|
246
|
+
// than papers over, because a baseline whose version is unknown cannot be
|
|
247
|
+
// compared to the module's and is not evidence of anything.
|
|
248
|
+
version: text('version'),
|
|
244
249
|
signature: text('signature'), // Nullable - null for directory imports, populated for .netapp packages
|
|
245
250
|
importedAt: integer('imported_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
246
251
|
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
@@ -29,6 +29,17 @@ export default function registerHost(context) {
|
|
|
29
29
|
}
|
|
30
30
|
`;
|
|
31
31
|
|
|
32
|
+
const TEST_DHCP_SERVER_MODULE = `
|
|
33
|
+
export default function createDhcpServer(context) {
|
|
34
|
+
return {
|
|
35
|
+
async setDnsServers() {},
|
|
36
|
+
async getDnsServers() { return [context.config.marker]; },
|
|
37
|
+
async setDomainName() {},
|
|
38
|
+
async getDomainName() { return context.config.marker; },
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
`;
|
|
42
|
+
|
|
32
43
|
describe('Capability Loader', () => {
|
|
33
44
|
let db: DbClient;
|
|
34
45
|
let tempDir: string;
|
|
@@ -97,6 +108,50 @@ describe('Capability Loader', () => {
|
|
|
97
108
|
expect(result.dns_registrar).toBeTruthy();
|
|
98
109
|
});
|
|
99
110
|
|
|
111
|
+
test('prefers a provider explicitly scoped to the well-known capability zone', async () => {
|
|
112
|
+
for (const moduleId of ['upstream-dhcp', 'internal-dhcp']) {
|
|
113
|
+
const modulePath = join(tempDir, moduleId);
|
|
114
|
+
const scriptsDir = join(modulePath, 'scripts');
|
|
115
|
+
mkdirSync(scriptsDir, { recursive: true });
|
|
116
|
+
writeFileSync(join(scriptsDir, 'dhcp-server-functions.ts'), TEST_DHCP_SERVER_MODULE);
|
|
117
|
+
db.$client.run(
|
|
118
|
+
`INSERT INTO modules (id, name, version, source_path, manifest_data) VALUES ('${moduleId}', '${moduleId}', '1.0.0', '${modulePath}', '{}')`,
|
|
119
|
+
);
|
|
120
|
+
upsertModuleConfig(db, moduleId, 'marker', moduleId);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Insert the zone-agnostic upstream provider first to prove selection is
|
|
124
|
+
// policy-driven rather than database-order-driven.
|
|
125
|
+
db.$client.run(
|
|
126
|
+
`INSERT INTO capabilities (module_id, capability_name, version, data, zones, registered_at) VALUES ('upstream-dhcp', 'dhcp_server', '1.0.0', '{}', NULL, unixepoch())`,
|
|
127
|
+
);
|
|
128
|
+
db.$client.run(
|
|
129
|
+
`INSERT INTO capabilities (module_id, capability_name, version, data, zones, registered_at) VALUES ('internal-dhcp', 'dhcp_server', '1.0.0', '{}', '["internal"]', unixepoch())`,
|
|
130
|
+
);
|
|
131
|
+
|
|
132
|
+
const result = await loadCapabilityFunctions('dns-consumer', db, noopLogger);
|
|
133
|
+
const dhcp = result.dhcp_server as { getDomainName(): Promise<string> };
|
|
134
|
+
expect(await dhcp.getDomainName()).toBe('internal-dhcp');
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
test('falls back to a zone-agnostic provider when no explicit zone provider exists', async () => {
|
|
138
|
+
const modulePath = join(tempDir, 'upstream-dhcp');
|
|
139
|
+
const scriptsDir = join(modulePath, 'scripts');
|
|
140
|
+
mkdirSync(scriptsDir, { recursive: true });
|
|
141
|
+
writeFileSync(join(scriptsDir, 'dhcp-server-functions.ts'), TEST_DHCP_SERVER_MODULE);
|
|
142
|
+
db.$client.run(
|
|
143
|
+
`INSERT INTO modules (id, name, version, source_path, manifest_data) VALUES ('upstream-dhcp', 'upstream-dhcp', '1.0.0', '${modulePath}', '{}')`,
|
|
144
|
+
);
|
|
145
|
+
db.$client.run(
|
|
146
|
+
`INSERT INTO capabilities (module_id, capability_name, version, data, zones, registered_at) VALUES ('upstream-dhcp', 'dhcp_server', '1.0.0', '{}', NULL, unixepoch())`,
|
|
147
|
+
);
|
|
148
|
+
upsertModuleConfig(db, 'upstream-dhcp', 'marker', 'upstream-dhcp');
|
|
149
|
+
|
|
150
|
+
const result = await loadCapabilityFunctions('dns-consumer', db, noopLogger);
|
|
151
|
+
const dhcp = result.dhcp_server as { getDomainName(): Promise<string> };
|
|
152
|
+
expect(await dhcp.getDomainName()).toBe('upstream-dhcp');
|
|
153
|
+
});
|
|
154
|
+
|
|
100
155
|
test('injects a read-only web_routes view into the public_web provider own hooks (ISS-0035)', async () => {
|
|
101
156
|
const modulePath = join(tempDir, 'caddy');
|
|
102
157
|
mkdirSync(modulePath, { recursive: true });
|
|
@@ -27,6 +27,8 @@ import type {
|
|
|
27
27
|
TrustedSourceStore,
|
|
28
28
|
} from '@celilo/capabilities';
|
|
29
29
|
import { and, eq } from 'drizzle-orm';
|
|
30
|
+
import { selectCapabilityProvider } from '../capabilities/lookup';
|
|
31
|
+
import { WELL_KNOWN_CAPABILITIES } from '../capabilities/well-known';
|
|
30
32
|
import type { DbClient } from '../db/client';
|
|
31
33
|
import {
|
|
32
34
|
NETWORK_ZONES,
|
|
@@ -279,7 +281,20 @@ export async function loadCapabilityFunctions(
|
|
|
279
281
|
continue;
|
|
280
282
|
}
|
|
281
283
|
|
|
282
|
-
|
|
284
|
+
// A well-known capability's required zone is also its runtime selection
|
|
285
|
+
// context. Prefer a provider that explicitly serves that zone, then fall
|
|
286
|
+
// back to a zone-agnostic provider. Without this, multiple DHCP providers
|
|
287
|
+
// (for example an upstream router plus an internal LAN server) are selected
|
|
288
|
+
// by database insertion order and a consumer can reconfigure the wrong
|
|
289
|
+
// network boundary.
|
|
290
|
+
const selectionZone = WELL_KNOWN_CAPABILITIES[capName]?.required_zone;
|
|
291
|
+
const capability = selectCapabilityProvider(allProviders, selectionZone);
|
|
292
|
+
if (!capability) {
|
|
293
|
+
debugLog(
|
|
294
|
+
`${capName}: no provider serves required zone ${selectionZone ?? '(unspecified)'}, skipping`,
|
|
295
|
+
);
|
|
296
|
+
continue;
|
|
297
|
+
}
|
|
283
298
|
debugLog(`${capName}: found provider module ${capability.moduleId}`);
|
|
284
299
|
|
|
285
300
|
const providerModule = db
|
|
@@ -3,10 +3,18 @@
|
|
|
3
3
|
* under both the Zod schema (ModuleManifestSchema) and the exported JSON
|
|
4
4
|
* Schema at <repo>/schemas/module-manifest.schema.json.
|
|
5
5
|
*
|
|
6
|
-
* This
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
6
|
+
* This is the INDIRECT drift detector, not the primary guard. `bun run
|
|
7
|
+
* check:schema` (a step in the CI validate job) regenerates the schema and
|
|
8
|
+
* compares it byte-for-byte, so it fails the moment schema.ts changes without a
|
|
9
|
+
* regenerate. This test only fails once some real manifest in the repo happens
|
|
10
|
+
* to declare something a stale schema rejects, which can be days later and in
|
|
11
|
+
* an unrelated PR. Do not mistake one for the other: if you add a field to
|
|
12
|
+
* schema.ts, `check:schema` is what tells you to run `bun run export:schema`.
|
|
13
|
+
*
|
|
14
|
+
* What this test is actually for:
|
|
15
|
+
* 1. Every manifest the repo ships really does validate under both
|
|
16
|
+
* validators, so a stale schema cannot sit unnoticed once it starts
|
|
17
|
+
* rejecting live manifests.
|
|
10
18
|
* 2. zod-to-json-schema's translation hasn't lost fidelity in a way that
|
|
11
19
|
* would let bad manifests pass the editor (red underlines) while still
|
|
12
20
|
* failing at module import time. If a real manifest in the repo is
|
package/src/manifest/schema.ts
CHANGED
|
@@ -677,6 +677,29 @@ export const ModuleManifestSchema = z
|
|
|
677
677
|
|
|
678
678
|
hooks: z.object(HOOK_SCHEMAS).strict().optional(),
|
|
679
679
|
|
|
680
|
+
/**
|
|
681
|
+
* How much of `celilo module verify` this module can honestly answer.
|
|
682
|
+
*
|
|
683
|
+
* `deep: false` opts the module out of the host plane — the `--deep` pass
|
|
684
|
+
* that evaluates its generated playbook in check mode. The escape hatch
|
|
685
|
+
* exists because Ansible's check mode SKIPS a task it cannot evaluate, so a
|
|
686
|
+
* role driven by `command:` / `shell:` can report `changed=0` having never
|
|
687
|
+
* been applied. For most roles that lands as `unmeasured`, which is honest.
|
|
688
|
+
* A role where it would land as a persistent false `drift` should say so
|
|
689
|
+
* here instead of training operators to ignore the output.
|
|
690
|
+
*
|
|
691
|
+
* `reason` is REQUIRED. An opt-out with no stated reason is a check that
|
|
692
|
+
* disappeared, and celilo prints the reason wherever the module is verified
|
|
693
|
+
* (openspec/changes/module-integrity-rigor, D4 / task 6.3).
|
|
694
|
+
*/
|
|
695
|
+
verify: z
|
|
696
|
+
.object({
|
|
697
|
+
deep: z.literal(false),
|
|
698
|
+
reason: z.string().min(1),
|
|
699
|
+
})
|
|
700
|
+
.strict()
|
|
701
|
+
.optional(),
|
|
702
|
+
|
|
680
703
|
build: z
|
|
681
704
|
.object({
|
|
682
705
|
/** Inline shell command to build the module (run via bash -c). Mutually exclusive with script. */
|
package/src/module/import.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { execSync } from 'node:child_process';
|
|
2
2
|
import { existsSync, statSync, writeFileSync } from 'node:fs';
|
|
3
3
|
import { copyFile, mkdir, readFile, readdir } from 'node:fs/promises';
|
|
4
|
-
import { join } from 'node:path';
|
|
4
|
+
import { join, relative } from 'node:path';
|
|
5
5
|
import { eq } from 'drizzle-orm';
|
|
6
6
|
import { z } from 'zod';
|
|
7
7
|
import { getWellKnownCapability, isWellKnown } from '../capabilities/well-known';
|
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
} from '../manifest/validate';
|
|
24
24
|
import { parseJsonWithValidation } from '../validation/schemas';
|
|
25
25
|
import { cleanupTempDir, extractPackage, verifyPackageIntegrity } from './packaging/extract';
|
|
26
|
+
import { classifyModulePath } from './packaging/package-rules';
|
|
26
27
|
|
|
27
28
|
/**
|
|
28
29
|
* Phase-timing helper for `celilo module import`. Set CELILO_IMPORT_DEBUG=1
|
|
@@ -232,29 +233,14 @@ export async function copyModuleFiles(sourcePath: string, targetPath: string): P
|
|
|
232
233
|
for (const entry of entries) {
|
|
233
234
|
const srcPath = join(src, entry.name);
|
|
234
235
|
const destPath = join(dest, entry.name);
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
//
|
|
242
|
-
|
|
243
|
-
// runtime deps (@celilo/capabilities etc.). Top-level node_modules
|
|
244
|
-
// (monorepo workspace deps, build tools) are skipped.
|
|
245
|
-
if (
|
|
246
|
-
entry.isDirectory() &&
|
|
247
|
-
(entry.name === '.git' || entry.name === '.next' || entry.name === '.cache')
|
|
248
|
-
) {
|
|
249
|
-
continue;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
if (
|
|
253
|
-
entry.isDirectory() &&
|
|
254
|
-
entry.name === 'node_modules' &&
|
|
255
|
-
!src.endsWith('/scripts') &&
|
|
256
|
-
!src.endsWith('\\scripts')
|
|
257
|
-
) {
|
|
236
|
+
const relPath = relative(sourcePath, srcPath);
|
|
237
|
+
|
|
238
|
+
// `unknown` is everything that belongs to the module's SOURCE tree and
|
|
239
|
+
// not to its install: `.git/`, `e2e/`, tests, `tsconfig.json`, the
|
|
240
|
+
// node_modules the canonical rule drops. Skipping it here is what keeps
|
|
241
|
+
// the `unknown` class empty on a healthy install, so any `unknown` that
|
|
242
|
+
// audit later reports is real. `package` and `derived` both land.
|
|
243
|
+
if (classifyModulePath(relPath) === 'unknown') {
|
|
258
244
|
continue;
|
|
259
245
|
}
|
|
260
246
|
|
|
@@ -314,7 +300,8 @@ export function moduleExists(moduleId: string, db = getDb()): boolean {
|
|
|
314
300
|
* Validate well-known capabilities
|
|
315
301
|
*
|
|
316
302
|
* Policy function - checks if module's well-known capabilities are valid:
|
|
317
|
-
* 1. No other module provides the same well-known capability
|
|
303
|
+
* 1. No other module provides the same well-known capability in an
|
|
304
|
+
* overlapping scope (zone-aware uniqueness)
|
|
318
305
|
* 2. Module's zone matches capability's required zone (zone enforcement)
|
|
319
306
|
*
|
|
320
307
|
* @param manifest - Module manifest
|
|
@@ -335,16 +322,30 @@ export async function validateWellKnownCapabilities(
|
|
|
335
322
|
|
|
336
323
|
const wellKnown = getWellKnownCapability(capability.name);
|
|
337
324
|
|
|
338
|
-
// Check 1: Capability uniqueness
|
|
325
|
+
// Check 1: Capability uniqueness within an overlapping scope. An explicit
|
|
326
|
+
// zone-scoped provider may coexist with a zone-agnostic fallback because
|
|
327
|
+
// lookup deterministically prefers the explicit match. Two agnostic
|
|
328
|
+
// providers, or two explicit providers sharing a zone, remain ambiguous.
|
|
339
329
|
const existingCapability = await db
|
|
340
330
|
.select()
|
|
341
331
|
.from(capabilities)
|
|
342
332
|
.where(eq(capabilities.capabilityName, capability.name))
|
|
343
333
|
.all();
|
|
344
334
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
335
|
+
const newZones = capability.zones ?? null;
|
|
336
|
+
const conflictingModule = existingCapability.find((candidate) => {
|
|
337
|
+
const existingZones = candidate.zones ?? null;
|
|
338
|
+
|
|
339
|
+
if (newZones === null || existingZones === null) {
|
|
340
|
+
return newZones === null && existingZones === null;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
return newZones.some((zone) => existingZones.includes(zone));
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
if (conflictingModule) {
|
|
347
|
+
const scope = newZones ? ` zone(s) ${newZones.join(', ')}` : ' the zone-agnostic scope';
|
|
348
|
+
return `Well-known capability '${capability.name}' is already provided in${scope} by module '${conflictingModule.moduleId}'. Remove '${conflictingModule.moduleId}' or use a non-overlapping explicit zone scope before importing this module.`;
|
|
348
349
|
}
|
|
349
350
|
|
|
350
351
|
// Check 2: Zone enforcement - module must be in the correct zone
|
|
@@ -699,17 +700,32 @@ export async function importModule(options: ModuleImportOptions): Promise<Module
|
|
|
699
700
|
// Execution: Store integrity data from the package's checksums + signature.
|
|
700
701
|
// Directory imports go through the packager too (see top of importModule),
|
|
701
702
|
// so by the time we reach this point we always have these.
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
703
|
+
//
|
|
704
|
+
// UPSERT, not INSERT. `moduleId` is UNIQUE, so a re-import of an already
|
|
705
|
+
// imported module used to collide, get caught by a warn-and-continue, and
|
|
706
|
+
// leave the FIRST import's checksums in place forever. Every file that
|
|
707
|
+
// legitimately changed since then read as [MODIFIED] and the baseline
|
|
708
|
+
// described a version nobody could name. Failing to record the baseline is
|
|
709
|
+
// not "non-fatal": it is the state that made `module verify` useless, so
|
|
710
|
+
// this no longer swallows its own errors.
|
|
711
|
+
const integrityData: NewModuleIntegrity = {
|
|
712
|
+
moduleId: manifest.id,
|
|
713
|
+
checksums: checksums ?? {},
|
|
714
|
+
version: manifest.version,
|
|
715
|
+
signature: signature?.trim() ?? null,
|
|
716
|
+
};
|
|
717
|
+
db.insert(moduleIntegrity)
|
|
718
|
+
.values(integrityData)
|
|
719
|
+
.onConflictDoUpdate({
|
|
720
|
+
target: moduleIntegrity.moduleId,
|
|
721
|
+
set: {
|
|
722
|
+
checksums: integrityData.checksums,
|
|
723
|
+
version: integrityData.version,
|
|
724
|
+
signature: integrityData.signature,
|
|
725
|
+
updatedAt: new Date(),
|
|
726
|
+
},
|
|
727
|
+
})
|
|
728
|
+
.run();
|
|
713
729
|
|
|
714
730
|
// Record a successful build entry so deploy-time validation sees the
|
|
715
731
|
// artifacts in place and skips rebuild. The packager runs the manifest
|
|
@@ -4,8 +4,12 @@ import { join, relative } from 'node:path';
|
|
|
4
4
|
import { eq } from 'drizzle-orm';
|
|
5
5
|
import { getDb } from '../../db/client';
|
|
6
6
|
import { moduleIntegrity, modules } from '../../db/schema';
|
|
7
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
7
8
|
import { computeFileChecksum } from './checksum';
|
|
8
9
|
import type { IntegrityViolation } from './extract';
|
|
10
|
+
import { compareVerbatimRoleAssets, readVerbatimRoleAssets } from './generated-plane';
|
|
11
|
+
import { type HostPlaneResult, verifyModuleOnHosts } from './host-plane';
|
|
12
|
+
import { classifyModulePath } from './package-rules';
|
|
9
13
|
|
|
10
14
|
/**
|
|
11
15
|
* Audit result for a module
|
|
@@ -15,24 +19,27 @@ export interface AuditResult {
|
|
|
15
19
|
moduleId: string;
|
|
16
20
|
violations: IntegrityViolation[];
|
|
17
21
|
error?: string;
|
|
22
|
+
/** What the module records, and what version the baseline describes. */
|
|
23
|
+
moduleVersion?: string;
|
|
24
|
+
baselineVersion?: string | null;
|
|
25
|
+
/** Present only under `deep`. See `host-plane.ts`. */
|
|
26
|
+
hostPlane?: HostPlaneResult;
|
|
18
27
|
}
|
|
19
28
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
* Phase 2 belt-and-suspenders). A fresh copy appears on disk after
|
|
28
|
-
* extracting the package, but it isn't in `checksums.json` because the
|
|
29
|
-
* package may have been built before the type was generated.
|
|
30
|
-
*/
|
|
31
|
-
const FRAMEWORK_OWNED_PATHS = new Set(['celilo/types.d.ts']);
|
|
29
|
+
export interface AuditOptions {
|
|
30
|
+
/**
|
|
31
|
+
* Also ask each of the module's systems whether what is running is what
|
|
32
|
+
* celilo generated. One SSH per system, so it is off by default.
|
|
33
|
+
*/
|
|
34
|
+
deep?: boolean;
|
|
35
|
+
}
|
|
32
36
|
|
|
33
37
|
/**
|
|
34
|
-
* Recursively scan
|
|
35
|
-
*
|
|
38
|
+
* Recursively scan the installed tree, keeping only paths whose content is a
|
|
39
|
+
* stable integrity claim (`package`) or whose presence is a finding in itself
|
|
40
|
+
* (`unknown`). `derived` paths — `generated/**`, the hook runtime closure,
|
|
41
|
+
* `checksums.json` — are celilo's own and are dropped here, because a check
|
|
42
|
+
* that reports them can only ever be wrong.
|
|
36
43
|
*/
|
|
37
44
|
async function scanDirectory(dir: string, baseDir: string): Promise<string[]> {
|
|
38
45
|
const files: string[] = [];
|
|
@@ -42,21 +49,14 @@ async function scanDirectory(dir: string, baseDir: string): Promise<string[]> {
|
|
|
42
49
|
const fullPath = join(dir, entry.name);
|
|
43
50
|
const relativePath = relative(baseDir, fullPath);
|
|
44
51
|
|
|
45
|
-
if (FRAMEWORK_OWNED_PATHS.has(relativePath)) {
|
|
46
|
-
continue;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// Skip node_modules anywhere in the tree — these are installed by
|
|
50
|
-
// the framework during `module import` (NPM_PACKAGE_RESOLUTION
|
|
51
|
-
// Option B), not part of the module's signed checksums.
|
|
52
|
-
if (entry.isDirectory() && entry.name === 'node_modules') {
|
|
53
|
-
continue;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
52
|
if (entry.isDirectory()) {
|
|
53
|
+
// Prune whole derived subtrees rather than walking them. `generated/`
|
|
54
|
+
// alone carries terraform provider binaries.
|
|
55
|
+
if (classifyModulePath(relativePath) === 'derived') continue;
|
|
57
56
|
const subFiles = await scanDirectory(fullPath, baseDir);
|
|
58
57
|
files.push(...subFiles);
|
|
59
58
|
} else if (entry.isFile()) {
|
|
59
|
+
if (classifyModulePath(relativePath) === 'derived') continue;
|
|
60
60
|
files.push(relativePath);
|
|
61
61
|
}
|
|
62
62
|
}
|
|
@@ -71,7 +71,11 @@ async function scanDirectory(dir: string, baseDir: string): Promise<string[]> {
|
|
|
71
71
|
* @param db - Database client (optional, for testing)
|
|
72
72
|
* @returns Audit result with any violations found
|
|
73
73
|
*/
|
|
74
|
-
export async function auditModule(
|
|
74
|
+
export async function auditModule(
|
|
75
|
+
moduleId: string,
|
|
76
|
+
db = getDb(),
|
|
77
|
+
options: AuditOptions = {},
|
|
78
|
+
): Promise<AuditResult> {
|
|
75
79
|
const violations: IntegrityViolation[] = [];
|
|
76
80
|
|
|
77
81
|
try {
|
|
@@ -106,6 +110,26 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
106
110
|
const expectedChecksums: Record<string, string> = integrity.checksums;
|
|
107
111
|
const moduleDir = module.sourcePath;
|
|
108
112
|
|
|
113
|
+
// Which version do these checksums describe? Ahead of every file finding,
|
|
114
|
+
// because when the answer is "not the installed one" the file findings are
|
|
115
|
+
// a consequence of it and not independent evidence. Before D1 this was
|
|
116
|
+
// unanswerable: the row was written once at first import and `module
|
|
117
|
+
// update` never touched it, so verify reported the same violations whether
|
|
118
|
+
// the files were old or the checksums were old.
|
|
119
|
+
if (integrity.version === null) {
|
|
120
|
+
violations.push({
|
|
121
|
+
type: 'stale-baseline',
|
|
122
|
+
path: 'checksums.json',
|
|
123
|
+
message: `Baseline records no version — it was written before celilo stamped them, so it cannot be compared to the installed ${module.version}. Re-run 'celilo module update' for this module to refresh it.`,
|
|
124
|
+
});
|
|
125
|
+
} else if (integrity.version !== module.version) {
|
|
126
|
+
violations.push({
|
|
127
|
+
type: 'stale-baseline',
|
|
128
|
+
path: 'checksums.json',
|
|
129
|
+
message: `Baseline describes ${integrity.version}, module records ${module.version}. The checksums are old, not the files. Re-run 'celilo module update' for this module to refresh it.`,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
|
|
109
133
|
// Check if module directory exists
|
|
110
134
|
if (!existsSync(moduleDir)) {
|
|
111
135
|
return {
|
|
@@ -116,8 +140,12 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
116
140
|
};
|
|
117
141
|
}
|
|
118
142
|
|
|
119
|
-
// Validate all expected files exist and have correct checksums
|
|
143
|
+
// Validate all expected files exist and have correct checksums. A baseline
|
|
144
|
+
// entry for a derived path is not checkable: celilo rewrites those bytes
|
|
145
|
+
// after install (`bun install` over the hook runtime closure), so comparing
|
|
146
|
+
// them to what the package shipped can only ever produce a false positive.
|
|
120
147
|
for (const [filePath, expectedChecksum] of Object.entries(expectedChecksums)) {
|
|
148
|
+
if (classifyModulePath(filePath) === 'derived') continue;
|
|
121
149
|
const fullPath = join(moduleDir, filePath);
|
|
122
150
|
|
|
123
151
|
if (!existsSync(fullPath)) {
|
|
@@ -125,6 +153,8 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
125
153
|
type: 'missing',
|
|
126
154
|
path: filePath,
|
|
127
155
|
message: `Missing file: ${filePath}`,
|
|
156
|
+
expectedDigest: expectedChecksum,
|
|
157
|
+
actualDigest: null,
|
|
128
158
|
});
|
|
129
159
|
continue;
|
|
130
160
|
}
|
|
@@ -135,6 +165,33 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
135
165
|
type: 'modified',
|
|
136
166
|
path: filePath,
|
|
137
167
|
message: `Checksum mismatch: ${filePath}`,
|
|
168
|
+
expectedDigest: expectedChecksum,
|
|
169
|
+
actualDigest: actualChecksum,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Plane two: is what we would deploy built from what we installed? Only
|
|
175
|
+
// verbatim role assets have a meaningful expected digest — Ansible
|
|
176
|
+
// templates the rest, and a `.j2` in `generated/` is SUPPOSED to differ.
|
|
177
|
+
// A module that has never been generated has nothing to compare and is not
|
|
178
|
+
// a finding; it is simply pre-deploy.
|
|
179
|
+
const generatedDir = join(moduleDir, 'generated');
|
|
180
|
+
if (existsSync(generatedDir)) {
|
|
181
|
+
const differences = compareVerbatimRoleAssets(
|
|
182
|
+
await readVerbatimRoleAssets(moduleDir),
|
|
183
|
+
await readVerbatimRoleAssets(generatedDir),
|
|
184
|
+
);
|
|
185
|
+
for (const difference of differences) {
|
|
186
|
+
violations.push({
|
|
187
|
+
type: 'stale-generated',
|
|
188
|
+
path: difference.relPath,
|
|
189
|
+
expectedDigest: difference.installedDigest,
|
|
190
|
+
actualDigest: difference.generatedDigest,
|
|
191
|
+
message:
|
|
192
|
+
difference.reason === 'missing'
|
|
193
|
+
? `Generated project is missing ${difference.relPath} — the next deploy would ship nothing for it. Run 'celilo module generate ${moduleId}'.`
|
|
194
|
+
: `Generated project holds different bytes for ${difference.relPath} (generated ${difference.generatedDigest}, installed ${difference.installedDigest}) — the next deploy would ship the wrong ones. Run 'celilo module generate ${moduleId}'.`,
|
|
138
195
|
});
|
|
139
196
|
}
|
|
140
197
|
}
|
|
@@ -153,10 +210,28 @@ export async function auditModule(moduleId: string, db = getDb()): Promise<Audit
|
|
|
153
210
|
}
|
|
154
211
|
}
|
|
155
212
|
|
|
213
|
+
// Plane three: is what is running what we generated? One SSH per system,
|
|
214
|
+
// so it is asked only when the caller says so.
|
|
215
|
+
let hostPlane: HostPlaneResult | undefined;
|
|
216
|
+
if (options.deep) {
|
|
217
|
+
hostPlane = await verifyModuleOnHosts({
|
|
218
|
+
moduleId,
|
|
219
|
+
manifest: module.manifestData as unknown as ModuleManifest,
|
|
220
|
+
generatedPath: generatedDir,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// An `unmeasured` host is not a pass. A check that could not reach its
|
|
225
|
+
// subject says so, and does not count as green.
|
|
226
|
+
const hostPlaneClean = (hostPlane?.findings ?? []).every((f) => f.state === 'converged');
|
|
227
|
+
|
|
156
228
|
return {
|
|
157
|
-
success: violations.length === 0,
|
|
229
|
+
success: violations.length === 0 && hostPlaneClean,
|
|
158
230
|
moduleId,
|
|
159
231
|
violations,
|
|
232
|
+
moduleVersion: module.version,
|
|
233
|
+
baselineVersion: integrity.version,
|
|
234
|
+
hostPlane,
|
|
160
235
|
};
|
|
161
236
|
} catch (error) {
|
|
162
237
|
return {
|
|
@@ -9,7 +9,7 @@ import { log } from '../../cli/prompts';
|
|
|
9
9
|
import { formatViolations, scanModuleDirectory } from '../../policy/module-script-scan';
|
|
10
10
|
import { validateModuleDirectory } from '../import';
|
|
11
11
|
import { computeFileChecksum } from './checksum';
|
|
12
|
-
import { includeNodeModulesPath } from './package-rules';
|
|
12
|
+
import { classifyModulePath, includeNodeModulesPath } from './package-rules';
|
|
13
13
|
import { signChecksums } from './signature';
|
|
14
14
|
import { rewriteWorkspaceDeps } from './workspace-deps';
|
|
15
15
|
|
|
@@ -49,63 +49,22 @@ export interface ModuleBuildResult {
|
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
/**
|
|
52
|
-
*
|
|
52
|
+
* Check if a path inside the source dir should be excluded from the package.
|
|
53
53
|
*
|
|
54
|
-
* `
|
|
55
|
-
*
|
|
56
|
-
* closure
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
'.DS_Store',
|
|
61
|
-
'*.netapp',
|
|
62
|
-
'*.test.ts',
|
|
63
|
-
// Dev-only, same bucket as the tests: scripts/tsconfig.json exists so tsc can
|
|
64
|
-
// check hooks in CI. Nothing on a target ever runs tsc, and shipping it would
|
|
65
|
-
// make every module's packaged content change whenever the shared base moves.
|
|
66
|
-
'tsconfig.json',
|
|
67
|
-
'checksums.json',
|
|
68
|
-
'signature.sig',
|
|
69
|
-
];
|
|
70
|
-
|
|
71
|
-
/**
|
|
72
|
-
* Path-relative files the framework manages and that should never be
|
|
73
|
-
* bundled into a `.netapp` package. `celilo/types.d.ts` is generated
|
|
74
|
-
* post-import by `module import` (HOOK_API_V2 Phase 2 belt-and-suspenders),
|
|
75
|
-
* so shipping the local copy would just stamp the package with whatever
|
|
76
|
-
* was on the author's disk at build time — including potentially stale
|
|
77
|
-
* versions if the manifest changed since.
|
|
78
|
-
*/
|
|
79
|
-
const FRAMEWORK_OWNED_PATHS = new Set(['celilo/types.d.ts']);
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Check if a path inside the source dir should be excluded from the
|
|
83
|
-
* package.
|
|
84
|
-
*
|
|
85
|
-
* The `node_modules` decision is delegated to the canonical
|
|
86
|
-
* `includeNodeModulesPath` rule (package-rules.ts) — the single source of truth
|
|
87
|
-
* the registry-server's bootstrap packager is held to as well (ISS-0046).
|
|
54
|
+
* `classifyModulePath` (package-rules.ts) is the one answer to what belongs to
|
|
55
|
+
* a module. Packaging differs from it in exactly one place: the hook runtime
|
|
56
|
+
* closure under `scripts/node_modules/` is `derived` (celilo's `bun install`
|
|
57
|
+
* owns the on-disk copy) and still SHIPS, because a target may have no
|
|
58
|
+
* reachable registry (ISS-0046). Everything else `derived` is celilo's own
|
|
59
|
+
* output, or the checksum manifest that cannot list itself.
|
|
88
60
|
*/
|
|
89
61
|
function shouldExclude(filePath: string): boolean {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
// Module's e2e/ directory at the source root is tests + their deps —
|
|
95
|
-
// not part of the deployed module.
|
|
96
|
-
if (segments[0] === 'e2e') return true;
|
|
97
|
-
|
|
98
|
-
if (segments.includes('node_modules')) {
|
|
62
|
+
const cls = classifyModulePath(filePath);
|
|
63
|
+
if (cls === 'package') return false;
|
|
64
|
+
if (cls === 'derived' && filePath.split('/').includes('node_modules')) {
|
|
99
65
|
return !includeNodeModulesPath(filePath);
|
|
100
66
|
}
|
|
101
|
-
|
|
102
|
-
const name = basename(filePath);
|
|
103
|
-
return EXCLUDE_PATTERNS.some((pattern) => {
|
|
104
|
-
if (pattern.startsWith('*')) {
|
|
105
|
-
return name.endsWith(pattern.slice(1));
|
|
106
|
-
}
|
|
107
|
-
return name === pattern;
|
|
108
|
-
});
|
|
67
|
+
return true;
|
|
109
68
|
}
|
|
110
69
|
|
|
111
70
|
/**
|