@celilo/cli 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +16 -1
- package/package.json +4 -4
- package/src/cli/commands/hook-run.ts +5 -8
- package/src/cli/commands/ipam.ts +93 -0
- package/src/cli/commands/machine-add.ts +22 -0
- package/src/cli/commands/system-audit.ts +2 -0
- package/src/cli/commands/system-doctor.ts +148 -5
- package/src/cli/commands/system-update.ts +2 -0
- package/src/cli/completion.ts +38 -5
- package/src/cli/index.ts +10 -1
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/db/schema.ts +41 -1
- package/src/hooks/artifact-retention.test.ts +136 -0
- package/src/hooks/artifact-retention.ts +159 -0
- package/src/hooks/executor.test.ts +80 -0
- package/src/hooks/executor.ts +68 -23
- package/src/hooks/test-fixtures/artifact-writing-hook.ts +25 -0
- package/src/hooks/types.ts +20 -2
- package/src/ipam/allocator.test.ts +38 -0
- package/src/ipam/allocator.ts +63 -1
- package/src/ipam/auto-allocator.ts +7 -0
- package/src/policy/module-business-baseline.ts +404 -0
- package/src/policy/no-module-business-in-core.test.ts +504 -0
- package/src/services/alerting/keys.ts +21 -1
- package/src/services/alerting/run-monitor.ts +6 -1
- package/src/services/aspect-reconcile.test.ts +460 -0
- package/src/services/aspect-runner.test.ts +1 -0
- package/src/services/aspect-runner.ts +408 -37
- package/src/services/audit/browser-pin.test.ts +167 -0
- package/src/services/audit/browser-pin.ts +185 -0
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/types.ts +1 -0
- package/src/services/deploy-ansible-recap.test.ts +76 -0
- package/src/services/deploy-ansible.ts +56 -1
- package/src/services/health-runner.ts +15 -1
- package/src/services/module-deploy.ts +70 -16
- package/src/services/update/orchestrator.test.ts +1 -0
- package/src/system/browser-provisioning.test.ts +67 -0
- package/src/system/prereqs.test.ts +73 -0
- package/src/system/prereqs.ts +89 -12
- package/src/templates/ingress-ip.test.ts +108 -0
|
@@ -47,7 +47,7 @@ import {
|
|
|
47
47
|
EVENT_TYPES,
|
|
48
48
|
busInterviewGuarded,
|
|
49
49
|
} from './bus-interview';
|
|
50
|
-
import { executeAnsible } from './deploy-ansible';
|
|
50
|
+
import { type AnsibleHostRecap, executeAnsible, parseAnsibleRecap } from './deploy-ansible';
|
|
51
51
|
import { getContainerSystemsInZones } from './deployed-systems';
|
|
52
52
|
import { getSystemsByZone } from './machine-pool';
|
|
53
53
|
import { executeProxmoxReconcile, planProxmoxReconcile } from './proxmox-reconcile';
|
|
@@ -89,6 +89,12 @@ export interface AspectRunResult {
|
|
|
89
89
|
error?: string;
|
|
90
90
|
/** The plan that was executed, for caller logging / event emission. */
|
|
91
91
|
plan: AspectFanOutPlan;
|
|
92
|
+
/**
|
|
93
|
+
* Per-host `PLAY RECAP` counters. Empty when Ansible produced no recap — a
|
|
94
|
+
* state callers must NOT read as success. `verifyAspectCoverage` classifies
|
|
95
|
+
* from this.
|
|
96
|
+
*/
|
|
97
|
+
recap: AnsibleHostRecap[];
|
|
92
98
|
}
|
|
93
99
|
|
|
94
100
|
export interface AspectRunOptions {
|
|
@@ -101,6 +107,21 @@ export interface AspectRunOptions {
|
|
|
101
107
|
* they want it, the framework doesn't force a skip).
|
|
102
108
|
*/
|
|
103
109
|
excludeHostnames?: string[];
|
|
110
|
+
/**
|
|
111
|
+
* Restrict the fan-out to exactly these hostnames, intersected with the
|
|
112
|
+
* zone-derived target set. Used by the inbound reconcile
|
|
113
|
+
* (`reconcileAspectsForSystems`), which applies an aspect to the systems a
|
|
114
|
+
* deploy just created rather than to the whole fleet. Narrows only — a
|
|
115
|
+
* hostname the aspect's zones do not cover is still not a target.
|
|
116
|
+
*/
|
|
117
|
+
onlyHostnames?: string[];
|
|
118
|
+
/**
|
|
119
|
+
* Run Ansible in CHECK mode: evaluate the role against each target without
|
|
120
|
+
* changing anything. Used by `verifyAspectCoverage` to ask the HOST whether
|
|
121
|
+
* an aspect is applied, rather than consulting a stored claim that it once
|
|
122
|
+
* ran (celilo#902 design D6).
|
|
123
|
+
*/
|
|
124
|
+
check?: boolean;
|
|
104
125
|
/**
|
|
105
126
|
* Override for noInteractive mode passed to executeAnsible.
|
|
106
127
|
* Defaults to `true` because aspect fan-outs run as part of
|
|
@@ -117,8 +138,12 @@ export interface AspectRunOptions {
|
|
|
117
138
|
*/
|
|
118
139
|
export async function planAspectFanOut(
|
|
119
140
|
aspect: BaseModuleAspect,
|
|
120
|
-
options: Pick<AspectRunOptions, 'excludeHostnames'> = {},
|
|
141
|
+
options: Pick<AspectRunOptions, 'excludeHostnames' | 'onlyHostnames'> = {},
|
|
121
142
|
): Promise<AspectFanOutPlan> {
|
|
143
|
+
// `onlyHostnames` narrows, never widens: it is intersected with the
|
|
144
|
+
// zone-derived set below rather than replacing it, so an inbound reconcile
|
|
145
|
+
// cannot apply an aspect to a system outside its `applicable_zones`.
|
|
146
|
+
const only = options.onlyHostnames ? new Set(options.onlyHostnames) : undefined;
|
|
122
147
|
// Machine-pool systems in the zones (api_only recorded as skips for
|
|
123
148
|
// observability). excludeHostnames is applied by getSystemsByZone.
|
|
124
149
|
const allInZones = await getSystemsByZone(aspect.applicable_zones, {
|
|
@@ -134,6 +159,7 @@ export async function planAspectFanOut(
|
|
|
134
159
|
skipped.push({ machine: m, reason: 'api_only' });
|
|
135
160
|
continue;
|
|
136
161
|
}
|
|
162
|
+
if (only && !only.has(m.hostname)) continue;
|
|
137
163
|
targetSystems.push({
|
|
138
164
|
hostname: m.hostname,
|
|
139
165
|
ipAddress: m.ipAddress,
|
|
@@ -151,6 +177,7 @@ export async function planAspectFanOut(
|
|
|
151
177
|
const exclude = new Set(options.excludeHostnames ?? []);
|
|
152
178
|
for (const sys of getContainerSystemsInZones(aspect.applicable_zones, getDb())) {
|
|
153
179
|
if (exclude.has(sys.hostname) || seen.has(sys.hostname)) continue;
|
|
180
|
+
if (only && !only.has(sys.hostname)) continue;
|
|
154
181
|
targetSystems.push({
|
|
155
182
|
hostname: sys.hostname,
|
|
156
183
|
ipAddress: sys.ipv4_address,
|
|
@@ -326,13 +353,14 @@ export async function runAspectFanOut(args: {
|
|
|
326
353
|
|
|
327
354
|
const plan = await planAspectFanOut(aspect, {
|
|
328
355
|
excludeHostnames: options.excludeHostnames,
|
|
356
|
+
onlyHostnames: options.onlyHostnames,
|
|
329
357
|
});
|
|
330
358
|
|
|
331
359
|
if (plan.targetSystems.length === 0) {
|
|
332
360
|
log.info(
|
|
333
361
|
`Aspect fan-out for '${moduleId}' (${options.trigger}): no eligible systems in zones [${aspect.applicable_zones.join(', ')}]`,
|
|
334
362
|
);
|
|
335
|
-
return { success: true, output: '', plan };
|
|
363
|
+
return { success: true, output: '', plan, recap: [] };
|
|
336
364
|
}
|
|
337
365
|
|
|
338
366
|
log.info(
|
|
@@ -351,6 +379,7 @@ export async function runAspectFanOut(args: {
|
|
|
351
379
|
|
|
352
380
|
const result = await executeAnsible(workDir, {
|
|
353
381
|
noInteractive: options.noInteractive ?? true,
|
|
382
|
+
check: options.check,
|
|
354
383
|
});
|
|
355
384
|
|
|
356
385
|
// Proxmox reconciliation (D5): if the aspect declares
|
|
@@ -386,6 +415,7 @@ export async function runAspectFanOut(args: {
|
|
|
386
415
|
output: result.output,
|
|
387
416
|
error: result.error,
|
|
388
417
|
plan,
|
|
418
|
+
recap: parseAnsibleRecap(result.output),
|
|
389
419
|
};
|
|
390
420
|
} finally {
|
|
391
421
|
if (workDir) {
|
|
@@ -458,6 +488,61 @@ export interface AspectGlueResult {
|
|
|
458
488
|
runResult?: AspectRunResult;
|
|
459
489
|
}
|
|
460
490
|
|
|
491
|
+
/**
|
|
492
|
+
* The consent gate, shared by both fan-out directions.
|
|
493
|
+
*
|
|
494
|
+
* Already-approved → run. Already-DENIED → skip silently: the operator made a
|
|
495
|
+
* durable decision and re-prompting every deploy would be nagging (ISS-0027).
|
|
496
|
+
* Only the undecided ('no_approval') and stale ('scope_changed') states warrant
|
|
497
|
+
* an interview, and the decision is persisted either way so a denial isn't
|
|
498
|
+
* re-asked next deploy.
|
|
499
|
+
*
|
|
500
|
+
* Extracted so `reconcileAspectsForSystems` cannot drift from
|
|
501
|
+
* `maybeRunAspectForTrigger`: an inbound reconcile that skipped this would
|
|
502
|
+
* apply an aspect the operator never approved, or one they refused.
|
|
503
|
+
*/
|
|
504
|
+
async function ensureAspectConsent(args: {
|
|
505
|
+
moduleId: string;
|
|
506
|
+
version: string;
|
|
507
|
+
aspect: BaseModuleAspect;
|
|
508
|
+
trigger: BaseModuleAspectTrigger;
|
|
509
|
+
db: DbClient;
|
|
510
|
+
requestConsent?: AspectConsentRequest;
|
|
511
|
+
}): Promise<{ consented: true } | { consented: false; reason: AspectSkipReason }> {
|
|
512
|
+
const { moduleId, version, aspect, trigger, db } = args;
|
|
513
|
+
const approvalStatus = checkAspectApproval(moduleId, version, aspect, db);
|
|
514
|
+
|
|
515
|
+
if (approvalStatus === 'denied') {
|
|
516
|
+
return { consented: false, reason: 'denied' };
|
|
517
|
+
}
|
|
518
|
+
if (approvalStatus === 'no_approval' || approvalStatus === 'scope_changed') {
|
|
519
|
+
// ISS-0027: don't silently skip a declared aspect. Interview the operator
|
|
520
|
+
// for consent on the bus and WAIT. A responder (terminal, `events reply`,
|
|
521
|
+
// the celilo-deploy skill) approves or denies.
|
|
522
|
+
const requestConsent = args.requestConsent ?? requestAspectConsentViaBus;
|
|
523
|
+
const consented = await requestConsent({
|
|
524
|
+
moduleId,
|
|
525
|
+
version,
|
|
526
|
+
aspect,
|
|
527
|
+
trigger,
|
|
528
|
+
reason: approvalStatus,
|
|
529
|
+
});
|
|
530
|
+
recordAspectConsent({
|
|
531
|
+
moduleId,
|
|
532
|
+
version,
|
|
533
|
+
scopeHash: computeAspectScopeHash(aspect),
|
|
534
|
+
approver: process.env.USER ?? null,
|
|
535
|
+
consented,
|
|
536
|
+
db,
|
|
537
|
+
});
|
|
538
|
+
if (!consented) {
|
|
539
|
+
log.warn(`Aspect for '${moduleId}' consent refused; aspect skipped (will not re-prompt).`);
|
|
540
|
+
return { consented: false, reason: 'denied' };
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
return { consented: true };
|
|
544
|
+
}
|
|
545
|
+
|
|
461
546
|
/**
|
|
462
547
|
* Deploy-flow glue (SC4): consulted by `module-deploy.ts` after a
|
|
463
548
|
* primary deploy successfully completes. Decides whether to fan an
|
|
@@ -517,40 +602,16 @@ export async function maybeRunAspectForTrigger(args: {
|
|
|
517
602
|
return { ran: false, reason: 'no_approval' };
|
|
518
603
|
}
|
|
519
604
|
|
|
520
|
-
const
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
// ISS-0027: don't silently skip a declared aspect. Interview the operator
|
|
531
|
-
// for consent on the bus and WAIT. A responder (terminal, `events reply`,
|
|
532
|
-
// the celilo-deploy skill) approves or denies. We persist the decision —
|
|
533
|
-
// either way — so a denial isn't re-asked next deploy.
|
|
534
|
-
const requestConsent = args.requestConsent ?? requestAspectConsentViaBus;
|
|
535
|
-
const consented = await requestConsent({
|
|
536
|
-
moduleId,
|
|
537
|
-
version: moduleRow.version,
|
|
538
|
-
aspect,
|
|
539
|
-
trigger,
|
|
540
|
-
reason: approvalStatus,
|
|
541
|
-
});
|
|
542
|
-
recordAspectConsent({
|
|
543
|
-
moduleId,
|
|
544
|
-
version: moduleRow.version,
|
|
545
|
-
scopeHash: computeAspectScopeHash(aspect),
|
|
546
|
-
approver: process.env.USER ?? null,
|
|
547
|
-
consented,
|
|
548
|
-
db,
|
|
549
|
-
});
|
|
550
|
-
if (!consented) {
|
|
551
|
-
log.warn(`Aspect for '${moduleId}' consent refused; aspect skipped (will not re-prompt).`);
|
|
552
|
-
return { ran: false, reason: 'denied' };
|
|
553
|
-
}
|
|
605
|
+
const consent = await ensureAspectConsent({
|
|
606
|
+
moduleId,
|
|
607
|
+
version: moduleRow.version,
|
|
608
|
+
aspect,
|
|
609
|
+
trigger,
|
|
610
|
+
db,
|
|
611
|
+
requestConsent: args.requestConsent,
|
|
612
|
+
});
|
|
613
|
+
if (!consent.consented) {
|
|
614
|
+
return { ran: false, reason: consent.reason };
|
|
554
615
|
}
|
|
555
616
|
|
|
556
617
|
const runner = args.runner ?? runAspectFanOut;
|
|
@@ -566,3 +627,313 @@ export async function maybeRunAspectForTrigger(args: {
|
|
|
566
627
|
});
|
|
567
628
|
return { ran: true, success: runResult.success, runResult };
|
|
568
629
|
}
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* A system the inbound reconcile may apply aspects to. Deliberately just a
|
|
633
|
+
* hostname and a zone: the reconcile decides eligibility from the zone and
|
|
634
|
+
* hands the hostname to `planAspectFanOut`, which already knows how to reach
|
|
635
|
+
* both a pool machine and a container_service LXC.
|
|
636
|
+
*/
|
|
637
|
+
export interface AspectReconcileSystem {
|
|
638
|
+
hostname: string;
|
|
639
|
+
/**
|
|
640
|
+
* `string`, not `NetworkZone`, to match what everything on this path already
|
|
641
|
+
* uses: `aspect.applicable_zones` is `string[]` from the manifest schema,
|
|
642
|
+
* `getSystemsByZone` takes `string[]`, and `DeployedSystem.zone` — the
|
|
643
|
+
* capability contract this is fed from — is `string`. Narrowing here would
|
|
644
|
+
* only add a cast at every call site.
|
|
645
|
+
*/
|
|
646
|
+
zone: string;
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/** One provider's aspect, applied (or not) to the systems it covered. */
|
|
650
|
+
export interface AspectReconcileOutcome {
|
|
651
|
+
/** The module whose aspect this is — NOT the module being deployed. */
|
|
652
|
+
providerModuleId: string;
|
|
653
|
+
role: string;
|
|
654
|
+
/** The subset of the offered systems this aspect's zones actually cover. */
|
|
655
|
+
hostnames: string[];
|
|
656
|
+
ran: boolean;
|
|
657
|
+
success: boolean;
|
|
658
|
+
error?: string;
|
|
659
|
+
/** Set when `ran === false`. */
|
|
660
|
+
reason?: AspectSkipReason | 'paused' | 'not_deployed' | 'no_covered_systems';
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
export interface AspectReconcileResult {
|
|
664
|
+
outcomes: AspectReconcileOutcome[];
|
|
665
|
+
/** The subset that ran and failed — what a caller decides fatality on. */
|
|
666
|
+
failures: AspectReconcileOutcome[];
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
/**
|
|
670
|
+
* The INBOUND direction: given systems that have just come into existence,
|
|
671
|
+
* apply every approved aspect the fleet already provides that covers their
|
|
672
|
+
* zones.
|
|
673
|
+
*
|
|
674
|
+
* This is the fix for celilo#902. `maybeRunAspectForTrigger` is provider-scoped
|
|
675
|
+
* — one module's aspect across the whole fleet, enumerated at that instant —
|
|
676
|
+
* and nothing re-runs it, so a system provisioned later never receives an
|
|
677
|
+
* aspect and nothing reports the gap.
|
|
678
|
+
*
|
|
679
|
+
* TWO THINGS ARE DELIBERATELY NOT CONSULTED HERE, and both are load-bearing:
|
|
680
|
+
*
|
|
681
|
+
* 1. `aspect.triggers` (design D2). Eligibility is `applicable_zones` plus
|
|
682
|
+
* operator approval, full stop. `triggers` declares which PROVIDER-side
|
|
683
|
+
* events cause a fleet-wide fan-out; a system joining a zone is a framework
|
|
684
|
+
* event, not a provider event, and the spec's scenario does not condition
|
|
685
|
+
* on it either ("every approved aspect whose `applicable_zones` includes
|
|
686
|
+
* that zone SHALL run on the new system"). Requiring modules to declare
|
|
687
|
+
* `on_new_system_in_zone` would change every aspect's scope hash — raising
|
|
688
|
+
* a re-approval interview on the live fleet as a side effect of a bug fix —
|
|
689
|
+
* and would make correct convergence opt-in, so the next module that forgot
|
|
690
|
+
* the declaration would reintroduce this bug quietly.
|
|
691
|
+
* 2. The deploying module's own aspect. `on_install` already fans that one out
|
|
692
|
+
* across the whole fleet, so the caller passes it in `excludeModuleIds`.
|
|
693
|
+
*
|
|
694
|
+
* PAUSED PROVIDERS ARE SKIPPED (design D4a), and that is the documented escape
|
|
695
|
+
* hatch: an inbound aspect failure is fatal to the deploy that created the
|
|
696
|
+
* system, so an operator whose non-essential aspect is wedging every deploy
|
|
697
|
+
* pauses its provider, deploys, and unpauses.
|
|
698
|
+
*
|
|
699
|
+
* Consent is checked exactly as the outbound direction checks it, through the
|
|
700
|
+
* shared `ensureAspectConsent`.
|
|
701
|
+
*/
|
|
702
|
+
export async function reconcileAspectsForSystems(args: {
|
|
703
|
+
systems: AspectReconcileSystem[];
|
|
704
|
+
db: DbClient;
|
|
705
|
+
/** Providers to skip — the deploying module, whose own aspect already ran. */
|
|
706
|
+
excludeModuleIds?: string[];
|
|
707
|
+
runner?: typeof runAspectFanOut;
|
|
708
|
+
requestConsent?: AspectConsentRequest;
|
|
709
|
+
}): Promise<AspectReconcileResult> {
|
|
710
|
+
const { systems, db } = args;
|
|
711
|
+
const outcomes: AspectReconcileOutcome[] = [];
|
|
712
|
+
if (systems.length === 0) return { outcomes, failures: [] };
|
|
713
|
+
|
|
714
|
+
const exclude = new Set(args.excludeModuleIds ?? []);
|
|
715
|
+
const runner = args.runner ?? runAspectFanOut;
|
|
716
|
+
|
|
717
|
+
for (const moduleRow of db.select().from(modules).all()) {
|
|
718
|
+
if (exclude.has(moduleRow.id)) continue;
|
|
719
|
+
|
|
720
|
+
const manifest = moduleRow.manifestData as ModuleManifest | null;
|
|
721
|
+
const aspect = manifest?.base_module_aspect;
|
|
722
|
+
if (!aspect) continue;
|
|
723
|
+
|
|
724
|
+
const base = { providerModuleId: moduleRow.id, role: aspect.ansible_role };
|
|
725
|
+
|
|
726
|
+
// A module that never deployed has nothing to fan out FROM — its
|
|
727
|
+
// ansible_vars resolve against infrastructure that does not exist.
|
|
728
|
+
if (moduleRow.state !== 'INSTALLED' && moduleRow.state !== 'VERIFIED') {
|
|
729
|
+
if (moduleRow.state === 'PAUSED') {
|
|
730
|
+
// Reported rather than silent: pausing to get past a wedged aspect is
|
|
731
|
+
// legitimate, but it must not quietly become permanent.
|
|
732
|
+
log.warn(
|
|
733
|
+
`Aspect '${aspect.ansible_role}' from paused module '${moduleRow.id}' NOT applied to ${systems.map((s) => s.hostname).join(', ')}. Unpause and redeploy it to converge them.`,
|
|
734
|
+
);
|
|
735
|
+
outcomes.push({ ...base, hostnames: [], ran: false, success: true, reason: 'paused' });
|
|
736
|
+
continue;
|
|
737
|
+
}
|
|
738
|
+
outcomes.push({ ...base, hostnames: [], ran: false, success: true, reason: 'not_deployed' });
|
|
739
|
+
continue;
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
const zones = new Set<string>(aspect.applicable_zones);
|
|
743
|
+
const covered = systems.filter((s) => zones.has(s.zone));
|
|
744
|
+
if (covered.length === 0) {
|
|
745
|
+
outcomes.push({
|
|
746
|
+
...base,
|
|
747
|
+
hostnames: [],
|
|
748
|
+
ran: false,
|
|
749
|
+
success: true,
|
|
750
|
+
reason: 'no_covered_systems',
|
|
751
|
+
});
|
|
752
|
+
continue;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
const consent = await ensureAspectConsent({
|
|
756
|
+
moduleId: moduleRow.id,
|
|
757
|
+
version: moduleRow.version,
|
|
758
|
+
aspect,
|
|
759
|
+
trigger: 'on_new_system_in_zone',
|
|
760
|
+
db,
|
|
761
|
+
requestConsent: args.requestConsent,
|
|
762
|
+
});
|
|
763
|
+
if (!consent.consented) {
|
|
764
|
+
outcomes.push({
|
|
765
|
+
...base,
|
|
766
|
+
hostnames: covered.map((s) => s.hostname),
|
|
767
|
+
ran: false,
|
|
768
|
+
success: true,
|
|
769
|
+
reason: consent.reason,
|
|
770
|
+
});
|
|
771
|
+
continue;
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
const hostnames = covered.map((s) => s.hostname);
|
|
775
|
+
log.info(
|
|
776
|
+
`Applying aspect '${aspect.ansible_role}' from '${moduleRow.id}' to newly created system(s): ${hostnames.join(', ')}`,
|
|
777
|
+
);
|
|
778
|
+
const runResult = await runner({
|
|
779
|
+
moduleId: moduleRow.id,
|
|
780
|
+
aspect,
|
|
781
|
+
moduleSourcePath: moduleRow.sourcePath,
|
|
782
|
+
options: { trigger: 'on_new_system_in_zone', onlyHostnames: hostnames },
|
|
783
|
+
db,
|
|
784
|
+
});
|
|
785
|
+
outcomes.push({
|
|
786
|
+
...base,
|
|
787
|
+
hostnames,
|
|
788
|
+
ran: true,
|
|
789
|
+
success: runResult.success,
|
|
790
|
+
error: runResult.error,
|
|
791
|
+
});
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
return { outcomes, failures: outcomes.filter((o) => o.ran && !o.success) };
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
/**
|
|
798
|
+
* How an aspect stands on one system, as MEASURED against that system.
|
|
799
|
+
*
|
|
800
|
+
* `unknown` is not a hedge and must not be collapsed into `applied`. Ansible's
|
|
801
|
+
* check mode does not evaluate a task that cannot support it — it SKIPS it —
|
|
802
|
+
* so a role built from `command:` / `shell:` tasks can finish a check run with
|
|
803
|
+
* `changed=0` having never been applied to the host at all. A two-state answer
|
|
804
|
+
* would report that as applied: a confidently clean verdict about an
|
|
805
|
+
* unconverged system, which is the same failure this whole approach exists to
|
|
806
|
+
* avoid (celilo#902 design D6). Absence of a change is not evidence of
|
|
807
|
+
* convergence when nothing was assessed.
|
|
808
|
+
*/
|
|
809
|
+
export type AspectCoverageState = 'applied' | 'missing' | 'unknown' | 'unreachable';
|
|
810
|
+
|
|
811
|
+
export interface AspectCoverageFinding {
|
|
812
|
+
providerModuleId: string;
|
|
813
|
+
role: string;
|
|
814
|
+
hostname: string;
|
|
815
|
+
state: AspectCoverageState;
|
|
816
|
+
/** Why, in operator-facing words. Always set for anything but 'applied'. */
|
|
817
|
+
detail?: string;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/**
|
|
821
|
+
* Ask every system the fleet's approved aspects entitle it to whether it
|
|
822
|
+
* actually has them (celilo#902 design D6/D7).
|
|
823
|
+
*
|
|
824
|
+
* NOTHING IS STORED AND NOTHING IS READ FROM A RECORD. Both halves of the
|
|
825
|
+
* question are answered from live state:
|
|
826
|
+
*
|
|
827
|
+
* - "should have" is `planAspectFanOut(aspect)` — literally the code that
|
|
828
|
+
* performs the fan-out, so the entitlement set cannot disagree with it. No
|
|
829
|
+
* separate query to drift.
|
|
830
|
+
* - "does have" is the aspect evaluated against the host in check mode. A row
|
|
831
|
+
* saying "applied" would be a claim about a remote host, and celilo has been
|
|
832
|
+
* burned by trusting exactly that (celilo#626: `dns_registrations` rows held
|
|
833
|
+
* a pinned `ip` and the refresher republished a dead address over correct
|
|
834
|
+
* public DNS, with every in-fleet check green).
|
|
835
|
+
*
|
|
836
|
+
* This SSHes to every entitled system, so it is not cheap and must not run in a
|
|
837
|
+
* default `system doctor` pass — it is gated behind `--deep`.
|
|
838
|
+
*
|
|
839
|
+
* Consent is checked but never REQUESTED: a read-only diagnostic must not raise
|
|
840
|
+
* an interview. An aspect that is unapproved, denied, or whose scope changed is
|
|
841
|
+
* simply not verified.
|
|
842
|
+
*/
|
|
843
|
+
export async function verifyAspectCoverage(args: {
|
|
844
|
+
db: DbClient;
|
|
845
|
+
/** Restrict to one provider — `doctor --deep <module>`. */
|
|
846
|
+
onlyModuleId?: string;
|
|
847
|
+
/** Injected in tests, exactly as the other entry points here do it. */
|
|
848
|
+
runner?: typeof runAspectFanOut;
|
|
849
|
+
}): Promise<AspectCoverageFinding[]> {
|
|
850
|
+
const { db } = args;
|
|
851
|
+
const runner = args.runner ?? runAspectFanOut;
|
|
852
|
+
const findings: AspectCoverageFinding[] = [];
|
|
853
|
+
|
|
854
|
+
for (const moduleRow of db.select().from(modules).all()) {
|
|
855
|
+
if (args.onlyModuleId && moduleRow.id !== args.onlyModuleId) continue;
|
|
856
|
+
|
|
857
|
+
const manifest = moduleRow.manifestData as ModuleManifest | null;
|
|
858
|
+
const aspect = manifest?.base_module_aspect;
|
|
859
|
+
if (!aspect) continue;
|
|
860
|
+
if (moduleRow.state !== 'INSTALLED' && moduleRow.state !== 'VERIFIED') {
|
|
861
|
+
// A paused provider's entitled systems are still reported (D4a) — pausing
|
|
862
|
+
// to get past a wedged aspect is legitimate, but it must not quietly
|
|
863
|
+
// become permanent. Reported without a probe: the provider is not running
|
|
864
|
+
// and re-running its role against every host would be a change, not a read.
|
|
865
|
+
if (moduleRow.state === 'PAUSED') {
|
|
866
|
+
const plan = await planAspectFanOut(aspect);
|
|
867
|
+
for (const target of plan.targetSystems) {
|
|
868
|
+
findings.push({
|
|
869
|
+
providerModuleId: moduleRow.id,
|
|
870
|
+
role: aspect.ansible_role,
|
|
871
|
+
hostname: target.hostname,
|
|
872
|
+
state: 'unknown',
|
|
873
|
+
detail: `'${moduleRow.id}' is PAUSED, so its aspect is not being applied to new systems. Unpause and redeploy it to converge.`,
|
|
874
|
+
});
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
continue;
|
|
878
|
+
}
|
|
879
|
+
if (checkAspectApproval(moduleRow.id, moduleRow.version, aspect, db) !== 'approved') continue;
|
|
880
|
+
|
|
881
|
+
const plan = await planAspectFanOut(aspect);
|
|
882
|
+
if (plan.targetSystems.length === 0) continue;
|
|
883
|
+
|
|
884
|
+
const result = await runner({
|
|
885
|
+
moduleId: moduleRow.id,
|
|
886
|
+
aspect,
|
|
887
|
+
moduleSourcePath: moduleRow.sourcePath,
|
|
888
|
+
options: { trigger: 'on_new_system_in_zone', check: true, noInteractive: true },
|
|
889
|
+
db,
|
|
890
|
+
});
|
|
891
|
+
|
|
892
|
+
const byHost = new Map(result.recap.map((r) => [r.host, r]));
|
|
893
|
+
for (const target of plan.targetSystems) {
|
|
894
|
+
const base = {
|
|
895
|
+
providerModuleId: moduleRow.id,
|
|
896
|
+
role: aspect.ansible_role,
|
|
897
|
+
hostname: target.hostname,
|
|
898
|
+
};
|
|
899
|
+
const recap = byHost.get(target.hostname);
|
|
900
|
+
if (!recap) {
|
|
901
|
+
findings.push({
|
|
902
|
+
...base,
|
|
903
|
+
state: 'unknown',
|
|
904
|
+
detail:
|
|
905
|
+
'Ansible produced no recap line for this host, so nothing was measured. Absence of a recap is not evidence the aspect is applied.',
|
|
906
|
+
});
|
|
907
|
+
continue;
|
|
908
|
+
}
|
|
909
|
+
if (recap.unreachable > 0 || recap.failed > 0) {
|
|
910
|
+
findings.push({
|
|
911
|
+
...base,
|
|
912
|
+
state: 'unreachable',
|
|
913
|
+
detail: 'The host could not be evaluated — it did not answer, or the role errored on it.',
|
|
914
|
+
});
|
|
915
|
+
continue;
|
|
916
|
+
}
|
|
917
|
+
if (recap.skipped > 0) {
|
|
918
|
+
// THE CASE THAT MUST NOT READ AS APPLIED. See AspectCoverageState.
|
|
919
|
+
findings.push({
|
|
920
|
+
...base,
|
|
921
|
+
state: 'unknown',
|
|
922
|
+
detail: `The role has ${recap.skipped} task(s) check mode cannot evaluate, so convergence was not measured. Prefer check-capable Ansible modules (copy, template, lineinfile, file, package, service) in aspect roles, or give a command/shell task an honest changed_when:.`,
|
|
923
|
+
});
|
|
924
|
+
continue;
|
|
925
|
+
}
|
|
926
|
+
if (recap.changed > 0) {
|
|
927
|
+
findings.push({
|
|
928
|
+
...base,
|
|
929
|
+
state: 'missing',
|
|
930
|
+
detail: `The aspect would change ${recap.changed} thing(s) on this host, so it is not applied. Run \`celilo module deploy ${moduleRow.id}\` to converge it.`,
|
|
931
|
+
});
|
|
932
|
+
continue;
|
|
933
|
+
}
|
|
934
|
+
findings.push({ ...base, state: 'applied' });
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
return findings;
|
|
939
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two assertions that matter here are both about STAYING QUIET.
|
|
3
|
+
*
|
|
4
|
+
* A host with no browser provisioned is the NORMAL case — installation is
|
|
5
|
+
* opt-in — so a check that fired there would put a finding on every default
|
|
6
|
+
* host in the fleet forever. And a module that bundles no browser client is
|
|
7
|
+
* almost every module. Getting either wrong turns a guardrail into noise
|
|
8
|
+
* that trains an operator to ignore the audit.
|
|
9
|
+
*
|
|
10
|
+
* The revision is read from the bundle's own `browsers.json`, never from a
|
|
11
|
+
* version→revision table, so the reader tests use real files.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
15
|
+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
16
|
+
import { tmpdir } from 'node:os';
|
|
17
|
+
import { join } from 'node:path';
|
|
18
|
+
import { auditBrowserPin, readBundledBrowserClient } from './browser-pin';
|
|
19
|
+
|
|
20
|
+
const PROVISIONED = { revision: '1223', playwrightVersion: '1.60.0' };
|
|
21
|
+
|
|
22
|
+
describe('auditBrowserPin', () => {
|
|
23
|
+
test('says nothing when no browser is provisioned', async () => {
|
|
24
|
+
// The opt-in default. `system doctor` reports the absence; this must not,
|
|
25
|
+
// or every host that never wanted a browser carries a permanent finding.
|
|
26
|
+
const findings = await auditBrowserPin({
|
|
27
|
+
consumers: [{ moduleId: 'lunacycle', clientVersion: '1.55.1', expectedRevision: '1193' }],
|
|
28
|
+
provisioned: null,
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
expect(findings).toEqual([]);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test('says nothing when no module bundles a browser client', async () => {
|
|
35
|
+
expect(await auditBrowserPin({ consumers: [], provisioned: PROVISIONED })).toEqual([]);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test('says nothing when the bundled client expects the installed revision', async () => {
|
|
39
|
+
const findings = await auditBrowserPin({
|
|
40
|
+
consumers: [{ moduleId: 'lunacycle', clientVersion: '1.60.0', expectedRevision: '1223' }],
|
|
41
|
+
provisioned: PROVISIONED,
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
expect(findings).toEqual([]);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test('reports drift — never blocked — when the revisions differ', async () => {
|
|
48
|
+
const findings = await auditBrowserPin({
|
|
49
|
+
consumers: [{ moduleId: 'lunacycle', clientVersion: '1.55.1', expectedRevision: '1193' }],
|
|
50
|
+
provisioned: PROVISIONED,
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
expect(findings).toHaveLength(1);
|
|
54
|
+
const finding = findings[0];
|
|
55
|
+
// Task 4.2: it warns, it never blocks. Under D2 the consumer passes an
|
|
56
|
+
// explicit executablePath, so this is a soft protocol risk, not a
|
|
57
|
+
// failure — blocking a deploy on it would be wrong.
|
|
58
|
+
expect(finding?.severity).toBe('drift');
|
|
59
|
+
expect(finding?.category).toBe('browser_pin');
|
|
60
|
+
expect(finding?.subject).toBe('lunacycle');
|
|
61
|
+
expect(finding?.actionable).toBe(false);
|
|
62
|
+
// Both revisions named, so the operator can tell which end to move.
|
|
63
|
+
expect(finding?.message).toContain('1193');
|
|
64
|
+
expect(finding?.message).toContain('1223');
|
|
65
|
+
expect(finding?.remediation).toContain('1.60.0');
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test('reports one finding per drifting module', async () => {
|
|
69
|
+
const findings = await auditBrowserPin({
|
|
70
|
+
consumers: [
|
|
71
|
+
{ moduleId: 'lunacycle', clientVersion: '1.55.1', expectedRevision: '1193' },
|
|
72
|
+
{ moduleId: 'aligned', clientVersion: '1.60.0', expectedRevision: '1223' },
|
|
73
|
+
{ moduleId: 'other', clientVersion: '1.50.0', expectedRevision: '1150' },
|
|
74
|
+
],
|
|
75
|
+
provisioned: PROVISIONED,
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
expect(findings.map((f) => f.subject)).toEqual(['lunacycle', 'other']);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe('readBundledBrowserClient', () => {
|
|
83
|
+
let root: string;
|
|
84
|
+
|
|
85
|
+
beforeEach(() => {
|
|
86
|
+
root = mkdtempSync(join(tmpdir(), 'celilo-browser-pin-'));
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
afterEach(() => {
|
|
90
|
+
rmSync(root, { recursive: true, force: true });
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
/** Write a bundled playwright-core under `<root>/<bundleDir>/node_modules`. */
|
|
94
|
+
function bundle(bundleDir: string, version: string, revision: number | string): void {
|
|
95
|
+
const pkg = join(root, bundleDir, 'node_modules', 'playwright-core');
|
|
96
|
+
mkdirSync(pkg, { recursive: true });
|
|
97
|
+
writeFileSync(join(pkg, 'package.json'), JSON.stringify({ version }));
|
|
98
|
+
writeFileSync(
|
|
99
|
+
join(pkg, 'browsers.json'),
|
|
100
|
+
JSON.stringify({
|
|
101
|
+
browsers: [
|
|
102
|
+
{ name: 'chromium', revision: 9999 },
|
|
103
|
+
{ name: 'chromium-headless-shell', revision },
|
|
104
|
+
],
|
|
105
|
+
}),
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
test('finds a bundle under the conventional scripts/ directory', () => {
|
|
110
|
+
bundle('scripts', '1.60.0', 1223);
|
|
111
|
+
|
|
112
|
+
expect(readBundledBrowserClient('m', root, ['./scripts/health-check.ts'])).toEqual({
|
|
113
|
+
moduleId: 'm',
|
|
114
|
+
clientVersion: '1.60.0',
|
|
115
|
+
expectedRevision: '1223',
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test('finds a bundle beside a hook script in a non-standard layout', () => {
|
|
120
|
+
// The one real browser consumer keeps its hooks in celilo/scripts/,
|
|
121
|
+
// so a hardcoded scripts/ would have missed exactly the module this
|
|
122
|
+
// check exists for.
|
|
123
|
+
bundle('celilo/scripts', '1.55.1', 1193);
|
|
124
|
+
|
|
125
|
+
const found = readBundledBrowserClient('lunacycle', root, ['./celilo/scripts/health-check.ts']);
|
|
126
|
+
|
|
127
|
+
expect(found?.expectedRevision).toBe('1193');
|
|
128
|
+
expect(found?.clientVersion).toBe('1.55.1');
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test('reads the revision as a STRING even though the file holds a number', () => {
|
|
132
|
+
// browsers.json stores it unquoted; comparing a number to the
|
|
133
|
+
// descriptor's string would never match and the check would be silent.
|
|
134
|
+
bundle('scripts', '1.60.0', 1223);
|
|
135
|
+
|
|
136
|
+
expect(readBundledBrowserClient('m', root, [])?.expectedRevision).toBe('1223');
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test('returns null when the module bundles no browser client', () => {
|
|
140
|
+
expect(readBundledBrowserClient('m', root, ['./scripts/on_install.ts'])).toBeNull();
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test('returns null when browsers.json is unreadable rather than throwing', () => {
|
|
144
|
+
const pkg = join(root, 'scripts', 'node_modules', 'playwright-core');
|
|
145
|
+
mkdirSync(pkg, { recursive: true });
|
|
146
|
+
writeFileSync(join(pkg, 'browsers.json'), 'not json {');
|
|
147
|
+
|
|
148
|
+
expect(readBundledBrowserClient('m', root, [])).toBeNull();
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test('a bundle without a readable package.json still reports its revision', () => {
|
|
152
|
+
// The revision is the load-bearing half; a missing version should not
|
|
153
|
+
// suppress a real drift finding.
|
|
154
|
+
const pkg = join(root, 'scripts', 'node_modules', 'playwright-core');
|
|
155
|
+
mkdirSync(pkg, { recursive: true });
|
|
156
|
+
writeFileSync(
|
|
157
|
+
join(pkg, 'browsers.json'),
|
|
158
|
+
JSON.stringify({ browsers: [{ name: 'chromium-headless-shell', revision: 1223 }] }),
|
|
159
|
+
);
|
|
160
|
+
|
|
161
|
+
expect(readBundledBrowserClient('m', root, [])).toEqual({
|
|
162
|
+
moduleId: 'm',
|
|
163
|
+
clientVersion: 'unknown',
|
|
164
|
+
expectedRevision: '1223',
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
});
|