@celilo/cli 0.26.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +3 -0
- package/CELILO_SUBSYSTEMS.md +4 -2
- package/drizzle/0025_port_forward_owner.sql +29 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +3 -3
- package/src/__integration__/container-services-cli.integration.test.ts +0 -4
- package/src/ansible/dependencies.test.ts +233 -289
- package/src/ansible/dependencies.ts +151 -83
- package/src/cli/commands/alerts-sweep.ts +14 -3
- package/src/cli/commands/machine-add.ts +0 -1
- package/src/cli/commands/machine-list.ts +10 -4
- package/src/cli/commands/machine-remove.ts +13 -7
- package/src/cli/commands/machine-status.ts +9 -11
- package/src/cli/commands/module-remove.ts +26 -23
- package/src/cli/commands/system-audit.ts +5 -1
- package/src/cli/commands/system-update.ts +10 -2
- package/src/db/schema.ts +30 -10
- package/src/hooks/capability-loader.ts +65 -13
- package/src/hooks/define-hook.test.ts +4 -6
- package/src/hooks/executor.ts +2 -1
- package/src/hooks/types.ts +9 -17
- package/src/infrastructure/property-extractor.test.ts +0 -2
- package/src/manifest/contracts/index.ts +20 -0
- package/src/manifest/contracts/v1.ts +33 -1
- package/src/manifest/schema.ts +48 -58
- package/src/services/alerting/sweep-runner.test.ts +5 -1
- package/src/services/alerting/sweep-runner.ts +14 -8
- package/src/services/aspect-runner.test.ts +0 -1
- package/src/services/audit/undeployed-modules.ts +18 -1
- package/src/services/consumer-cleanup.test.ts +347 -0
- package/src/services/consumer-cleanup.ts +244 -0
- package/src/services/infrastructure-selector.test.ts +0 -7
- package/src/services/infrastructure-selector.ts +24 -25
- package/src/services/infrastructure-variable-resolver.test.ts +0 -6
- package/src/services/infrastructure-variable-resolver.ts +0 -3
- package/src/services/machine-pool.test.ts +53 -85
- package/src/services/machine-pool.ts +68 -84
- package/src/services/module-deploy.ts +17 -39
- package/src/services/module-validator/index.test.ts +9 -0
- package/src/services/port-forwards.test.ts +93 -40
- package/src/services/port-forwards.ts +74 -48
- package/src/services/ssh-key-manager.test.ts +0 -10
- package/src/services/trusted-sources.test.ts +52 -13
- package/src/services/trusted-sources.ts +25 -15
- package/src/test-utils/cli-context.ts +15 -2
- package/src/types/infrastructure.ts +11 -1
- package/src/services/web-route-cleanup.test.ts +0 -250
- package/src/services/web-route-cleanup.ts +0 -144
|
@@ -575,7 +575,17 @@ export async function getInstalledCollections(): Promise<Map<string, InstalledCo
|
|
|
575
575
|
version: parseSemanticVersion(versionStr),
|
|
576
576
|
path: path as string,
|
|
577
577
|
});
|
|
578
|
-
} catch {
|
|
578
|
+
} catch (error) {
|
|
579
|
+
// Rule 6.2: never a bare catch. An unparseable version dropped the
|
|
580
|
+
// collection from the map entirely, so it read as NOT INSTALLED —
|
|
581
|
+
// and the installer would then try to install it again, every time,
|
|
582
|
+
// reporting success while nothing changed.
|
|
583
|
+
console.warn(
|
|
584
|
+
` ⚠ Ignoring installed collection '${name}': unparseable version '${versionStr}' (${
|
|
585
|
+
error instanceof Error ? error.message : String(error)
|
|
586
|
+
})`,
|
|
587
|
+
);
|
|
588
|
+
}
|
|
579
589
|
}
|
|
580
590
|
}
|
|
581
591
|
}
|
|
@@ -587,6 +597,41 @@ export async function getInstalledCollections(): Promise<Map<string, InstalledCo
|
|
|
587
597
|
}
|
|
588
598
|
}
|
|
589
599
|
|
|
600
|
+
/**
|
|
601
|
+
* Decide what an integrity outcome means for the import, and say so.
|
|
602
|
+
*
|
|
603
|
+
* A REFUTED collection stops the import; an UNCHECKABLE one warns. The split is
|
|
604
|
+
* the point (celilo#524):
|
|
605
|
+
*
|
|
606
|
+
* - `mismatch` — files on disk do not match the collection's own manifest.
|
|
607
|
+
* Ansible is about to execute that content, and celilo cannot account for
|
|
608
|
+
* it. Refusing costs an import; accepting runs unaccounted-for code.
|
|
609
|
+
* - `unverifiable` — the check could not run, most often because the
|
|
610
|
+
* collection ships no `file_manifest_file` checksum at all. That is a
|
|
611
|
+
* property of the publisher, not evidence of tampering, and blocking on it
|
|
612
|
+
* would refuse ordinary collections forever. It warns.
|
|
613
|
+
*
|
|
614
|
+
* Refusing is cheap HERE specifically. This runs during `module import`
|
|
615
|
+
* (`module/import.ts`), the only caller, and nothing installs collections in
|
|
616
|
+
* the deploy path — so a false positive means "this module did not import",
|
|
617
|
+
* not "the fleet stopped deploying". There is also no module row yet, so
|
|
618
|
+
* nothing is left half-created to clean up.
|
|
619
|
+
*
|
|
620
|
+
* ⚠️ This is SELF-ATTESTATION, not provenance: the checksums live inside the
|
|
621
|
+
* artifact being checked. It catches corruption and post-install modification.
|
|
622
|
+
* It cannot catch a coherently re-signed tampered collection, and a clean
|
|
623
|
+
* import must not be read as saying otherwise.
|
|
624
|
+
*/
|
|
625
|
+
export function reportIntegrity(name: string, outcome: IntegrityOutcome): string | null {
|
|
626
|
+
if (outcome.status === 'mismatch') {
|
|
627
|
+
return `Integrity check FAILED for Ansible collection '${name}': ${outcome.detail}.\nThe installed files do not match the collection's own manifest — this is corruption or tampering.\nImport refused rather than run unverified content.`;
|
|
628
|
+
}
|
|
629
|
+
if (outcome.status === 'unverifiable') {
|
|
630
|
+
console.warn(` ⚠ Integrity could not be checked for ${name}: ${outcome.reason} (non-fatal)`);
|
|
631
|
+
}
|
|
632
|
+
return null;
|
|
633
|
+
}
|
|
634
|
+
|
|
590
635
|
/**
|
|
591
636
|
* Install result
|
|
592
637
|
*/
|
|
@@ -599,16 +644,22 @@ export interface InstallResult {
|
|
|
599
644
|
}
|
|
600
645
|
|
|
601
646
|
/**
|
|
602
|
-
*
|
|
647
|
+
* The three things checking one file's checksum can tell you.
|
|
603
648
|
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
649
|
+
* Named states rather than `boolean | null` (celilo#524). The tri-state was
|
|
650
|
+
* right — a file that is not on disk is genuinely neither a match nor a
|
|
651
|
+
* mismatch — but spelled as `null` it read as "nothing to worry about", and
|
|
652
|
+
* every caller duly treated it as one. `absent` cannot be misread.
|
|
653
|
+
*/
|
|
654
|
+
export type FileCheck = 'match' | 'mismatch' | 'absent';
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* Verify one file against its expected SHA-256.
|
|
658
|
+
*
|
|
659
|
+
* `absent` means the file could not be read at all, which is a statement about
|
|
660
|
+
* our knowledge, not about the file's integrity.
|
|
661
|
+
*/
|
|
662
|
+
async function verifyFileChecksum(filePath: string, expectedChecksum: string): Promise<FileCheck> {
|
|
612
663
|
const { createHash } = await import('node:crypto');
|
|
613
664
|
const { readFile, access } = await import('node:fs/promises');
|
|
614
665
|
const { constants } = await import('node:fs');
|
|
@@ -621,112 +672,130 @@ async function verifyFileChecksum(
|
|
|
621
672
|
const hash = createHash('sha256');
|
|
622
673
|
hash.update(content);
|
|
623
674
|
const actualChecksum = hash.digest('hex');
|
|
624
|
-
return actualChecksum === expectedChecksum;
|
|
675
|
+
return actualChecksum === expectedChecksum ? 'match' : 'mismatch';
|
|
625
676
|
} catch (_error: unknown) {
|
|
626
|
-
//
|
|
627
|
-
return
|
|
677
|
+
// Unreadable. NOT evidence of integrity — see the type's note.
|
|
678
|
+
return 'absent';
|
|
628
679
|
}
|
|
629
680
|
}
|
|
630
681
|
|
|
631
682
|
/**
|
|
632
|
-
*
|
|
683
|
+
* What a verification run actually established.
|
|
684
|
+
*
|
|
685
|
+
* Three states, because there are three (celilo#524). The function used to
|
|
686
|
+
* return `boolean` and could not, in practice, return `false`: three
|
|
687
|
+
* independent paths reported "integrity verified" for a collection that had
|
|
688
|
+
* been tampered with, and no constructible input reached the failure return.
|
|
689
|
+
*
|
|
690
|
+
* `unverifiable` is the state that was missing. A manifest that declares no
|
|
691
|
+
* checksum, or a file set with nothing on disk to sample, tells you NOTHING
|
|
692
|
+
* about integrity — and folding that into `verified` is what let the one thing
|
|
693
|
+
* an attacker fully controls (the manifest they ship) switch the check off.
|
|
694
|
+
* Same absent-vs-empty distinction as `parseInterfaceBaseline`.
|
|
695
|
+
*/
|
|
696
|
+
export type IntegrityOutcome =
|
|
697
|
+
| { status: 'verified'; filesChecked: number }
|
|
698
|
+
/** Something was checked and did not match. Integrity is refuted. */
|
|
699
|
+
| { status: 'mismatch'; detail: string }
|
|
700
|
+
/** Nothing could be checked. Integrity is unknown — NOT confirmed. */
|
|
701
|
+
| { status: 'unverifiable'; reason: string };
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* Verify an installed Galaxy collection against the checksums in its own
|
|
705
|
+
* MANIFEST.json / FILES.json.
|
|
633
706
|
*
|
|
634
|
-
*
|
|
635
|
-
*
|
|
707
|
+
* ⚠️ This is self-attestation, not provenance: the checksums live inside the
|
|
708
|
+
* artifact being checked. It detects corruption and post-install modification,
|
|
709
|
+
* and it CANNOT detect a coherently re-signed tampered collection. That is a
|
|
710
|
+
* reason to be precise about what it reports, not a reason to report a
|
|
711
|
+
* comforting answer.
|
|
636
712
|
*/
|
|
637
|
-
async function verifyCollectionIntegrity(
|
|
713
|
+
export async function verifyCollectionIntegrity(
|
|
714
|
+
collectionInfo: InstalledCollection,
|
|
715
|
+
): Promise<IntegrityOutcome> {
|
|
638
716
|
const { readFile } = await import('node:fs/promises');
|
|
639
717
|
const { join } = await import('node:path');
|
|
640
718
|
|
|
719
|
+
const [namespace, collection] = collectionInfo.name.split('.');
|
|
720
|
+
const dir = join(collectionInfo.path, namespace, collection);
|
|
721
|
+
|
|
641
722
|
try {
|
|
642
|
-
// Read MANIFEST.json
|
|
643
|
-
const manifestPath = join(
|
|
644
|
-
collectionInfo.path,
|
|
645
|
-
collectionInfo.name.split('.')[0],
|
|
646
|
-
collectionInfo.name.split('.')[1],
|
|
647
|
-
'MANIFEST.json',
|
|
648
|
-
);
|
|
649
|
-
const manifestContent = await readFile(manifestPath, 'utf-8');
|
|
650
723
|
const manifest = parseJsonWithValidation(
|
|
651
|
-
|
|
724
|
+
await readFile(join(dir, 'MANIFEST.json'), 'utf-8'),
|
|
652
725
|
GalaxyManifestSchema,
|
|
653
726
|
'Ansible Galaxy MANIFEST.json',
|
|
654
727
|
);
|
|
655
728
|
|
|
656
|
-
// Verify FILES.json checksum
|
|
657
|
-
const filesJsonPath = join(
|
|
658
|
-
collectionInfo.path,
|
|
659
|
-
collectionInfo.name.split('.')[0],
|
|
660
|
-
collectionInfo.name.split('.')[1],
|
|
661
|
-
'FILES.json',
|
|
662
|
-
);
|
|
663
|
-
|
|
664
729
|
const expectedFilesChecksum = manifest.file_manifest_file?.chksum_sha256;
|
|
665
730
|
if (!expectedFilesChecksum) {
|
|
666
|
-
//
|
|
667
|
-
|
|
731
|
+
// Was `return true`. Stripping `file_manifest_file` from the manifest was
|
|
732
|
+
// enough to report a tampered collection as verified.
|
|
733
|
+
return {
|
|
734
|
+
status: 'unverifiable',
|
|
735
|
+
reason: 'MANIFEST.json declares no FILES.json checksum',
|
|
736
|
+
};
|
|
668
737
|
}
|
|
669
738
|
|
|
670
|
-
const
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
739
|
+
const filesJsonPath = join(dir, 'FILES.json');
|
|
740
|
+
switch (await verifyFileChecksum(filesJsonPath, expectedFilesChecksum)) {
|
|
741
|
+
case 'mismatch':
|
|
742
|
+
return { status: 'mismatch', detail: 'FILES.json does not match its manifest checksum' };
|
|
743
|
+
case 'absent':
|
|
744
|
+
return { status: 'unverifiable', reason: 'FILES.json is missing or unreadable' };
|
|
674
745
|
}
|
|
675
746
|
|
|
676
|
-
// Sample verify a few files from FILES.json
|
|
677
|
-
const filesContent = await readFile(filesJsonPath, 'utf-8');
|
|
678
747
|
const filesData = parseJsonWithValidation(
|
|
679
|
-
|
|
748
|
+
await readFile(filesJsonPath, 'utf-8'),
|
|
680
749
|
GalaxyFilesSchema,
|
|
681
750
|
'Ansible Galaxy FILES.json',
|
|
682
751
|
);
|
|
683
752
|
const files = filesData.files.filter(
|
|
684
753
|
(f: { ftype: string; chksum_sha256?: string }) => f.ftype === 'file' && f.chksum_sha256,
|
|
685
754
|
);
|
|
755
|
+
if (files.length === 0) {
|
|
756
|
+
return { status: 'unverifiable', reason: 'FILES.json lists no checksummed files' };
|
|
757
|
+
}
|
|
686
758
|
|
|
687
|
-
//
|
|
759
|
+
// Sample rather than verify everything: a large collection is thousands of
|
|
760
|
+
// files and this runs on every deploy. Sampling bounds the cost; what it
|
|
761
|
+
// must never do is report a sample of ZERO as a pass.
|
|
688
762
|
let verified = 0;
|
|
689
763
|
let attempts = 0;
|
|
690
764
|
const maxAttempts = Math.min(20, files.length);
|
|
691
765
|
|
|
692
766
|
while (verified < 5 && attempts < maxAttempts) {
|
|
767
|
+
attempts++;
|
|
693
768
|
const file = files[Math.floor(Math.random() * files.length)];
|
|
769
|
+
if (!file.chksum_sha256) continue;
|
|
694
770
|
|
|
695
|
-
|
|
696
|
-
if (
|
|
697
|
-
|
|
698
|
-
continue;
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
const filePath = join(
|
|
702
|
-
collectionInfo.path,
|
|
703
|
-
collectionInfo.name.split('.')[0],
|
|
704
|
-
collectionInfo.name.split('.')[1],
|
|
705
|
-
file.name,
|
|
706
|
-
);
|
|
707
|
-
|
|
708
|
-
const fileChecksumValid = await verifyFileChecksum(filePath, file.chksum_sha256);
|
|
709
|
-
|
|
710
|
-
if (fileChecksumValid === false) {
|
|
711
|
-
// File exists but checksum mismatch - this is a real error
|
|
712
|
-
console.warn(` ⚠ Checksum mismatch for ${file.name} in ${collectionInfo.name}`);
|
|
713
|
-
return false;
|
|
771
|
+
const result = await verifyFileChecksum(join(dir, file.name), file.chksum_sha256);
|
|
772
|
+
if (result === 'mismatch') {
|
|
773
|
+
return { status: 'mismatch', detail: `${file.name} does not match its checksum` };
|
|
714
774
|
}
|
|
775
|
+
if (result === 'match') verified++;
|
|
776
|
+
// 'absent' — try another. Bounded by maxAttempts, and a run that finds
|
|
777
|
+
// nothing present is reported as unverifiable below, not as a pass.
|
|
778
|
+
}
|
|
715
779
|
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
attempts++;
|
|
780
|
+
if (verified === 0) {
|
|
781
|
+
// Was `return true`. A collection whose every listed file was missing
|
|
782
|
+
// from disk verified clean.
|
|
783
|
+
return {
|
|
784
|
+
status: 'unverifiable',
|
|
785
|
+
reason: `none of the ${attempts} sampled file(s) were present on disk`,
|
|
786
|
+
};
|
|
724
787
|
}
|
|
725
788
|
|
|
726
|
-
return
|
|
727
|
-
} catch {
|
|
728
|
-
//
|
|
729
|
-
|
|
789
|
+
return { status: 'verified', filesChecked: verified };
|
|
790
|
+
} catch (error) {
|
|
791
|
+
// Was `return true` behind a comment claiming it logged, which it did not —
|
|
792
|
+
// so ANY exception in the body above (unreadable manifest, schema
|
|
793
|
+
// violation, malformed JSON) was reported as integrity verified. Rule 6.2:
|
|
794
|
+
// the caller renders this, and it is now a distinct outcome from success.
|
|
795
|
+
return {
|
|
796
|
+
status: 'unverifiable',
|
|
797
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
798
|
+
};
|
|
730
799
|
}
|
|
731
800
|
}
|
|
732
801
|
|
|
@@ -778,10 +847,11 @@ export async function installAnsibleCollections(
|
|
|
778
847
|
const freshInstalled = await getInstalledCollections();
|
|
779
848
|
const installedCollection = freshInstalled.get(req.name);
|
|
780
849
|
if (installedCollection) {
|
|
781
|
-
const
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
850
|
+
const refusal = reportIntegrity(
|
|
851
|
+
req.name,
|
|
852
|
+
await verifyCollectionIntegrity(installedCollection),
|
|
853
|
+
);
|
|
854
|
+
if (refusal) return { success: false, installed, skipped, error: refusal };
|
|
785
855
|
}
|
|
786
856
|
} catch (installError) {
|
|
787
857
|
// Installation failed
|
|
@@ -800,10 +870,8 @@ export async function installAnsibleCollections(
|
|
|
800
870
|
skipped.push(`${req.name} ${existingVersionStr}`);
|
|
801
871
|
|
|
802
872
|
// Verify integrity of existing collection
|
|
803
|
-
const
|
|
804
|
-
if (
|
|
805
|
-
console.warn(` ⚠ Integrity check failed for ${req.name} (non-fatal)`);
|
|
806
|
-
}
|
|
873
|
+
const refusal = reportIntegrity(req.name, await verifyCollectionIntegrity(existing));
|
|
874
|
+
if (refusal) return { success: false, installed, skipped, error: refusal };
|
|
807
875
|
}
|
|
808
876
|
}
|
|
809
877
|
|
|
@@ -94,9 +94,20 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
|
|
|
94
94
|
|
|
95
95
|
// The reason escalation declined is the single most useful fact when someone
|
|
96
96
|
// asks "why was I not paged", so name it rather than aggregating it away.
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
97
|
+
if (report.skipped.length > 0) {
|
|
98
|
+
// Aggregate first — the shape of a sweep at a glance — then name each alert.
|
|
99
|
+
// A bare `within_grace×2` does not answer "why was I not paged" for the
|
|
100
|
+
// alert someone is actually looking at, and that question is the whole
|
|
101
|
+
// reason the reason is reported at all (#450).
|
|
102
|
+
const byReason = new Map<string, number>();
|
|
103
|
+
for (const { reason } of report.skipped) {
|
|
104
|
+
byReason.set(reason, (byReason.get(reason) ?? 0) + 1);
|
|
105
|
+
}
|
|
106
|
+
const counts = [...byReason].sort(([, a], [, b]) => b - a);
|
|
107
|
+
lines.push(` not delivered: ${counts.map(([r, n]) => `${r}×${n}`).join(', ')}`);
|
|
108
|
+
for (const { alertKey, reason } of report.skipped) {
|
|
109
|
+
lines.push(` ${alertKey} (${reason})`);
|
|
110
|
+
}
|
|
100
111
|
}
|
|
101
112
|
// The error itself, not just a count: the transport is loaded lazily inside
|
|
102
113
|
// the send, so a capability that will not load produces no other record
|
|
@@ -4,7 +4,11 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import type { NetworkZone } from '../../db/schema';
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
type MachineFilters,
|
|
9
|
+
getModulesOnMachine,
|
|
10
|
+
listMachines,
|
|
11
|
+
} from '../../services/machine-pool';
|
|
8
12
|
import { celiloIntro } from '../prompts';
|
|
9
13
|
import type { CommandResult } from '../types';
|
|
10
14
|
|
|
@@ -38,9 +42,11 @@ export async function handleMachineList(
|
|
|
38
42
|
|
|
39
43
|
console.log('');
|
|
40
44
|
for (const machine of machines) {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
45
|
+
// Derived, not a stored snapshot (celilo#773): this line reported
|
|
46
|
+
// "None (available)" for a machine that was in fact hosting a VERIFIED
|
|
47
|
+
// module, which is the one place an operator would look to check.
|
|
48
|
+
const occupants = getModulesOnMachine(machine.id);
|
|
49
|
+
const assignedText = occupants.length === 0 ? 'None (available)' : occupants.join(', ');
|
|
44
50
|
|
|
45
51
|
const roleLabel = machine.role === 'router' ? ' [router]' : '';
|
|
46
52
|
console.log(`${machine.hostname} (${machine.zone})${roleLabel}`);
|
|
@@ -4,7 +4,12 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { askConfirm, withInterviewSession } from '../../services/bus-interview';
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
getMachineByHostname,
|
|
9
|
+
getMachineByIp,
|
|
10
|
+
getModulesOnMachine,
|
|
11
|
+
removeMachine,
|
|
12
|
+
} from '../../services/machine-pool';
|
|
8
13
|
import { celiloIntro, celiloOutro } from '../prompts';
|
|
9
14
|
import type { CommandResult } from '../types';
|
|
10
15
|
|
|
@@ -43,12 +48,13 @@ export async function handleMachineRemove(
|
|
|
43
48
|
}
|
|
44
49
|
const hostname = machine.hostname;
|
|
45
50
|
|
|
46
|
-
//
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
51
|
+
// Derived from the same source placement uses (celilo#773), so the two
|
|
52
|
+
// cannot disagree. Previously this refused to remove an empty machine over
|
|
53
|
+
// a module that no longer existed, and let an occupied one be removed.
|
|
54
|
+
const occupants = getModulesOnMachine(machine.id);
|
|
55
|
+
if (occupants.length > 0) {
|
|
56
|
+
console.log(`\nError: Machine '${hostname}' has ${occupants.length} assigned module(s):`);
|
|
57
|
+
for (const moduleId of occupants) {
|
|
52
58
|
console.log(` - ${moduleId}`);
|
|
53
59
|
}
|
|
54
60
|
console.log('\nModules must be unassigned or shut down before removing the machine.\n');
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { detectMachineInfo, testSshConnection } from '../../services/machine-detector';
|
|
7
|
-
import { getMachineByHostname,
|
|
7
|
+
import { getMachineByHostname, getModulesOnMachine } from '../../services/machine-pool';
|
|
8
8
|
import { ManagedSshKey } from '../../services/ssh-key-manager';
|
|
9
9
|
import { celiloIntro } from '../prompts';
|
|
10
10
|
import type { CommandResult } from '../types';
|
|
@@ -57,21 +57,19 @@ export async function handleMachineStatus(
|
|
|
57
57
|
|
|
58
58
|
console.log('Assigned Modules');
|
|
59
59
|
console.log('───────────────');
|
|
60
|
-
|
|
60
|
+
const occupants = getModulesOnMachine(machine.id);
|
|
61
|
+
if (occupants.length === 0) {
|
|
61
62
|
console.log('None (available)');
|
|
62
63
|
} else {
|
|
63
|
-
for (const moduleId of
|
|
64
|
+
for (const moduleId of occupants) {
|
|
64
65
|
console.log(` - ${moduleId}`);
|
|
65
66
|
}
|
|
66
67
|
|
|
67
|
-
//
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
console.log(`CPU: ${allocated.cpu} / ${machine.hardware.cpu_cores} cores`);
|
|
73
|
-
console.log(`Memory: ${allocated.memory} / ${machine.hardware.memory_mb} MB`);
|
|
74
|
-
console.log(`Disk: ${allocated.disk} / ${machine.hardware.disk_gb} GB`);
|
|
68
|
+
// No "Resource Allocation" block (celilo#773). It printed
|
|
69
|
+
// `0 / <total>` for every resource on every machine, because its source
|
|
70
|
+
// returned hard-coded zeros behind a TODO — an operator reading it would
|
|
71
|
+
// conclude a fully-committed box was entirely free. The machine's own
|
|
72
|
+
// hardware is already printed above; that part is real.
|
|
75
73
|
}
|
|
76
74
|
console.log('');
|
|
77
75
|
|
|
@@ -14,7 +14,7 @@ import { capabilities as capabilitiesTable, moduleInfrastructure, modules } from
|
|
|
14
14
|
import { createGaugeLogger } from '../../hooks/logger';
|
|
15
15
|
import { runNamedHook } from '../../hooks/run-named-hook';
|
|
16
16
|
import { deallocateForModule } from '../../ipam/auto-allocator';
|
|
17
|
-
import { ModuleManifestSchema } from '../../manifest/schema';
|
|
17
|
+
import { type ModuleManifest, ModuleManifestSchema } from '../../manifest/schema';
|
|
18
18
|
import { executeBuildWithProgress } from '../../services/build-stream';
|
|
19
19
|
import { askConfirm, withInterviewSession } from '../../services/bus-interview';
|
|
20
20
|
import {
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
emitUninstallFailed,
|
|
23
23
|
emitUninstallStarted,
|
|
24
24
|
} from '../../services/celilo-events';
|
|
25
|
+
import { loadConsumerCleanupPlan, runConsumerCleanup } from '../../services/consumer-cleanup';
|
|
25
26
|
import { getContainerService, getServiceCredentials } from '../../services/container-service';
|
|
26
27
|
import { completeOperation, failOperation, startOperation } from '../../services/module-operations';
|
|
27
28
|
import {
|
|
@@ -29,7 +30,6 @@ import {
|
|
|
29
30
|
describeRemovalRefusal,
|
|
30
31
|
findRemovalBlockers,
|
|
31
32
|
} from '../../services/remove-guard';
|
|
32
|
-
import { cleanupWebRoutesForModule } from '../../services/web-route-cleanup';
|
|
33
33
|
import { getArg, hasFlag, validateRequiredArgs } from '../parser';
|
|
34
34
|
import { log } from '../prompts';
|
|
35
35
|
import type { CommandResult } from '../types';
|
|
@@ -243,28 +243,31 @@ async function performModuleRemove(
|
|
|
243
243
|
}
|
|
244
244
|
}
|
|
245
245
|
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
-
//
|
|
246
|
+
// Tell every provider whose capability this module consumed that it is
|
|
247
|
+
// leaving, so each withdraws what it minted on its behalf
|
|
248
|
+
// (openspec/changes/consumer-removal-cleanup). BEFORE terraform destroy, so
|
|
249
|
+
// provider hosts are still reachable; AFTER on_uninstall, so a module that
|
|
250
|
+
// tears its own state down first still wins.
|
|
249
251
|
//
|
|
250
|
-
//
|
|
251
|
-
//
|
|
252
|
-
//
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
252
|
+
// This replaces the by-name `cleanupWebRoutesForModule` call, which did the
|
|
253
|
+
// same job for exactly one capability. A failed withdrawal never blocks the
|
|
254
|
+
// removal — the failing PROVIDER is marked ERROR instead (D6).
|
|
255
|
+
const cleanupLogger = {
|
|
256
|
+
info: (m: string) => log.info(m),
|
|
257
|
+
warn: (m: string) => log.warn(m),
|
|
258
|
+
error: (m: string) => log.warn(m),
|
|
259
|
+
success: (m: string) => log.info(m),
|
|
260
|
+
};
|
|
261
|
+
const cleanup = await runConsumerCleanup(
|
|
262
|
+
moduleId,
|
|
263
|
+
loadConsumerCleanupPlan(moduleId, module.manifestData as ModuleManifest, db),
|
|
264
|
+
db,
|
|
265
|
+
cleanupLogger,
|
|
266
|
+
);
|
|
267
|
+
if (cleanup.failures.length > 0) {
|
|
268
|
+
log.warn(
|
|
269
|
+
`${cleanup.failures.length} provider(s) could not withdraw state for '${moduleId}' and are now marked ERROR: ${cleanup.failures.map((f) => f.providerId).join(', ')}. Removal continues; run \`celilo system audit\` to see what each is holding.`,
|
|
270
|
+
);
|
|
268
271
|
}
|
|
269
272
|
|
|
270
273
|
// Check if module has infrastructure that needs to be destroyed
|
|
@@ -371,7 +371,11 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
|
|
|
371
371
|
backups: { modules: installedBackupInfo },
|
|
372
372
|
abandonedOperations: { records: loadAbandonedOperations(db) },
|
|
373
373
|
undeployedModules: {
|
|
374
|
-
modules: installed.map((m) => ({
|
|
374
|
+
modules: installed.map((m) => ({
|
|
375
|
+
id: m.id,
|
|
376
|
+
state: m.state,
|
|
377
|
+
errorMessage: m.errorMessage,
|
|
378
|
+
})),
|
|
375
379
|
},
|
|
376
380
|
unconfiguredModules: {
|
|
377
381
|
modules: installed.map((m) => ({
|
|
@@ -573,7 +573,11 @@ export async function handleSystemUpdate(
|
|
|
573
573
|
},
|
|
574
574
|
abandonedOperations: { records: loadAbandonedOperations(db) },
|
|
575
575
|
undeployedModules: {
|
|
576
|
-
modules: installed.map((m) => ({
|
|
576
|
+
modules: installed.map((m) => ({
|
|
577
|
+
id: m.id,
|
|
578
|
+
state: m.state,
|
|
579
|
+
errorMessage: m.errorMessage,
|
|
580
|
+
})),
|
|
577
581
|
},
|
|
578
582
|
unconfiguredModules: {
|
|
579
583
|
modules: installed.map((m) => ({
|
|
@@ -801,7 +805,11 @@ export function rebuildAuditDepsForRerun(
|
|
|
801
805
|
})),
|
|
802
806
|
},
|
|
803
807
|
undeployedModules: {
|
|
804
|
-
modules: installed.map((m) => ({
|
|
808
|
+
modules: installed.map((m) => ({
|
|
809
|
+
id: m.id,
|
|
810
|
+
state: m.state,
|
|
811
|
+
errorMessage: m.errorMessage,
|
|
812
|
+
})),
|
|
805
813
|
},
|
|
806
814
|
unconfiguredModules: {
|
|
807
815
|
modules: installed.map((m) => ({
|
package/src/db/schema.ts
CHANGED
|
@@ -428,10 +428,12 @@ export const machines = sqliteTable('machines', {
|
|
|
428
428
|
.$type<Array<{ name: string; ipAddress: string; zone: string }>>()
|
|
429
429
|
.notNull()
|
|
430
430
|
.default(sql`'[]'`),
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
431
|
+
// No `assigned_module_ids` (celilo#773). Occupancy is derived from
|
|
432
|
+
// `module_infrastructure` / `module_systems` at the point of use — both are
|
|
433
|
+
// written by the deploy path and both cascade on module removal, so a machine
|
|
434
|
+
// frees itself. The dropped column had one append-only writer, no removal
|
|
435
|
+
// path, and no reader that reconciled it, and it had already diverged in both
|
|
436
|
+
// directions on the live fleet.
|
|
435
437
|
/** Module ID this machine is earmarked for. If set, only this module can use this machine. */
|
|
436
438
|
earmarkedModule: text('earmarked_module'),
|
|
437
439
|
/**
|
|
@@ -589,19 +591,34 @@ export const portForwards = sqliteTable(
|
|
|
589
591
|
/** Dedicated INTERNAL ingress IP (ISS-0156); NULL for the normal public path. */
|
|
590
592
|
ingressIp: text('ingress_ip'),
|
|
591
593
|
description: text('description').notNull().default(''),
|
|
594
|
+
/**
|
|
595
|
+
* The CONSUMER that registered it, stamped by the store from the calling
|
|
596
|
+
* module (openspec/changes/consumer-removal-cleanup, D5a). `''` for rows
|
|
597
|
+
* written before this column existed — unattributed, so no consumer removal
|
|
598
|
+
* withdraws them.
|
|
599
|
+
*/
|
|
600
|
+
registeredBy: text('registered_by').notNull().default(''),
|
|
592
601
|
createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
593
602
|
},
|
|
594
603
|
(table) => ({
|
|
595
|
-
// One row per (firewall, backend, port, protocol, ingress). The
|
|
596
|
-
//
|
|
597
|
-
//
|
|
598
|
-
//
|
|
604
|
+
// One row per (OWNER, firewall, backend, port, protocol, ingress). The
|
|
605
|
+
// owner is IN the index, not merely alongside it, and that is the whole
|
|
606
|
+
// point: without it, consumer B exposing a forward A already has silently
|
|
607
|
+
// REPLACES A's row, and B leaving then deletes a rule A still needs. Two
|
|
608
|
+
// owners of one forward are two rows — the set, denormalised into the
|
|
609
|
+
// table that already exists rather than a second table. `renderRuleset`
|
|
610
|
+
// dedupes on the rule tuple so the duplicate renders once.
|
|
611
|
+
//
|
|
612
|
+
// This is the `dns_registration_consumers` lesson (see below): a single
|
|
613
|
+
// overwritten owner column is not a label, because what cascades on it
|
|
614
|
+
// decides when a LIVE record is forgotten.
|
|
599
615
|
forwardUnique: uniqueIndex('port_forwards_unique_idx').on(
|
|
600
616
|
table.firewallIp,
|
|
601
617
|
table.internalIp,
|
|
602
618
|
table.port,
|
|
603
619
|
table.protocol,
|
|
604
620
|
table.ingressIp,
|
|
621
|
+
table.registeredBy,
|
|
605
622
|
),
|
|
606
623
|
}),
|
|
607
624
|
);
|
|
@@ -630,11 +647,14 @@ export const trustedSources = sqliteTable(
|
|
|
630
647
|
createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
631
648
|
},
|
|
632
649
|
(table) => ({
|
|
633
|
-
// One row per (firewall, subnet) — the
|
|
634
|
-
//
|
|
650
|
+
// One row per (OWNER, firewall, subnet) — same reason the owner is in
|
|
651
|
+
// `port_forwards_unique_idx`. Two modules trusting the same subnet are two
|
|
652
|
+
// rows, so one leaving does not revoke the other's reach. The renderer
|
|
653
|
+
// already dedupes trusted subnets (`new Set`), so the pair renders once.
|
|
635
654
|
trustedSourceUnique: uniqueIndex('trusted_sources_unique_idx').on(
|
|
636
655
|
table.firewallIp,
|
|
637
656
|
table.subnet,
|
|
657
|
+
table.registeredBy,
|
|
638
658
|
),
|
|
639
659
|
}),
|
|
640
660
|
);
|