@celilo/cli 0.26.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CELILO_CORE_MODULES.md +3 -0
  2. package/CELILO_SUBSYSTEMS.md +4 -2
  3. package/drizzle/0025_port_forward_owner.sql +29 -0
  4. package/drizzle/meta/_journal.json +7 -0
  5. package/package.json +3 -3
  6. package/src/__integration__/container-services-cli.integration.test.ts +0 -4
  7. package/src/ansible/dependencies.test.ts +233 -289
  8. package/src/ansible/dependencies.ts +151 -83
  9. package/src/cli/commands/alerts-sweep.ts +14 -3
  10. package/src/cli/commands/machine-add.ts +0 -1
  11. package/src/cli/commands/machine-list.ts +10 -4
  12. package/src/cli/commands/machine-remove.ts +13 -7
  13. package/src/cli/commands/machine-status.ts +9 -11
  14. package/src/cli/commands/module-remove.ts +26 -23
  15. package/src/cli/commands/system-audit.ts +5 -1
  16. package/src/cli/commands/system-update.ts +10 -2
  17. package/src/db/schema.ts +30 -10
  18. package/src/hooks/capability-loader.ts +65 -13
  19. package/src/hooks/define-hook.test.ts +4 -6
  20. package/src/hooks/executor.ts +2 -1
  21. package/src/hooks/types.ts +9 -17
  22. package/src/infrastructure/property-extractor.test.ts +0 -2
  23. package/src/manifest/contracts/index.ts +20 -0
  24. package/src/manifest/contracts/v1.ts +33 -1
  25. package/src/manifest/schema.ts +48 -58
  26. package/src/services/alerting/sweep-runner.test.ts +5 -1
  27. package/src/services/alerting/sweep-runner.ts +14 -8
  28. package/src/services/aspect-runner.test.ts +0 -1
  29. package/src/services/audit/undeployed-modules.ts +18 -1
  30. package/src/services/consumer-cleanup.test.ts +347 -0
  31. package/src/services/consumer-cleanup.ts +244 -0
  32. package/src/services/infrastructure-selector.test.ts +0 -7
  33. package/src/services/infrastructure-selector.ts +24 -25
  34. package/src/services/infrastructure-variable-resolver.test.ts +0 -6
  35. package/src/services/infrastructure-variable-resolver.ts +0 -3
  36. package/src/services/machine-pool.test.ts +53 -85
  37. package/src/services/machine-pool.ts +68 -84
  38. package/src/services/module-deploy.ts +17 -39
  39. package/src/services/module-validator/index.test.ts +9 -0
  40. package/src/services/port-forwards.test.ts +93 -40
  41. package/src/services/port-forwards.ts +74 -48
  42. package/src/services/ssh-key-manager.test.ts +0 -10
  43. package/src/services/trusted-sources.test.ts +52 -13
  44. package/src/services/trusted-sources.ts +25 -15
  45. package/src/test-utils/cli-context.ts +15 -2
  46. package/src/types/infrastructure.ts +11 -1
  47. package/src/services/web-route-cleanup.test.ts +0 -250
  48. package/src/services/web-route-cleanup.ts +0 -144
@@ -575,7 +575,17 @@ export async function getInstalledCollections(): Promise<Map<string, InstalledCo
575
575
  version: parseSemanticVersion(versionStr),
576
576
  path: path as string,
577
577
  });
578
- } catch {}
578
+ } catch (error) {
579
+ // Rule 6.2: never a bare catch. An unparseable version dropped the
580
+ // collection from the map entirely, so it read as NOT INSTALLED —
581
+ // and the installer would then try to install it again, every time,
582
+ // reporting success while nothing changed.
583
+ console.warn(
584
+ ` ⚠ Ignoring installed collection '${name}': unparseable version '${versionStr}' (${
585
+ error instanceof Error ? error.message : String(error)
586
+ })`,
587
+ );
588
+ }
579
589
  }
580
590
  }
581
591
  }
@@ -587,6 +597,41 @@ export async function getInstalledCollections(): Promise<Map<string, InstalledCo
587
597
  }
588
598
  }
589
599
 
600
+ /**
601
+ * Decide what an integrity outcome means for the import, and say so.
602
+ *
603
+ * A REFUTED collection stops the import; an UNCHECKABLE one warns. The split is
604
+ * the point (celilo#524):
605
+ *
606
+ * - `mismatch` — files on disk do not match the collection's own manifest.
607
+ * Ansible is about to execute that content, and celilo cannot account for
608
+ * it. Refusing costs an import; accepting runs unaccounted-for code.
609
+ * - `unverifiable` — the check could not run, most often because the
610
+ * collection ships no `file_manifest_file` checksum at all. That is a
611
+ * property of the publisher, not evidence of tampering, and blocking on it
612
+ * would refuse ordinary collections forever. It warns.
613
+ *
614
+ * Refusing is cheap HERE specifically. This runs during `module import`
615
+ * (`module/import.ts`), the only caller, and nothing installs collections in
616
+ * the deploy path — so a false positive means "this module did not import",
617
+ * not "the fleet stopped deploying". There is also no module row yet, so
618
+ * nothing is left half-created to clean up.
619
+ *
620
+ * ⚠️ This is SELF-ATTESTATION, not provenance: the checksums live inside the
621
+ * artifact being checked. It catches corruption and post-install modification.
622
+ * It cannot catch a coherently re-signed tampered collection, and a clean
623
+ * import must not be read as saying otherwise.
624
+ */
625
+ export function reportIntegrity(name: string, outcome: IntegrityOutcome): string | null {
626
+ if (outcome.status === 'mismatch') {
627
+ return `Integrity check FAILED for Ansible collection '${name}': ${outcome.detail}.\nThe installed files do not match the collection's own manifest — this is corruption or tampering.\nImport refused rather than run unverified content.`;
628
+ }
629
+ if (outcome.status === 'unverifiable') {
630
+ console.warn(` ⚠ Integrity could not be checked for ${name}: ${outcome.reason} (non-fatal)`);
631
+ }
632
+ return null;
633
+ }
634
+
590
635
  /**
591
636
  * Install result
592
637
  */
@@ -599,16 +644,22 @@ export interface InstallResult {
599
644
  }
600
645
 
601
646
  /**
602
- * Verify checksum of a file
647
+ * The three things checking one file's checksum can tell you.
603
648
  *
604
- * @param filePath - Path to file
605
- * @param expectedChecksum - Expected SHA256 checksum
606
- * @returns True if checksum matches, null if file doesn't exist
607
- */
608
- async function verifyFileChecksum(
609
- filePath: string,
610
- expectedChecksum: string,
611
- ): Promise<boolean | null> {
649
+ * Named states rather than `boolean | null` (celilo#524). The tri-state was
650
+ * right a file that is not on disk is genuinely neither a match nor a
651
+ * mismatch but spelled as `null` it read as "nothing to worry about", and
652
+ * every caller duly treated it as one. `absent` cannot be misread.
653
+ */
654
+ export type FileCheck = 'match' | 'mismatch' | 'absent';
655
+
656
+ /**
657
+ * Verify one file against its expected SHA-256.
658
+ *
659
+ * `absent` means the file could not be read at all, which is a statement about
660
+ * our knowledge, not about the file's integrity.
661
+ */
662
+ async function verifyFileChecksum(filePath: string, expectedChecksum: string): Promise<FileCheck> {
612
663
  const { createHash } = await import('node:crypto');
613
664
  const { readFile, access } = await import('node:fs/promises');
614
665
  const { constants } = await import('node:fs');
@@ -621,112 +672,130 @@ async function verifyFileChecksum(
621
672
  const hash = createHash('sha256');
622
673
  hash.update(content);
623
674
  const actualChecksum = hash.digest('hex');
624
- return actualChecksum === expectedChecksum;
675
+ return actualChecksum === expectedChecksum ? 'match' : 'mismatch';
625
676
  } catch (_error: unknown) {
626
- // File doesn't exist or can't be read - this is OK, not all files are installed
627
- return null;
677
+ // Unreadable. NOT evidence of integrity see the type's note.
678
+ return 'absent';
628
679
  }
629
680
  }
630
681
 
631
682
  /**
632
- * Verify integrity of installed collection
683
+ * What a verification run actually established.
684
+ *
685
+ * Three states, because there are three (celilo#524). The function used to
686
+ * return `boolean` and could not, in practice, return `false`: three
687
+ * independent paths reported "integrity verified" for a collection that had
688
+ * been tampered with, and no constructible input reached the failure return.
689
+ *
690
+ * `unverifiable` is the state that was missing. A manifest that declares no
691
+ * checksum, or a file set with nothing on disk to sample, tells you NOTHING
692
+ * about integrity — and folding that into `verified` is what let the one thing
693
+ * an attacker fully controls (the manifest they ship) switch the check off.
694
+ * Same absent-vs-empty distinction as `parseInterfaceBaseline`.
695
+ */
696
+ export type IntegrityOutcome =
697
+ | { status: 'verified'; filesChecked: number }
698
+ /** Something was checked and did not match. Integrity is refuted. */
699
+ | { status: 'mismatch'; detail: string }
700
+ /** Nothing could be checked. Integrity is unknown — NOT confirmed. */
701
+ | { status: 'unverifiable'; reason: string };
702
+
703
+ /**
704
+ * Verify an installed Galaxy collection against the checksums in its own
705
+ * MANIFEST.json / FILES.json.
633
706
  *
634
- * @param collectionInfo - Collection installation info
635
- * @returns True if integrity checks pass
707
+ * ⚠️ This is self-attestation, not provenance: the checksums live inside the
708
+ * artifact being checked. It detects corruption and post-install modification,
709
+ * and it CANNOT detect a coherently re-signed tampered collection. That is a
710
+ * reason to be precise about what it reports, not a reason to report a
711
+ * comforting answer.
636
712
  */
637
- async function verifyCollectionIntegrity(collectionInfo: InstalledCollection): Promise<boolean> {
713
+ export async function verifyCollectionIntegrity(
714
+ collectionInfo: InstalledCollection,
715
+ ): Promise<IntegrityOutcome> {
638
716
  const { readFile } = await import('node:fs/promises');
639
717
  const { join } = await import('node:path');
640
718
 
719
+ const [namespace, collection] = collectionInfo.name.split('.');
720
+ const dir = join(collectionInfo.path, namespace, collection);
721
+
641
722
  try {
642
- // Read MANIFEST.json
643
- const manifestPath = join(
644
- collectionInfo.path,
645
- collectionInfo.name.split('.')[0],
646
- collectionInfo.name.split('.')[1],
647
- 'MANIFEST.json',
648
- );
649
- const manifestContent = await readFile(manifestPath, 'utf-8');
650
723
  const manifest = parseJsonWithValidation(
651
- manifestContent,
724
+ await readFile(join(dir, 'MANIFEST.json'), 'utf-8'),
652
725
  GalaxyManifestSchema,
653
726
  'Ansible Galaxy MANIFEST.json',
654
727
  );
655
728
 
656
- // Verify FILES.json checksum
657
- const filesJsonPath = join(
658
- collectionInfo.path,
659
- collectionInfo.name.split('.')[0],
660
- collectionInfo.name.split('.')[1],
661
- 'FILES.json',
662
- );
663
-
664
729
  const expectedFilesChecksum = manifest.file_manifest_file?.chksum_sha256;
665
730
  if (!expectedFilesChecksum) {
666
- // No checksum to verify
667
- return true;
731
+ // Was `return true`. Stripping `file_manifest_file` from the manifest was
732
+ // enough to report a tampered collection as verified.
733
+ return {
734
+ status: 'unverifiable',
735
+ reason: 'MANIFEST.json declares no FILES.json checksum',
736
+ };
668
737
  }
669
738
 
670
- const filesChecksumValid = await verifyFileChecksum(filesJsonPath, expectedFilesChecksum);
671
- if (!filesChecksumValid) {
672
- console.warn(` ⚠ Checksum mismatch for FILES.json in ${collectionInfo.name}`);
673
- return false;
739
+ const filesJsonPath = join(dir, 'FILES.json');
740
+ switch (await verifyFileChecksum(filesJsonPath, expectedFilesChecksum)) {
741
+ case 'mismatch':
742
+ return { status: 'mismatch', detail: 'FILES.json does not match its manifest checksum' };
743
+ case 'absent':
744
+ return { status: 'unverifiable', reason: 'FILES.json is missing or unreadable' };
674
745
  }
675
746
 
676
- // Sample verify a few files from FILES.json
677
- const filesContent = await readFile(filesJsonPath, 'utf-8');
678
747
  const filesData = parseJsonWithValidation(
679
- filesContent,
748
+ await readFile(filesJsonPath, 'utf-8'),
680
749
  GalaxyFilesSchema,
681
750
  'Ansible Galaxy FILES.json',
682
751
  );
683
752
  const files = filesData.files.filter(
684
753
  (f: { ftype: string; chksum_sha256?: string }) => f.ftype === 'file' && f.chksum_sha256,
685
754
  );
755
+ if (files.length === 0) {
756
+ return { status: 'unverifiable', reason: 'FILES.json lists no checksummed files' };
757
+ }
686
758
 
687
- // Verify up to 5 random files that exist
759
+ // Sample rather than verify everything: a large collection is thousands of
760
+ // files and this runs on every deploy. Sampling bounds the cost; what it
761
+ // must never do is report a sample of ZERO as a pass.
688
762
  let verified = 0;
689
763
  let attempts = 0;
690
764
  const maxAttempts = Math.min(20, files.length);
691
765
 
692
766
  while (verified < 5 && attempts < maxAttempts) {
767
+ attempts++;
693
768
  const file = files[Math.floor(Math.random() * files.length)];
769
+ if (!file.chksum_sha256) continue;
694
770
 
695
- // Skip files without checksums
696
- if (!file.chksum_sha256) {
697
- attempts++;
698
- continue;
699
- }
700
-
701
- const filePath = join(
702
- collectionInfo.path,
703
- collectionInfo.name.split('.')[0],
704
- collectionInfo.name.split('.')[1],
705
- file.name,
706
- );
707
-
708
- const fileChecksumValid = await verifyFileChecksum(filePath, file.chksum_sha256);
709
-
710
- if (fileChecksumValid === false) {
711
- // File exists but checksum mismatch - this is a real error
712
- console.warn(` ⚠ Checksum mismatch for ${file.name} in ${collectionInfo.name}`);
713
- return false;
771
+ const result = await verifyFileChecksum(join(dir, file.name), file.chksum_sha256);
772
+ if (result === 'mismatch') {
773
+ return { status: 'mismatch', detail: `${file.name} does not match its checksum` };
714
774
  }
775
+ if (result === 'match') verified++;
776
+ // 'absent' — try another. Bounded by maxAttempts, and a run that finds
777
+ // nothing present is reported as unverifiable below, not as a pass.
778
+ }
715
779
 
716
- if (fileChecksumValid === true) {
717
- // File exists and checksum matches
718
- verified++;
719
- }
720
-
721
- // If fileChecksumValid === null, file doesn't exist, try another
722
-
723
- attempts++;
780
+ if (verified === 0) {
781
+ // Was `return true`. A collection whose every listed file was missing
782
+ // from disk verified clean.
783
+ return {
784
+ status: 'unverifiable',
785
+ reason: `none of the ${attempts} sampled file(s) were present on disk`,
786
+ };
724
787
  }
725
788
 
726
- return true;
727
- } catch {
728
- // If verification fails, log warning but don't fail install
729
- return true;
789
+ return { status: 'verified', filesChecked: verified };
790
+ } catch (error) {
791
+ // Was `return true` behind a comment claiming it logged, which it did not
792
+ // so ANY exception in the body above (unreadable manifest, schema
793
+ // violation, malformed JSON) was reported as integrity verified. Rule 6.2:
794
+ // the caller renders this, and it is now a distinct outcome from success.
795
+ return {
796
+ status: 'unverifiable',
797
+ reason: error instanceof Error ? error.message : String(error),
798
+ };
730
799
  }
731
800
  }
732
801
 
@@ -778,10 +847,11 @@ export async function installAnsibleCollections(
778
847
  const freshInstalled = await getInstalledCollections();
779
848
  const installedCollection = freshInstalled.get(req.name);
780
849
  if (installedCollection) {
781
- const integrityOk = await verifyCollectionIntegrity(installedCollection);
782
- if (!integrityOk) {
783
- console.warn(` ⚠ Integrity check failed for ${req.name} (non-fatal)`);
784
- }
850
+ const refusal = reportIntegrity(
851
+ req.name,
852
+ await verifyCollectionIntegrity(installedCollection),
853
+ );
854
+ if (refusal) return { success: false, installed, skipped, error: refusal };
785
855
  }
786
856
  } catch (installError) {
787
857
  // Installation failed
@@ -800,10 +870,8 @@ export async function installAnsibleCollections(
800
870
  skipped.push(`${req.name} ${existingVersionStr}`);
801
871
 
802
872
  // Verify integrity of existing collection
803
- const integrityOk = await verifyCollectionIntegrity(existing);
804
- if (!integrityOk) {
805
- console.warn(` ⚠ Integrity check failed for ${req.name} (non-fatal)`);
806
- }
873
+ const refusal = reportIntegrity(req.name, await verifyCollectionIntegrity(existing));
874
+ if (refusal) return { success: false, installed, skipped, error: refusal };
807
875
  }
808
876
  }
809
877
 
@@ -94,9 +94,20 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
94
94
 
95
95
  // The reason escalation declined is the single most useful fact when someone
96
96
  // asks "why was I not paged", so name it rather than aggregating it away.
97
- const skipped = Object.entries(report.skipped).sort(([, a], [, b]) => b - a);
98
- if (skipped.length > 0) {
99
- lines.push(` not delivered: ${skipped.map(([r, n]) => `${r}×${n}`).join(', ')}`);
97
+ if (report.skipped.length > 0) {
98
+ // Aggregate first the shape of a sweep at a glance — then name each alert.
99
+ // A bare `within_grace×2` does not answer "why was I not paged" for the
100
+ // alert someone is actually looking at, and that question is the whole
101
+ // reason the reason is reported at all (#450).
102
+ const byReason = new Map<string, number>();
103
+ for (const { reason } of report.skipped) {
104
+ byReason.set(reason, (byReason.get(reason) ?? 0) + 1);
105
+ }
106
+ const counts = [...byReason].sort(([, a], [, b]) => b - a);
107
+ lines.push(` not delivered: ${counts.map(([r, n]) => `${r}×${n}`).join(', ')}`);
108
+ for (const { alertKey, reason } of report.skipped) {
109
+ lines.push(` ${alertKey} (${reason})`);
110
+ }
100
111
  }
101
112
  // The error itself, not just a count: the transport is loaded lazily inside
102
113
  // the send, so a capability that will not load produces no other record
@@ -298,7 +298,6 @@ export async function handleMachineAdd(
298
298
  hardware: detectedInfo.hardware,
299
299
  role,
300
300
  interfaces,
301
- assignedModuleIds: [],
302
301
  earmarkedModule: earmark || null,
303
302
  });
304
303
 
@@ -4,7 +4,11 @@
4
4
  */
5
5
 
6
6
  import type { NetworkZone } from '../../db/schema';
7
- import { type MachineFilters, listMachines } from '../../services/machine-pool';
7
+ import {
8
+ type MachineFilters,
9
+ getModulesOnMachine,
10
+ listMachines,
11
+ } from '../../services/machine-pool';
8
12
  import { celiloIntro } from '../prompts';
9
13
  import type { CommandResult } from '../types';
10
14
 
@@ -38,9 +42,11 @@ export async function handleMachineList(
38
42
 
39
43
  console.log('');
40
44
  for (const machine of machines) {
41
- const assignedCount = machine.assignedModuleIds.length;
42
- const assignedText =
43
- assignedCount === 0 ? 'None (available)' : machine.assignedModuleIds.join(', ');
45
+ // Derived, not a stored snapshot (celilo#773): this line reported
46
+ // "None (available)" for a machine that was in fact hosting a VERIFIED
47
+ // module, which is the one place an operator would look to check.
48
+ const occupants = getModulesOnMachine(machine.id);
49
+ const assignedText = occupants.length === 0 ? 'None (available)' : occupants.join(', ');
44
50
 
45
51
  const roleLabel = machine.role === 'router' ? ' [router]' : '';
46
52
  console.log(`${machine.hostname} (${machine.zone})${roleLabel}`);
@@ -4,7 +4,12 @@
4
4
  */
5
5
 
6
6
  import { askConfirm, withInterviewSession } from '../../services/bus-interview';
7
- import { getMachineByHostname, getMachineByIp, removeMachine } from '../../services/machine-pool';
7
+ import {
8
+ getMachineByHostname,
9
+ getMachineByIp,
10
+ getModulesOnMachine,
11
+ removeMachine,
12
+ } from '../../services/machine-pool';
8
13
  import { celiloIntro, celiloOutro } from '../prompts';
9
14
  import type { CommandResult } from '../types';
10
15
 
@@ -43,12 +48,13 @@ export async function handleMachineRemove(
43
48
  }
44
49
  const hostname = machine.hostname;
45
50
 
46
- // Check for assigned modules
47
- if (machine.assignedModuleIds.length > 0) {
48
- console.log(
49
- `\nError: Machine '${hostname}' has ${machine.assignedModuleIds.length} assigned module(s):`,
50
- );
51
- for (const moduleId of machine.assignedModuleIds) {
51
+ // Derived from the same source placement uses (celilo#773), so the two
52
+ // cannot disagree. Previously this refused to remove an empty machine over
53
+ // a module that no longer existed, and let an occupied one be removed.
54
+ const occupants = getModulesOnMachine(machine.id);
55
+ if (occupants.length > 0) {
56
+ console.log(`\nError: Machine '${hostname}' has ${occupants.length} assigned module(s):`);
57
+ for (const moduleId of occupants) {
52
58
  console.log(` - ${moduleId}`);
53
59
  }
54
60
  console.log('\nModules must be unassigned or shut down before removing the machine.\n');
@@ -4,7 +4,7 @@
4
4
  */
5
5
 
6
6
  import { detectMachineInfo, testSshConnection } from '../../services/machine-detector';
7
- import { getMachineByHostname, getModuleResourcesOnMachine } from '../../services/machine-pool';
7
+ import { getMachineByHostname, getModulesOnMachine } from '../../services/machine-pool';
8
8
  import { ManagedSshKey } from '../../services/ssh-key-manager';
9
9
  import { celiloIntro } from '../prompts';
10
10
  import type { CommandResult } from '../types';
@@ -57,21 +57,19 @@ export async function handleMachineStatus(
57
57
 
58
58
  console.log('Assigned Modules');
59
59
  console.log('───────────────');
60
- if (machine.assignedModuleIds.length === 0) {
60
+ const occupants = getModulesOnMachine(machine.id);
61
+ if (occupants.length === 0) {
61
62
  console.log('None (available)');
62
63
  } else {
63
- for (const moduleId of machine.assignedModuleIds) {
64
+ for (const moduleId of occupants) {
64
65
  console.log(` - ${moduleId}`);
65
66
  }
66
67
 
67
- // Show resource allocation
68
- const allocated = await getModuleResourcesOnMachine(machine.id);
69
- console.log('');
70
- console.log('Resource Allocation');
71
- console.log('──────────────────');
72
- console.log(`CPU: ${allocated.cpu} / ${machine.hardware.cpu_cores} cores`);
73
- console.log(`Memory: ${allocated.memory} / ${machine.hardware.memory_mb} MB`);
74
- console.log(`Disk: ${allocated.disk} / ${machine.hardware.disk_gb} GB`);
68
+ // No "Resource Allocation" block (celilo#773). It printed
69
+ // `0 / <total>` for every resource on every machine, because its source
70
+ // returned hard-coded zeros behind a TODO — an operator reading it would
71
+ // conclude a fully-committed box was entirely free. The machine's own
72
+ // hardware is already printed above; that part is real.
75
73
  }
76
74
  console.log('');
77
75
 
@@ -14,7 +14,7 @@ import { capabilities as capabilitiesTable, moduleInfrastructure, modules } from
14
14
  import { createGaugeLogger } from '../../hooks/logger';
15
15
  import { runNamedHook } from '../../hooks/run-named-hook';
16
16
  import { deallocateForModule } from '../../ipam/auto-allocator';
17
- import { ModuleManifestSchema } from '../../manifest/schema';
17
+ import { type ModuleManifest, ModuleManifestSchema } from '../../manifest/schema';
18
18
  import { executeBuildWithProgress } from '../../services/build-stream';
19
19
  import { askConfirm, withInterviewSession } from '../../services/bus-interview';
20
20
  import {
@@ -22,6 +22,7 @@ import {
22
22
  emitUninstallFailed,
23
23
  emitUninstallStarted,
24
24
  } from '../../services/celilo-events';
25
+ import { loadConsumerCleanupPlan, runConsumerCleanup } from '../../services/consumer-cleanup';
25
26
  import { getContainerService, getServiceCredentials } from '../../services/container-service';
26
27
  import { completeOperation, failOperation, startOperation } from '../../services/module-operations';
27
28
  import {
@@ -29,7 +30,6 @@ import {
29
30
  describeRemovalRefusal,
30
31
  findRemovalBlockers,
31
32
  } from '../../services/remove-guard';
32
- import { cleanupWebRoutesForModule } from '../../services/web-route-cleanup';
33
33
  import { getArg, hasFlag, validateRequiredArgs } from '../parser';
34
34
  import { log } from '../prompts';
35
35
  import type { CommandResult } from '../types';
@@ -243,28 +243,31 @@ async function performModuleRemove(
243
243
  }
244
244
  }
245
245
 
246
- // Reclaim public_web routes BEFORE terraform destroy, so the provider host
247
- // is still reachable for the /srv/www cleanup. Runs after on_uninstall so a
248
- // module that tears its own routes down still wins (this then no-ops).
246
+ // Tell every provider whose capability this module consumed that it is
247
+ // leaving, so each withdraws what it minted on its behalf
248
+ // (openspec/changes/consumer-removal-cleanup). BEFORE terraform destroy, so
249
+ // provider hosts are still reachable; AFTER on_uninstall, so a module that
250
+ // tears its own state down first still wins.
249
251
  //
250
- // Without this, removal deleted the rows via the FK cascade on `modules.id`
251
- // and emitted nothing, so the provider kept serving a route for a module that
252
- // no longer existed. See services/web-route-cleanup.ts.
253
- try {
254
- const routeCleanup = await cleanupWebRoutesForModule({ moduleId, db });
255
- if (routeCleanup.routesRemoved > 0) {
256
- const reclaimed =
257
- routeCleanup.assetDirsRemoved.length > 0
258
- ? ` and reclaimed ${routeCleanup.assetDirsRemoved.join(', ')}`
259
- : '';
260
- log.info(`Removed ${routeCleanup.routesRemoved} web route(s) for ${moduleId}${reclaimed}`);
261
- }
262
- for (const warning of routeCleanup.warnings) {
263
- log.warn(warning);
264
- }
265
- } catch (error) {
266
- const msg = error instanceof Error ? error.message : String(error);
267
- log.warn(`Web route cleanup failed (continuing): ${msg}`);
252
+ // This replaces the by-name `cleanupWebRoutesForModule` call, which did the
253
+ // same job for exactly one capability. A failed withdrawal never blocks the
254
+ // removal the failing PROVIDER is marked ERROR instead (D6).
255
+ const cleanupLogger = {
256
+ info: (m: string) => log.info(m),
257
+ warn: (m: string) => log.warn(m),
258
+ error: (m: string) => log.warn(m),
259
+ success: (m: string) => log.info(m),
260
+ };
261
+ const cleanup = await runConsumerCleanup(
262
+ moduleId,
263
+ loadConsumerCleanupPlan(moduleId, module.manifestData as ModuleManifest, db),
264
+ db,
265
+ cleanupLogger,
266
+ );
267
+ if (cleanup.failures.length > 0) {
268
+ log.warn(
269
+ `${cleanup.failures.length} provider(s) could not withdraw state for '${moduleId}' and are now marked ERROR: ${cleanup.failures.map((f) => f.providerId).join(', ')}. Removal continues; run \`celilo system audit\` to see what each is holding.`,
270
+ );
268
271
  }
269
272
 
270
273
  // Check if module has infrastructure that needs to be destroyed
@@ -371,7 +371,11 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
371
371
  backups: { modules: installedBackupInfo },
372
372
  abandonedOperations: { records: loadAbandonedOperations(db) },
373
373
  undeployedModules: {
374
- modules: installed.map((m) => ({ id: m.id, state: m.state })),
374
+ modules: installed.map((m) => ({
375
+ id: m.id,
376
+ state: m.state,
377
+ errorMessage: m.errorMessage,
378
+ })),
375
379
  },
376
380
  unconfiguredModules: {
377
381
  modules: installed.map((m) => ({
@@ -573,7 +573,11 @@ export async function handleSystemUpdate(
573
573
  },
574
574
  abandonedOperations: { records: loadAbandonedOperations(db) },
575
575
  undeployedModules: {
576
- modules: installed.map((m) => ({ id: m.id, state: m.state })),
576
+ modules: installed.map((m) => ({
577
+ id: m.id,
578
+ state: m.state,
579
+ errorMessage: m.errorMessage,
580
+ })),
577
581
  },
578
582
  unconfiguredModules: {
579
583
  modules: installed.map((m) => ({
@@ -801,7 +805,11 @@ export function rebuildAuditDepsForRerun(
801
805
  })),
802
806
  },
803
807
  undeployedModules: {
804
- modules: installed.map((m) => ({ id: m.id, state: m.state })),
808
+ modules: installed.map((m) => ({
809
+ id: m.id,
810
+ state: m.state,
811
+ errorMessage: m.errorMessage,
812
+ })),
805
813
  },
806
814
  unconfiguredModules: {
807
815
  modules: installed.map((m) => ({
package/src/db/schema.ts CHANGED
@@ -428,10 +428,12 @@ export const machines = sqliteTable('machines', {
428
428
  .$type<Array<{ name: string; ipAddress: string; zone: string }>>()
429
429
  .notNull()
430
430
  .default(sql`'[]'`),
431
- assignedModuleIds: text('assigned_module_ids', { mode: 'json' })
432
- .$type<string[]>()
433
- .notNull()
434
- .default(sql`'[]'`),
431
+ // No `assigned_module_ids` (celilo#773). Occupancy is derived from
432
+ // `module_infrastructure` / `module_systems` at the point of use — both are
433
+ // written by the deploy path and both cascade on module removal, so a machine
434
+ // frees itself. The dropped column had one append-only writer, no removal
435
+ // path, and no reader that reconciled it, and it had already diverged in both
436
+ // directions on the live fleet.
435
437
  /** Module ID this machine is earmarked for. If set, only this module can use this machine. */
436
438
  earmarkedModule: text('earmarked_module'),
437
439
  /**
@@ -589,19 +591,34 @@ export const portForwards = sqliteTable(
589
591
  /** Dedicated INTERNAL ingress IP (ISS-0156); NULL for the normal public path. */
590
592
  ingressIp: text('ingress_ip'),
591
593
  description: text('description').notNull().default(''),
594
+ /**
595
+ * The CONSUMER that registered it, stamped by the store from the calling
596
+ * module (openspec/changes/consumer-removal-cleanup, D5a). `''` for rows
597
+ * written before this column existed — unattributed, so no consumer removal
598
+ * withdraws them.
599
+ */
600
+ registeredBy: text('registered_by').notNull().default(''),
592
601
  createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
593
602
  },
594
603
  (table) => ({
595
- // One row per (firewall, backend, port, protocol, ingress). The store also
596
- // delete-then-inserts on this tuple so a re-expose is an idempotent upsert
597
- // (NULL ingress_ip is compared with `is null` there, since SQLite treats
598
- // NULLs as distinct in a unique index).
604
+ // One row per (OWNER, firewall, backend, port, protocol, ingress). The
605
+ // owner is IN the index, not merely alongside it, and that is the whole
606
+ // point: without it, consumer B exposing a forward A already has silently
607
+ // REPLACES A's row, and B leaving then deletes a rule A still needs. Two
608
+ // owners of one forward are two rows — the set, denormalised into the
609
+ // table that already exists rather than a second table. `renderRuleset`
610
+ // dedupes on the rule tuple so the duplicate renders once.
611
+ //
612
+ // This is the `dns_registration_consumers` lesson (see below): a single
613
+ // overwritten owner column is not a label, because what cascades on it
614
+ // decides when a LIVE record is forgotten.
599
615
  forwardUnique: uniqueIndex('port_forwards_unique_idx').on(
600
616
  table.firewallIp,
601
617
  table.internalIp,
602
618
  table.port,
603
619
  table.protocol,
604
620
  table.ingressIp,
621
+ table.registeredBy,
605
622
  ),
606
623
  }),
607
624
  );
@@ -630,11 +647,14 @@ export const trustedSources = sqliteTable(
630
647
  createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
631
648
  },
632
649
  (table) => ({
633
- // One row per (firewall, subnet) — the store delete-then-inserts on this
634
- // tuple so re-registering the same subnet is an idempotent upsert.
650
+ // One row per (OWNER, firewall, subnet) — same reason the owner is in
651
+ // `port_forwards_unique_idx`. Two modules trusting the same subnet are two
652
+ // rows, so one leaving does not revoke the other's reach. The renderer
653
+ // already dedupes trusted subnets (`new Set`), so the pair renders once.
635
654
  trustedSourceUnique: uniqueIndex('trusted_sources_unique_idx').on(
636
655
  table.firewallIp,
637
656
  table.subnet,
657
+ table.registeredBy,
638
658
  ),
639
659
  }),
640
660
  );