@celilo/cli 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +3 -1
- package/CELILO_SUBSYSTEMS.md +7 -1
- package/MODULE_PRIMITIVES.md +6 -1
- package/drizzle/0027_dns_internal_records_consumer_cascade.sql +43 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/src/capabilities/lookup.ts +39 -29
- package/src/capabilities/secret-ref.test.ts +24 -0
- package/src/capabilities/secret-validation.ts +50 -0
- package/src/capabilities/validation.test.ts +238 -2
- package/src/capabilities/validation.ts +67 -1
- package/src/cli/commands/alerts-sweep.ts +18 -0
- package/src/cli/commands/module-remove.ts +34 -2
- package/src/cli/commands/module-update.test.ts +149 -2
- package/src/cli/commands/module-update.ts +113 -25
- package/src/cli/commands/service-set-credentials.test.ts +108 -0
- package/src/cli/commands/service-set-credentials.ts +115 -0
- package/src/cli/commands/system-migrate.ts +6 -4
- package/src/cli/completion.ts +16 -1
- package/src/cli/index.ts +9 -0
- package/src/db/client.ts +10 -8
- package/src/db/dns-internal-cascade-migration.test.ts +184 -0
- package/src/db/migrate.test.ts +147 -0
- package/src/db/migrate.ts +69 -1
- package/src/db/schema.ts +21 -4
- package/src/hooks/capability-loader.test.ts +55 -0
- package/src/hooks/capability-loader.ts +16 -1
- package/src/manifest/template-validator.test.ts +47 -0
- package/src/manifest/template-validator.ts +18 -1
- package/src/module/import.ts +39 -6
- package/src/policy/capability-shape-baseline.ts +88 -0
- package/src/policy/capability-shape-drift.test.ts +162 -0
- package/src/policy/capability-shape.ts +117 -0
- package/src/policy/dns-aspect-coverage.test.ts +100 -0
- package/src/policy/module-business-baseline.ts +32 -18
- package/src/services/alerting/monitors.ts +54 -2
- package/src/services/alerting/sweep-runner.ts +38 -1
- package/src/services/capability-table-rows.test.ts +191 -0
- package/src/services/capability-table-rows.ts +103 -0
- package/src/services/consumer-cleanup.ts +18 -10
- package/src/services/container-service.test.ts +34 -0
- package/src/services/container-service.ts +44 -0
- package/src/services/deployed-systems.test.ts +101 -0
- package/src/services/deployed-systems.ts +43 -11
- package/src/services/dns-internal-records.test.ts +72 -1
- package/src/services/dns-provider-backfill.ts +30 -0
- package/src/services/fleet-checks.test.ts +26 -0
- package/src/services/fleet-checks.ts +11 -1
- package/src/services/module-deploy.ts +88 -41
- package/src/services/module-validator/capability-versions.test.ts +6 -1
- package/src/services/port-forwards.test.ts +6 -2
- package/src/services/port-forwards.ts +0 -11
- package/src/services/provider-arrival.test.ts +241 -0
- package/src/services/provider-arrival.ts +213 -0
- package/src/services/trusted-sources.ts +0 -5
- package/src/templates/generator.test.ts +35 -0
- package/src/templates/generator.ts +29 -1
- package/src/variables/context.test.ts +63 -0
- package/src/variables/context.ts +85 -12
- package/src/variables/declarative-derivation.test.ts +47 -8
- package/src/variables/declarative-derivation.ts +6 -4
- package/src/variables/lxc-nameserver.test.ts +144 -0
- package/src/services/public-web-republish.test.ts +0 -189
- package/src/services/public-web-republish.ts +0 -84
|
@@ -3,6 +3,20 @@ import { describe, expect, test } from 'bun:test';
|
|
|
3
3
|
import type { ModuleManifest } from '../manifest/schema';
|
|
4
4
|
import { checkAllowlist, getProviderManifest, validateCapabilityAccess } from './validation';
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* A consumer's reference to `dns_external`'s restricted `tsig` secret.
|
|
8
|
+
*
|
|
9
|
+
* The access gate fires only for a secret the consumer actually names
|
|
10
|
+
* (celilo#854), so every fixture that expects a REFUSAL has to name one.
|
|
11
|
+
*/
|
|
12
|
+
const TSIG_REFERENCE = {
|
|
13
|
+
name: 'tsig',
|
|
14
|
+
type: 'string' as const,
|
|
15
|
+
required: false,
|
|
16
|
+
source: 'capability' as const,
|
|
17
|
+
derive_from: '$capability:dns_external.tsig',
|
|
18
|
+
};
|
|
19
|
+
|
|
6
20
|
describe('Capability Access Validation', () => {
|
|
7
21
|
describe('checkAllowlist', () => {
|
|
8
22
|
test('should return true when consumer provides capability in allowlist', () => {
|
|
@@ -452,7 +466,7 @@ describe('Capability Access Validation', () => {
|
|
|
452
466
|
},
|
|
453
467
|
],
|
|
454
468
|
},
|
|
455
|
-
variables: { owns: [], imports: [] },
|
|
469
|
+
variables: { owns: [TSIG_REFERENCE], imports: [] },
|
|
456
470
|
};
|
|
457
471
|
|
|
458
472
|
const providerManifest: ModuleManifest = {
|
|
@@ -510,7 +524,7 @@ describe('Capability Access Validation', () => {
|
|
|
510
524
|
capabilities: [{ name: 'dns_external', version: '1.0.0' }],
|
|
511
525
|
},
|
|
512
526
|
provides: { capabilities: [] }, // No provides section - empty
|
|
513
|
-
variables: { owns: [], imports: [] },
|
|
527
|
+
variables: { owns: [TSIG_REFERENCE], imports: [] },
|
|
514
528
|
};
|
|
515
529
|
|
|
516
530
|
const providerManifest: ModuleManifest = {
|
|
@@ -640,4 +654,226 @@ describe('Capability Access Validation', () => {
|
|
|
640
654
|
expect(callCount).toBe(2); // Should query both capabilities
|
|
641
655
|
});
|
|
642
656
|
});
|
|
657
|
+
|
|
658
|
+
// celilo#854 — the gate is scoped to secrets the consumer actually names.
|
|
659
|
+
//
|
|
660
|
+
// knot-unbound-internal declares dns_internal's `tsig_key` with
|
|
661
|
+
// `readable_by: ["dns_internal"]`. Before this was fixed, ANY module listing
|
|
662
|
+
// dns_internal under `requires.capabilities` and not itself PROVIDING
|
|
663
|
+
// dns_internal was refused at import over a value it never references.
|
|
664
|
+
// caddy-internal was the first module to hit it, and it declared the
|
|
665
|
+
// capability under `optional` to get past the gate — a lie about the
|
|
666
|
+
// dependency graph that six core services read.
|
|
667
|
+
describe('celilo#854 — secret access is gated on reference, not on declaration', () => {
|
|
668
|
+
const knotManifest: ModuleManifest = {
|
|
669
|
+
celilo_contract: '1.0',
|
|
670
|
+
id: 'knot-unbound-internal',
|
|
671
|
+
name: 'Knot + Unbound',
|
|
672
|
+
version: '1.0.0',
|
|
673
|
+
description: 'Test',
|
|
674
|
+
requires: { capabilities: [] },
|
|
675
|
+
provides: {
|
|
676
|
+
capabilities: [
|
|
677
|
+
{
|
|
678
|
+
name: 'dns_internal',
|
|
679
|
+
version: '1.0.0',
|
|
680
|
+
data: {},
|
|
681
|
+
secrets: [
|
|
682
|
+
{
|
|
683
|
+
name: 'tsig_key',
|
|
684
|
+
type: 'string',
|
|
685
|
+
readable_by: ['dns_internal'],
|
|
686
|
+
},
|
|
687
|
+
],
|
|
688
|
+
},
|
|
689
|
+
],
|
|
690
|
+
},
|
|
691
|
+
variables: { owns: [], imports: [] },
|
|
692
|
+
};
|
|
693
|
+
|
|
694
|
+
const knotDb = {
|
|
695
|
+
prepare: () => ({
|
|
696
|
+
get: () => ({ manifest_data: JSON.stringify(knotManifest) }),
|
|
697
|
+
}),
|
|
698
|
+
} as unknown as Database;
|
|
699
|
+
|
|
700
|
+
test('a consumer that requires the capability but never names the secret imports', async () => {
|
|
701
|
+
const manifest: ModuleManifest = {
|
|
702
|
+
celilo_contract: '1.0',
|
|
703
|
+
id: 'caddy-internal',
|
|
704
|
+
name: 'Caddy (fleet-only ingress)',
|
|
705
|
+
version: '1.0.0',
|
|
706
|
+
description: 'Test',
|
|
707
|
+
requires: {
|
|
708
|
+
capabilities: [{ name: 'dns_internal', version: '1.0.0' }],
|
|
709
|
+
},
|
|
710
|
+
provides: {
|
|
711
|
+
capabilities: [{ name: 'private_web', version: '1.0.0', data: {} }],
|
|
712
|
+
},
|
|
713
|
+
variables: { owns: [], imports: [] },
|
|
714
|
+
};
|
|
715
|
+
|
|
716
|
+
const result = await validateCapabilityAccess(manifest, knotDb);
|
|
717
|
+
|
|
718
|
+
expect(result.success).toBe(true);
|
|
719
|
+
expect(result.error).toBeUndefined();
|
|
720
|
+
});
|
|
721
|
+
|
|
722
|
+
test('a consumer that DOES name the restricted secret is still refused', async () => {
|
|
723
|
+
const manifest: ModuleManifest = {
|
|
724
|
+
celilo_contract: '1.0',
|
|
725
|
+
id: 'nosy-app',
|
|
726
|
+
name: 'Nosy App',
|
|
727
|
+
version: '1.0.0',
|
|
728
|
+
description: 'Test',
|
|
729
|
+
requires: {
|
|
730
|
+
capabilities: [{ name: 'dns_internal', version: '1.0.0' }],
|
|
731
|
+
},
|
|
732
|
+
provides: {
|
|
733
|
+
capabilities: [{ name: 'private_web', version: '1.0.0', data: {} }],
|
|
734
|
+
},
|
|
735
|
+
variables: {
|
|
736
|
+
owns: [
|
|
737
|
+
{
|
|
738
|
+
name: 'stolen_key',
|
|
739
|
+
type: 'string',
|
|
740
|
+
required: false,
|
|
741
|
+
source: 'capability',
|
|
742
|
+
derive_from: '$capability:dns_internal.tsig_key',
|
|
743
|
+
},
|
|
744
|
+
],
|
|
745
|
+
imports: [],
|
|
746
|
+
},
|
|
747
|
+
};
|
|
748
|
+
|
|
749
|
+
const result = await validateCapabilityAccess(manifest, knotDb);
|
|
750
|
+
|
|
751
|
+
expect(result.success).toBe(false);
|
|
752
|
+
expect(result.error).toContain("Module 'nosy-app' cannot access secret 'tsig_key'");
|
|
753
|
+
expect(result.error).toContain("capability 'dns_internal'");
|
|
754
|
+
});
|
|
755
|
+
|
|
756
|
+
/**
|
|
757
|
+
* The gate is per-SECRET, not per-capability. A provider that declares two
|
|
758
|
+
* restricted secrets must not put a consumer on the hook for the second
|
|
759
|
+
* one's allow-list just because it named the first — that would move the
|
|
760
|
+
* over-refusal from "requires the capability" down one level to "reads any
|
|
761
|
+
* of its secrets", which is the same conflation wearing a smaller hat.
|
|
762
|
+
*/
|
|
763
|
+
const twoSecretManifest: ModuleManifest = {
|
|
764
|
+
...knotManifest,
|
|
765
|
+
provides: {
|
|
766
|
+
capabilities: [
|
|
767
|
+
{
|
|
768
|
+
name: 'dns_internal',
|
|
769
|
+
version: '1.0.0',
|
|
770
|
+
data: {},
|
|
771
|
+
secrets: [
|
|
772
|
+
{ name: 'tsig_key', type: 'string', readable_by: ['dns_internal'] },
|
|
773
|
+
{ name: 'api_token', type: 'string', readable_by: ['private_web'] },
|
|
774
|
+
],
|
|
775
|
+
},
|
|
776
|
+
],
|
|
777
|
+
},
|
|
778
|
+
};
|
|
779
|
+
|
|
780
|
+
const twoSecretDb = {
|
|
781
|
+
prepare: () => ({
|
|
782
|
+
get: () => ({ manifest_data: JSON.stringify(twoSecretManifest) }),
|
|
783
|
+
}),
|
|
784
|
+
} as unknown as Database;
|
|
785
|
+
|
|
786
|
+
test('naming one secret enforces that secret allow-list and no other', async () => {
|
|
787
|
+
function consumer(deriveFrom: string): ModuleManifest {
|
|
788
|
+
return {
|
|
789
|
+
celilo_contract: '1.0',
|
|
790
|
+
id: 'caddy-internal',
|
|
791
|
+
name: 'Caddy (fleet-only ingress)',
|
|
792
|
+
version: '1.0.0',
|
|
793
|
+
description: 'Test',
|
|
794
|
+
requires: { capabilities: [{ name: 'dns_internal', version: '1.0.0' }] },
|
|
795
|
+
provides: { capabilities: [{ name: 'private_web', version: '1.0.0', data: {} }] },
|
|
796
|
+
variables: {
|
|
797
|
+
owns: [
|
|
798
|
+
{
|
|
799
|
+
name: 'borrowed',
|
|
800
|
+
type: 'string',
|
|
801
|
+
required: false,
|
|
802
|
+
source: 'capability',
|
|
803
|
+
derive_from: deriveFrom,
|
|
804
|
+
},
|
|
805
|
+
],
|
|
806
|
+
imports: [],
|
|
807
|
+
},
|
|
808
|
+
};
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// `api_token` is readable by `private_web`, which this consumer provides.
|
|
812
|
+
// The unnamed `tsig_key`, which it does not satisfy, must not interfere.
|
|
813
|
+
const allowed = await validateCapabilityAccess(
|
|
814
|
+
consumer('$capability:dns_internal.api_token'),
|
|
815
|
+
twoSecretDb,
|
|
816
|
+
);
|
|
817
|
+
expect(allowed.success).toBe(true);
|
|
818
|
+
|
|
819
|
+
// Naming `tsig_key` is still refused, by name.
|
|
820
|
+
const refused = await validateCapabilityAccess(
|
|
821
|
+
consumer('$capability:dns_internal.tsig_key'),
|
|
822
|
+
twoSecretDb,
|
|
823
|
+
);
|
|
824
|
+
expect(refused.success).toBe(false);
|
|
825
|
+
expect(refused.error).toContain('tsig_key');
|
|
826
|
+
});
|
|
827
|
+
|
|
828
|
+
/**
|
|
829
|
+
* A reference in a TEMPLATE counts too. `resolver.ts` does refuse one at
|
|
830
|
+
* the point of use, so nothing is unsafe without this, but the refusal
|
|
831
|
+
* lands at generation rather than at import.
|
|
832
|
+
*
|
|
833
|
+
* That gap sits exactly where the repo sends people. CLAUDE.md's module
|
|
834
|
+
* Definition of Done says "Capability variable usage — Templates use
|
|
835
|
+
* `$capability:` syntax" and gives a `.tf.tpl` example, so the first author
|
|
836
|
+
* who follows that instruction with a RESTRICTED secret is the one who
|
|
837
|
+
* finds out late. No module in the tree references a capability from a
|
|
838
|
+
* template today, which is why the gap is currently invisible rather than
|
|
839
|
+
* absent.
|
|
840
|
+
*/
|
|
841
|
+
test('a secret named only in a template is refused at import', async () => {
|
|
842
|
+
const manifest: ModuleManifest = {
|
|
843
|
+
celilo_contract: '1.0',
|
|
844
|
+
id: 'nosy-app',
|
|
845
|
+
name: 'Nosy App',
|
|
846
|
+
version: '1.0.0',
|
|
847
|
+
description: 'Test',
|
|
848
|
+
requires: { capabilities: [{ name: 'dns_internal', version: '1.0.0' }] },
|
|
849
|
+
provides: { capabilities: [{ name: 'private_web', version: '1.0.0', data: {} }] },
|
|
850
|
+
variables: { owns: [], imports: [] },
|
|
851
|
+
};
|
|
852
|
+
|
|
853
|
+
// The manifest names nothing. Only the .tf.tpl does.
|
|
854
|
+
const result = await validateCapabilityAccess(manifest, knotDb, ['dns_internal.tsig_key']);
|
|
855
|
+
|
|
856
|
+
expect(result.success).toBe(false);
|
|
857
|
+
expect(result.error).toContain("cannot access secret 'tsig_key'");
|
|
858
|
+
});
|
|
859
|
+
|
|
860
|
+
test('a non-secret template reference is still not a secret reference', async () => {
|
|
861
|
+
const manifest: ModuleManifest = {
|
|
862
|
+
celilo_contract: '1.0',
|
|
863
|
+
id: 'caddy-internal',
|
|
864
|
+
name: 'Caddy (fleet-only ingress)',
|
|
865
|
+
version: '1.0.0',
|
|
866
|
+
description: 'Test',
|
|
867
|
+
requires: { capabilities: [{ name: 'dns_internal', version: '1.0.0' }] },
|
|
868
|
+
provides: { capabilities: [{ name: 'private_web', version: '1.0.0', data: {} }] },
|
|
869
|
+
variables: { owns: [], imports: [] },
|
|
870
|
+
};
|
|
871
|
+
|
|
872
|
+
const result = await validateCapabilityAccess(manifest, knotDb, [
|
|
873
|
+
'dns_internal.server.ip.primary',
|
|
874
|
+
]);
|
|
875
|
+
|
|
876
|
+
expect(result.success).toBe(true);
|
|
877
|
+
});
|
|
878
|
+
});
|
|
643
879
|
});
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import type { Database } from 'bun:sqlite';
|
|
7
7
|
import type { ModuleManifest } from '../manifest/schema';
|
|
8
8
|
import { isPrivilegedCapability } from '../manifest/validate';
|
|
9
|
+
import { parseVariables } from '../variables/parser';
|
|
9
10
|
|
|
10
11
|
export interface ValidationResult {
|
|
11
12
|
success: boolean;
|
|
@@ -20,11 +21,16 @@ export interface ValidationResult {
|
|
|
20
21
|
*
|
|
21
22
|
* @param manifest - Consumer module manifest
|
|
22
23
|
* @param db - Database connection
|
|
24
|
+
* @param templateReferences - `<capability>.<path>` references found outside
|
|
25
|
+
* the manifest, i.e. in the module's templates. Empty is the honest default
|
|
26
|
+
* for a caller holding no template context; `module import` passes what the
|
|
27
|
+
* template validator already parsed.
|
|
23
28
|
* @returns Validation result
|
|
24
29
|
*/
|
|
25
30
|
export async function validateCapabilityAccess(
|
|
26
31
|
manifest: ModuleManifest,
|
|
27
32
|
db: Database,
|
|
33
|
+
templateReferences: readonly string[] = [],
|
|
28
34
|
): Promise<ValidationResult> {
|
|
29
35
|
// If module doesn't require capabilities, validation passes
|
|
30
36
|
if (!manifest.requires?.capabilities || manifest.requires.capabilities.length === 0) {
|
|
@@ -34,6 +40,8 @@ export async function validateCapabilityAccess(
|
|
|
34
40
|
// Get list of capabilities this module provides
|
|
35
41
|
const consumerCapabilities = (manifest.provides?.capabilities || []).map((cap) => cap.name);
|
|
36
42
|
|
|
43
|
+
const references = collectCapabilityReferences(manifest, templateReferences);
|
|
44
|
+
|
|
37
45
|
// Check each required capability
|
|
38
46
|
for (const requiredCapability of manifest.requires.capabilities) {
|
|
39
47
|
// Framework-granted privileges (e.g. cross_module_read) are not
|
|
@@ -63,8 +71,12 @@ export async function validateCapabilityAccess(
|
|
|
63
71
|
continue;
|
|
64
72
|
}
|
|
65
73
|
|
|
66
|
-
// Check allowlist for each secret
|
|
74
|
+
// Check allowlist for each secret the consumer actually names (celilo#854).
|
|
67
75
|
for (const secret of capabilityDef.secrets) {
|
|
76
|
+
if (!referencesSecret(references, requiredCapability.name, secret.name)) {
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
|
|
68
80
|
if (secret.readable_by && secret.readable_by.length > 0) {
|
|
69
81
|
// Check if consumer provides any capability in the allowlist
|
|
70
82
|
const hasAccess = checkAllowlist(consumerCapabilities, secret.readable_by);
|
|
@@ -89,6 +101,60 @@ export async function validateCapabilityAccess(
|
|
|
89
101
|
return { success: true };
|
|
90
102
|
}
|
|
91
103
|
|
|
104
|
+
/**
|
|
105
|
+
* Every `$capability:<name>.<path>` reference the consumer's manifest makes.
|
|
106
|
+
*
|
|
107
|
+
* Policy function (Rule 10.1) - parses only, no I/O.
|
|
108
|
+
*
|
|
109
|
+
* Serializing the manifest and parsing the result finds a reference wherever it
|
|
110
|
+
* lives — a `variables.owns[].derive_from`, a default, a capability data block —
|
|
111
|
+
* without this having to track which fields may hold one.
|
|
112
|
+
*
|
|
113
|
+
* Template references are included too, supplied by the caller. `resolver.ts`
|
|
114
|
+
* does refuse a template reference at the point of use, so nothing is unsafe
|
|
115
|
+
* without them, but the refusal lands at generation rather than at import. That
|
|
116
|
+
* matters because a module's TEMPLATES are where this repo tells authors to put
|
|
117
|
+
* these: the Definition of Done in CLAUDE.md says "Capability variable usage —
|
|
118
|
+
* Templates use `$capability:` syntax" and gives a `.tf.tpl` example. A gate
|
|
119
|
+
* that does not fire where the documentation sends people is a gate with a hole
|
|
120
|
+
* in the shape of the instructions.
|
|
121
|
+
*/
|
|
122
|
+
function collectCapabilityReferences(
|
|
123
|
+
manifest: ModuleManifest,
|
|
124
|
+
templateReferences: readonly string[] = [],
|
|
125
|
+
): Set<string> {
|
|
126
|
+
return new Set([
|
|
127
|
+
...parseVariables(JSON.stringify(manifest))
|
|
128
|
+
.filter((variable) => variable.type === 'capability')
|
|
129
|
+
.map((variable) => variable.path),
|
|
130
|
+
...templateReferences,
|
|
131
|
+
]);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Does the consumer name this capability secret?
|
|
136
|
+
*
|
|
137
|
+
* Policy function (Rule 10.1) - pure logic, no I/O.
|
|
138
|
+
*
|
|
139
|
+
* A reference of `dns_internal.tsig_key` names the `tsig_key` secret, and so
|
|
140
|
+
* does `dns_internal.tsig_key.value` if the secret ever holds a structure.
|
|
141
|
+
* Requiring the capability alone names nothing (celilo#854): a module refused
|
|
142
|
+
* over a secret it never reads has to lie about its dependency graph to deploy.
|
|
143
|
+
*/
|
|
144
|
+
function referencesSecret(
|
|
145
|
+
references: Set<string>,
|
|
146
|
+
capabilityName: string,
|
|
147
|
+
secretName: string,
|
|
148
|
+
): boolean {
|
|
149
|
+
const base = `${capabilityName}.${secretName}`;
|
|
150
|
+
for (const reference of references) {
|
|
151
|
+
if (reference === base || reference.startsWith(`${base}.`)) {
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return false;
|
|
156
|
+
}
|
|
157
|
+
|
|
92
158
|
/**
|
|
93
159
|
* Check if consumer capabilities match provider allowlist
|
|
94
160
|
*
|
|
@@ -89,6 +89,12 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
|
|
|
89
89
|
if (report.deferredDelivered > 0) parts.push(`${report.deferredDelivered} deferred-delivered`);
|
|
90
90
|
if (report.failed > 0) parts.push(`${report.failed} FAILED`);
|
|
91
91
|
if (report.noPolicy.length > 0) parts.push(`${report.noPolicy.length} no-policy`);
|
|
92
|
+
// A monitor whose module is gone is dropped here rather than left to sit
|
|
93
|
+
// unschedulable forever (celilo#1029).
|
|
94
|
+
if (report.strandedDropped.length > 0) {
|
|
95
|
+
const names = report.strandedDropped.map((d) => d.moduleId).join(', ');
|
|
96
|
+
parts.push(`${report.strandedDropped.length} stranded-dropped (${names})`);
|
|
97
|
+
}
|
|
92
98
|
|
|
93
99
|
const lines = [`alert sweep: ${parts.join(', ')}`];
|
|
94
100
|
|
|
@@ -109,6 +115,18 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
|
|
|
109
115
|
lines.push(` ${alertKey} (${reason})`);
|
|
110
116
|
}
|
|
111
117
|
}
|
|
118
|
+
// A dropped monitor's alerts are deleted with it, not resolved — nothing else
|
|
119
|
+
// anywhere records that they existed. Name each one and what it said: the
|
|
120
|
+
// failure it was reporting is real and is now unwatched, which is the fact an
|
|
121
|
+
// operator has to act on and the one a module name alone does not carry.
|
|
122
|
+
for (const { moduleId, alerts: dropped } of report.strandedDropped) {
|
|
123
|
+
if (dropped.length === 0) continue;
|
|
124
|
+
lines.push(` ${moduleId} is gone; its monitor was holding ${dropped.length} live alert(s):`);
|
|
125
|
+
for (const alert of dropped) {
|
|
126
|
+
lines.push(` ${alert.key}: ${alert.message}`);
|
|
127
|
+
}
|
|
128
|
+
lines.push(` nothing checks ${moduleId} any more, so these will not be reported again.`);
|
|
129
|
+
}
|
|
112
130
|
// The error itself, not just a count: the transport is loaded lazily inside
|
|
113
131
|
// the send, so a capability that will not load produces no other record
|
|
114
132
|
// anywhere — nothing ever reaches the transport's own logs.
|
|
@@ -15,6 +15,7 @@ import { createGaugeLogger } from '../../hooks/logger';
|
|
|
15
15
|
import { runNamedHook } from '../../hooks/run-named-hook';
|
|
16
16
|
import { deallocateForModule } from '../../ipam/auto-allocator';
|
|
17
17
|
import { type ModuleManifest, ModuleManifestSchema } from '../../manifest/schema';
|
|
18
|
+
import { deleteMonitorForModule } from '../../services/alerting/monitors';
|
|
18
19
|
import { executeBuildWithProgress } from '../../services/build-stream';
|
|
19
20
|
import { askConfirm, withInterviewSession } from '../../services/bus-interview';
|
|
20
21
|
import {
|
|
@@ -22,7 +23,11 @@ import {
|
|
|
22
23
|
emitUninstallFailed,
|
|
23
24
|
emitUninstallStarted,
|
|
24
25
|
} from '../../services/celilo-events';
|
|
25
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
PRE_DEPLOY_STATES,
|
|
28
|
+
loadConsumerCleanupPlan,
|
|
29
|
+
runConsumerCleanup,
|
|
30
|
+
} from '../../services/consumer-cleanup';
|
|
26
31
|
import { getContainerService, getServiceCredentials } from '../../services/container-service';
|
|
27
32
|
import { completeOperation, failOperation, startOperation } from '../../services/module-operations';
|
|
28
33
|
import {
|
|
@@ -97,7 +102,7 @@ export async function handleModuleRemove(
|
|
|
97
102
|
id: m.id,
|
|
98
103
|
manifest: parsed.data,
|
|
99
104
|
paused: m.state === 'PAUSED',
|
|
100
|
-
deployed: !
|
|
105
|
+
deployed: !PRE_DEPLOY_STATES.has(m.state),
|
|
101
106
|
});
|
|
102
107
|
}
|
|
103
108
|
|
|
@@ -403,6 +408,33 @@ async function performModuleRemove(
|
|
|
403
408
|
log.warn(`Failed to unregister event-bus subscriptions: ${msg}`);
|
|
404
409
|
}
|
|
405
410
|
|
|
411
|
+
// Drop the module's health monitor. No cascade can reach it — `monitors.target`
|
|
412
|
+
// holds a module id or an audit check name depending on `kind`, so the column
|
|
413
|
+
// carries no foreign key (celilo#1029). Left behind, the monitor fires
|
|
414
|
+
// `Module not found` and then becomes permanently unschedulable, because its
|
|
415
|
+
// cadence resolves from a module row that no longer exists — so nothing ever
|
|
416
|
+
// runs it again to resolve the alert it just raised.
|
|
417
|
+
//
|
|
418
|
+
// What it was holding is named, not just counted. The alerts are deleted with
|
|
419
|
+
// it (cascade), so a firing check goes silent and the coverage of it goes at
|
|
420
|
+
// the same moment — an operator who is told only `removed monitor` never
|
|
421
|
+
// learns which real failure just stopped being reported.
|
|
422
|
+
try {
|
|
423
|
+
const droppedAlerts = deleteMonitorForModule(db, moduleId);
|
|
424
|
+
if (droppedAlerts) {
|
|
425
|
+
log.info(`Removed health monitor for ${moduleId}`);
|
|
426
|
+
for (const alert of droppedAlerts) {
|
|
427
|
+
log.warn(` dropped live alert ${alert.key}: ${alert.message}`);
|
|
428
|
+
}
|
|
429
|
+
if (droppedAlerts.length > 0) {
|
|
430
|
+
log.warn(` nothing checks ${moduleId} any more, so this will not be reported again.`);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
} catch (error) {
|
|
434
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
435
|
+
log.warn(`Failed to remove health monitor for ${moduleId}: ${msg}`);
|
|
436
|
+
}
|
|
437
|
+
|
|
406
438
|
// Delete module (cascade will remove configs, secrets, capabilities, infrastructure records)
|
|
407
439
|
db.delete(modules).where(eq(modules.id, moduleId)).run();
|
|
408
440
|
|
|
@@ -5,12 +5,21 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
8
|
-
import {
|
|
8
|
+
import { execFileSync } from 'node:child_process';
|
|
9
|
+
import {
|
|
10
|
+
existsSync,
|
|
11
|
+
mkdirSync,
|
|
12
|
+
mkdtempSync,
|
|
13
|
+
readFileSync,
|
|
14
|
+
readdirSync,
|
|
15
|
+
rmSync,
|
|
16
|
+
writeFileSync,
|
|
17
|
+
} from 'node:fs';
|
|
9
18
|
import { tmpdir } from 'node:os';
|
|
10
19
|
import { join, relative } from 'node:path';
|
|
11
20
|
import { eq } from 'drizzle-orm';
|
|
12
21
|
import { type DbClient, getDb } from '../../db/client';
|
|
13
|
-
import { modules } from '../../db/schema';
|
|
22
|
+
import { moduleIntegrity, modules } from '../../db/schema';
|
|
14
23
|
import { classifyVersionChange, handleModuleUpdate, updateOne } from './module-update';
|
|
15
24
|
|
|
16
25
|
describe('classifyVersionChange', () => {
|
|
@@ -409,3 +418,141 @@ description: fixture
|
|
|
409
418
|
expect(snapshot(installedDir)).toEqual(before);
|
|
410
419
|
});
|
|
411
420
|
});
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* celilo#1008. `updateOne` copied the new tree onto the installed one in
|
|
424
|
+
* place, so a copy that died partway left a module half old and half new,
|
|
425
|
+
* with no record anywhere of which files were which. Jeremy Banka's
|
|
426
|
+
* `f9a57f1b` staged into a sibling and swapped with two renames; the rest of
|
|
427
|
+
* that commit is superseded by celilo#925, but the atomicity is not, and this
|
|
428
|
+
* is where it lands.
|
|
429
|
+
*
|
|
430
|
+
* The failure is provoked rather than injected, so nothing test-only reaches
|
|
431
|
+
* production code. The source carries a FIFO, which `cpSync` refuses with
|
|
432
|
+
* ENOTSUP — a stand-in for any mid-copy failure a real update can hit (a full
|
|
433
|
+
* disk, a permission, an I/O error). It is chosen because it fails on the
|
|
434
|
+
* SOURCE, so it fires whether the copy targets the live install or a staging
|
|
435
|
+
* directory, which a destination-side collision would not. `z-` sorts last,
|
|
436
|
+
* so the entries ahead of it copy successfully first — precisely the
|
|
437
|
+
* half-applied state being ruled out.
|
|
438
|
+
*/
|
|
439
|
+
describe('updateOne — an update that fails partway leaves the install untouched', () => {
|
|
440
|
+
let tempDir: string;
|
|
441
|
+
let srcDir: string;
|
|
442
|
+
let installedDir: string;
|
|
443
|
+
let db: DbClient;
|
|
444
|
+
|
|
445
|
+
/** Every file under `root`, relative path → bytes, for an exact comparison. */
|
|
446
|
+
function snapshotTree(root: string, dir = root): Record<string, string> {
|
|
447
|
+
const out: Record<string, string> = {};
|
|
448
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
449
|
+
const full = join(dir, entry.name);
|
|
450
|
+
if (entry.isDirectory()) Object.assign(out, snapshotTree(root, full));
|
|
451
|
+
else out[relative(root, full)] = readFileSync(full, 'utf-8');
|
|
452
|
+
}
|
|
453
|
+
return out;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
beforeEach(() => {
|
|
457
|
+
tempDir = mkdtempSync(join(tmpdir(), 'celilo-atomic-'));
|
|
458
|
+
process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
|
|
459
|
+
process.env.CELILO_ORIGINAL_CWD = tempDir;
|
|
460
|
+
|
|
461
|
+
installedDir = join(tempDir, 'installed', 'testmod');
|
|
462
|
+
mkdirSync(join(installedDir, 'scripts'), { recursive: true });
|
|
463
|
+
mkdirSync(join(installedDir, 'generated'), { recursive: true });
|
|
464
|
+
writeFileSync(
|
|
465
|
+
join(installedDir, 'manifest.yml'),
|
|
466
|
+
'celilo_contract: "1.0"\nid: testmod\nname: Test Module\nversion: 1.0.0\ndescription: fixture\n',
|
|
467
|
+
);
|
|
468
|
+
writeFileSync(join(installedDir, 'scripts', 'on_install.ts'), 'export const OLD = 1;\n');
|
|
469
|
+
// `derived` — celilo's own output, must survive an update either way.
|
|
470
|
+
writeFileSync(join(installedDir, 'generated', 'terraform.tfstate'), '{"old":true}\n');
|
|
471
|
+
|
|
472
|
+
srcDir = join(tempDir, 'src');
|
|
473
|
+
mkdirSync(join(srcDir, 'scripts'), { recursive: true });
|
|
474
|
+
writeFileSync(
|
|
475
|
+
join(srcDir, 'manifest.yml'),
|
|
476
|
+
'celilo_contract: "1.0"\nid: testmod\nname: Test Module\nversion: 2.0.0\ndescription: fixture\n',
|
|
477
|
+
);
|
|
478
|
+
writeFileSync(join(srcDir, 'scripts', 'on_install.ts'), 'export const NEW = 2;\n');
|
|
479
|
+
// cpSync refuses a FIFO with ENOTSUP. Sorts last, so the real files copy first.
|
|
480
|
+
execFileSync('mkfifo', [join(srcDir, 'z-boom')]);
|
|
481
|
+
|
|
482
|
+
db = getDb();
|
|
483
|
+
db.insert(modules)
|
|
484
|
+
.values({
|
|
485
|
+
id: 'testmod',
|
|
486
|
+
name: 'Test Module',
|
|
487
|
+
sourcePath: installedDir,
|
|
488
|
+
version: '1.0.0',
|
|
489
|
+
manifestData: {
|
|
490
|
+
celilo_contract: '1.0',
|
|
491
|
+
id: 'testmod',
|
|
492
|
+
name: 'Test Module',
|
|
493
|
+
version: '1.0.0',
|
|
494
|
+
},
|
|
495
|
+
})
|
|
496
|
+
.run();
|
|
497
|
+
});
|
|
498
|
+
|
|
499
|
+
afterEach(() => {
|
|
500
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
501
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
502
|
+
process.env.CELILO_ORIGINAL_CWD = undefined;
|
|
503
|
+
});
|
|
504
|
+
|
|
505
|
+
test('a failed update leaves the installed tree byte-identical', async () => {
|
|
506
|
+
const before = snapshotTree(installedDir);
|
|
507
|
+
expect(before['scripts/on_install.ts']).toBe('export const OLD = 1;\n');
|
|
508
|
+
|
|
509
|
+
await expect(updateOne(srcDir, db, {}, { quiet: true })).rejects.toThrow();
|
|
510
|
+
|
|
511
|
+
expect(snapshotTree(installedDir)).toEqual(before);
|
|
512
|
+
});
|
|
513
|
+
|
|
514
|
+
test('a successful update keeps celilo output and drops what the version removed', async () => {
|
|
515
|
+
// Same fixture minus the FIFO, so the update runs to completion.
|
|
516
|
+
rmSync(join(srcDir, 'z-boom'));
|
|
517
|
+
// A file the previous version shipped and the new one does not.
|
|
518
|
+
writeFileSync(join(installedDir, 'scripts', 'gone_in_2.ts'), 'export const OLD = 1;\n');
|
|
519
|
+
|
|
520
|
+
const result = await updateOne(srcDir, db, {}, { quiet: true });
|
|
521
|
+
expect(result.status).toBe('success');
|
|
522
|
+
|
|
523
|
+
// `derived`: celilo's own output survives (task 11.6).
|
|
524
|
+
expect(readFileSync(join(installedDir, 'generated', 'terraform.tfstate'), 'utf-8')).toBe(
|
|
525
|
+
'{"old":true}\n',
|
|
526
|
+
);
|
|
527
|
+
// `package`: the new version's content landed...
|
|
528
|
+
expect(readFileSync(join(installedDir, 'scripts', 'on_install.ts'), 'utf-8')).toBe(
|
|
529
|
+
'export const NEW = 2;\n',
|
|
530
|
+
);
|
|
531
|
+
// ...and what it dropped is gone, pruned in staging rather than in place.
|
|
532
|
+
expect(existsSync(join(installedDir, 'scripts', 'gone_in_2.ts'))).toBe(false);
|
|
533
|
+
// No staging debris on the success path either.
|
|
534
|
+
expect(readdirSync(join(tempDir, 'installed'))).toEqual(['testmod']);
|
|
535
|
+
});
|
|
536
|
+
|
|
537
|
+
test('a failed update leaves no staging directory behind', async () => {
|
|
538
|
+
await expect(updateOne(srcDir, db, {}, { quiet: true })).rejects.toThrow();
|
|
539
|
+
|
|
540
|
+
const siblings = readdirSync(join(tempDir, 'installed'));
|
|
541
|
+
expect(siblings).toEqual(['testmod']);
|
|
542
|
+
});
|
|
543
|
+
|
|
544
|
+
test('a failed update does not advance the integrity baseline', async () => {
|
|
545
|
+
await expect(updateOne(srcDir, db, {}, { quiet: true })).rejects.toThrow();
|
|
546
|
+
|
|
547
|
+
const row = db
|
|
548
|
+
.select()
|
|
549
|
+
.from(moduleIntegrity)
|
|
550
|
+
.where(eq(moduleIntegrity.moduleId, 'testmod'))
|
|
551
|
+
.get();
|
|
552
|
+
// Nothing recorded at all: the update never reached a state worth claiming.
|
|
553
|
+
expect(row).toBeUndefined();
|
|
554
|
+
// And the module row still names the version actually on disk.
|
|
555
|
+
const mod = db.select().from(modules).where(eq(modules.id, 'testmod')).get();
|
|
556
|
+
expect(mod?.version).toBe('1.0.0');
|
|
557
|
+
});
|
|
558
|
+
});
|