@celilo/cli 1.12.0 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -1
- package/CELILO_SUBSYSTEMS.md +17 -2
- package/package.json +3 -3
- package/src/cli/commands/alerts-list.ts +16 -1
- package/src/cli/commands/backup-list.test.ts +82 -1
- package/src/cli/commands/backup-list.ts +113 -4
- package/src/cli/commands/console.ts +122 -0
- package/src/cli/commands/module-list.ts +3 -41
- package/src/cli/completion.ts +5 -0
- package/src/cli/index.ts +25 -1
- package/src/console/closure.test.ts +246 -0
- package/src/console/closure.ts +208 -0
- package/src/console/control-plane-boundary.test.ts +75 -0
- package/src/console/projection.test.ts +231 -0
- package/src/console/projection.ts +327 -0
- package/src/db/schema.ts +19 -14
- package/src/hooks/broker.test.ts +4 -6
- package/src/hooks/executor.test.ts +85 -4
- package/src/hooks/executor.ts +164 -9
- package/src/hooks/hook-jail-unreachability.test.ts +173 -0
- package/src/hooks/hook-state-dir.test.ts +14 -2
- package/src/hooks/hook-timeout.test.ts +2 -4
- package/src/hooks/hook-trespass.test.ts +50 -5
- package/src/hooks/jail.test.ts +370 -0
- package/src/hooks/jail.ts +491 -0
- package/src/hooks/mount-set.ts +24 -0
- package/src/hooks/test-fixtures/jail-probe-hook.ts +59 -0
- package/src/manifest/schema.ts +35 -0
- package/src/manifest/validate.test.ts +142 -0
- package/src/manifest/validate.ts +101 -0
- package/src/module/import.test.ts +116 -0
- package/src/module/import.ts +73 -1
- package/src/module/packaging/audit.ts +103 -1
- package/src/module/packaging/classify-module-path.test.ts +36 -0
- package/src/module/packaging/package-rules.ts +18 -0
- package/src/policy/capability-shape-baseline.ts +8 -0
- package/src/policy/module-business-baseline.ts +12 -0
- package/src/services/alerting/observed-health.ts +71 -0
- package/src/services/api-principal-enrolment.test.ts +179 -0
- package/src/services/api-principal-enrolment.ts +103 -0
- package/src/services/audit/backups.ts +10 -1
- package/src/services/backup-metadata.ts +19 -11
- package/src/services/consumer-cleanup.ts +31 -5
- package/src/services/instance-ops.test.ts +302 -0
- package/src/services/instance-ops.ts +292 -0
- package/src/services/module-instances.test.ts +428 -42
- package/src/services/module-instances.ts +219 -26
|
@@ -7,6 +7,8 @@ import {
|
|
|
7
7
|
validateHookContract,
|
|
8
8
|
validateManifest,
|
|
9
9
|
validateProvidesNoCrossCapabilityRefs,
|
|
10
|
+
validateSubmoduleDeclaration,
|
|
11
|
+
validateSubmoduleManifest,
|
|
10
12
|
validateVariableSources,
|
|
11
13
|
} from './validate';
|
|
12
14
|
|
|
@@ -1531,3 +1533,143 @@ base_module_aspect:
|
|
|
1531
1533
|
expect(result.success).toBe(false);
|
|
1532
1534
|
});
|
|
1533
1535
|
});
|
|
1536
|
+
|
|
1537
|
+
describe('submodules (openspec/changes/submodules D1, D9)', () => {
|
|
1538
|
+
const parent = (over: Record<string, unknown> = {}) => ({
|
|
1539
|
+
celilo_contract: '1.0' as const,
|
|
1540
|
+
id: 'forgejo',
|
|
1541
|
+
name: 'Forgejo',
|
|
1542
|
+
version: '1.0.0',
|
|
1543
|
+
requires: { capabilities: [] },
|
|
1544
|
+
provides: { capabilities: [] },
|
|
1545
|
+
variables: { owns: [], imports: [] },
|
|
1546
|
+
...over,
|
|
1547
|
+
});
|
|
1548
|
+
|
|
1549
|
+
const submodule = (over: Record<string, unknown> = {}) => ({
|
|
1550
|
+
celilo_contract: '1.0' as const,
|
|
1551
|
+
id: 'runner',
|
|
1552
|
+
name: 'Forgejo Runner',
|
|
1553
|
+
version: '1.0.0',
|
|
1554
|
+
requires: { capabilities: [] },
|
|
1555
|
+
provides: { capabilities: [] },
|
|
1556
|
+
variables: { owns: [], imports: [] },
|
|
1557
|
+
...over,
|
|
1558
|
+
});
|
|
1559
|
+
|
|
1560
|
+
describe('validateSubmoduleDeclaration', () => {
|
|
1561
|
+
test('a module declaring no submodules is unaffected', () => {
|
|
1562
|
+
expect(validateSubmoduleDeclaration(parent())).toBeNull();
|
|
1563
|
+
});
|
|
1564
|
+
|
|
1565
|
+
test('accepts a plain list of names', () => {
|
|
1566
|
+
expect(validateSubmoduleDeclaration(parent({ submodules: ['runner'] }))).toBeNull();
|
|
1567
|
+
});
|
|
1568
|
+
|
|
1569
|
+
test('rejects a duplicate name', () => {
|
|
1570
|
+
const result = validateSubmoduleDeclaration(parent({ submodules: ['runner', 'runner'] }));
|
|
1571
|
+
expect(result).not.toBeNull();
|
|
1572
|
+
expect(result?.errors[0]?.message).toContain('more than once');
|
|
1573
|
+
});
|
|
1574
|
+
|
|
1575
|
+
test('rejects a submodule sharing its parent id', () => {
|
|
1576
|
+
const result = validateSubmoduleDeclaration(parent({ submodules: ['forgejo'] }));
|
|
1577
|
+
expect(result).not.toBeNull();
|
|
1578
|
+
expect(result?.errors[0]?.message).toContain("parent module's id");
|
|
1579
|
+
});
|
|
1580
|
+
});
|
|
1581
|
+
|
|
1582
|
+
describe('validateSubmoduleManifest', () => {
|
|
1583
|
+
test('accepts an ordinary submodule manifest', () => {
|
|
1584
|
+
expect(validateSubmoduleManifest(parent(), 'runner', submodule())).toBeNull();
|
|
1585
|
+
});
|
|
1586
|
+
|
|
1587
|
+
test('rejects an id that disagrees with its directory', () => {
|
|
1588
|
+
const result = validateSubmoduleManifest(parent(), 'runner', submodule({ id: 'builder' }));
|
|
1589
|
+
expect(result).not.toBeNull();
|
|
1590
|
+
expect(result?.errors[0]?.message).toContain('does not match its directory');
|
|
1591
|
+
});
|
|
1592
|
+
|
|
1593
|
+
// D9: a capability resolves to ONE provider, and celilo's two bespoke
|
|
1594
|
+
// multi-provider rules RESOLVE rather than fail, so N instances would bind
|
|
1595
|
+
// a consumer to whichever was created first — silently.
|
|
1596
|
+
test('rejects a submodule that provides a capability', () => {
|
|
1597
|
+
const result = validateSubmoduleManifest(
|
|
1598
|
+
parent(),
|
|
1599
|
+
'runner',
|
|
1600
|
+
submodule({
|
|
1601
|
+
provides: { capabilities: [{ name: 'source_forge', version: '1.0.0', data: {} }] },
|
|
1602
|
+
}),
|
|
1603
|
+
);
|
|
1604
|
+
expect(result).not.toBeNull();
|
|
1605
|
+
expect(result?.errors[0]?.message).toContain('may not provide a capability');
|
|
1606
|
+
expect(result?.errors[0]?.message).toContain('source_forge');
|
|
1607
|
+
});
|
|
1608
|
+
|
|
1609
|
+
test('a submodule may still REQUIRE capabilities', () => {
|
|
1610
|
+
const result = validateSubmoduleManifest(
|
|
1611
|
+
parent(),
|
|
1612
|
+
'runner',
|
|
1613
|
+
submodule({ requires: { capabilities: [{ name: 'source_forge', version: '1.0.0' }] } }),
|
|
1614
|
+
);
|
|
1615
|
+
expect(result).toBeNull();
|
|
1616
|
+
});
|
|
1617
|
+
|
|
1618
|
+
test('rejects nesting — ownership is one level deep', () => {
|
|
1619
|
+
const result = validateSubmoduleManifest(
|
|
1620
|
+
parent(),
|
|
1621
|
+
'runner',
|
|
1622
|
+
submodule({ submodules: ['nested'] }),
|
|
1623
|
+
);
|
|
1624
|
+
expect(result).not.toBeNull();
|
|
1625
|
+
expect(result?.errors[0]?.message).toContain('may not declare submodules of its own');
|
|
1626
|
+
});
|
|
1627
|
+
|
|
1628
|
+
test('reports every problem at once rather than stopping at the first', () => {
|
|
1629
|
+
const result = validateSubmoduleManifest(
|
|
1630
|
+
parent(),
|
|
1631
|
+
'runner',
|
|
1632
|
+
submodule({
|
|
1633
|
+
id: 'builder',
|
|
1634
|
+
provides: { capabilities: [{ name: 'source_forge', version: '1.0.0', data: {} }] },
|
|
1635
|
+
submodules: ['nested'],
|
|
1636
|
+
}),
|
|
1637
|
+
);
|
|
1638
|
+
expect(result?.errors).toHaveLength(3);
|
|
1639
|
+
});
|
|
1640
|
+
});
|
|
1641
|
+
|
|
1642
|
+
describe('schema', () => {
|
|
1643
|
+
test('parses a manifest declaring submodules', () => {
|
|
1644
|
+
const yaml = `
|
|
1645
|
+
celilo_contract: "1.0"
|
|
1646
|
+
id: forgejo
|
|
1647
|
+
name: Forgejo
|
|
1648
|
+
version: 1.0.0
|
|
1649
|
+
submodules:
|
|
1650
|
+
- runner
|
|
1651
|
+
requires:
|
|
1652
|
+
capabilities: []
|
|
1653
|
+
`;
|
|
1654
|
+
const result = validateManifest(yaml);
|
|
1655
|
+
expect(result.success).toBe(true);
|
|
1656
|
+
if (result.success) expect(result.data.submodules).toEqual(['runner']);
|
|
1657
|
+
});
|
|
1658
|
+
|
|
1659
|
+
test('rejects a non-kebab-case submodule name', () => {
|
|
1660
|
+
const yaml = `
|
|
1661
|
+
celilo_contract: "1.0"
|
|
1662
|
+
id: forgejo
|
|
1663
|
+
name: Forgejo
|
|
1664
|
+
version: 1.0.0
|
|
1665
|
+
submodules:
|
|
1666
|
+
- Runner_One
|
|
1667
|
+
requires:
|
|
1668
|
+
capabilities: []
|
|
1669
|
+
`;
|
|
1670
|
+
const result = validateManifest(yaml);
|
|
1671
|
+
expect(result.success).toBe(false);
|
|
1672
|
+
if (!result.success) expect(result.errors[0]?.message).toContain('kebab-case');
|
|
1673
|
+
});
|
|
1674
|
+
});
|
|
1675
|
+
});
|
package/src/manifest/validate.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { KNOWN_CAPABILITY_NAMES, isProviderView } from '@celilo/capabilities';
|
|
2
2
|
import { parse as parseYaml } from 'yaml';
|
|
3
3
|
import type { ZodError } from 'zod';
|
|
4
|
+
import { SUBMODULES_DIR } from '../module/packaging/package-rules';
|
|
4
5
|
import { validateModuleZoneRequirements } from '../services/zone-policy';
|
|
5
6
|
import { resolveContract, supportedContractVersions } from './contracts';
|
|
6
7
|
import { ModuleManifestSchema, getSingularSystemSpec } from './schema';
|
|
@@ -495,3 +496,103 @@ export function validateProvidesNoCrossCapabilityRefs(
|
|
|
495
496
|
|
|
496
497
|
return null;
|
|
497
498
|
}
|
|
499
|
+
|
|
500
|
+
/**
|
|
501
|
+
* The directory a parent's submodules live in, relative to the parent's own
|
|
502
|
+
* source root. Re-exported rather than redefined: the packaging rules need the
|
|
503
|
+
* same name and `package-rules.ts` is deliberately dependency-free (the
|
|
504
|
+
* registry server imports it), so it owns the definition and everything
|
|
505
|
+
* manifest-facing reads it from here.
|
|
506
|
+
*/
|
|
507
|
+
export { SUBMODULES_DIR };
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Validate a PARENT's `submodules` declaration.
|
|
511
|
+
*
|
|
512
|
+
* Policy function - checks the declaration itself, not the submodules it names.
|
|
513
|
+
* Reading those is I/O and belongs to the caller.
|
|
514
|
+
*
|
|
515
|
+
* @param manifest - Validated manifest of the parent
|
|
516
|
+
* @returns Validation errors if any, null if clean
|
|
517
|
+
*/
|
|
518
|
+
export function validateSubmoduleDeclaration(manifest: ModuleManifest): ValidationError | null {
|
|
519
|
+
const declared = manifest.submodules ?? [];
|
|
520
|
+
if (declared.length === 0) return null;
|
|
521
|
+
|
|
522
|
+
const errors: Array<{ path: string; message: string }> = [];
|
|
523
|
+
const seen = new Set<string>();
|
|
524
|
+
|
|
525
|
+
for (const [index, name] of declared.entries()) {
|
|
526
|
+
if (seen.has(name)) {
|
|
527
|
+
errors.push({
|
|
528
|
+
path: `submodules.${index}`,
|
|
529
|
+
message: `Submodule '${name}' is declared more than once`,
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
seen.add(name);
|
|
533
|
+
|
|
534
|
+
// The derived id of an instance is built from the parent and the submodule
|
|
535
|
+
// name, so a submodule sharing its parent's id yields an instance id that
|
|
536
|
+
// reads as the parent's own. Cheap to refuse, confusing to debug.
|
|
537
|
+
if (name === manifest.id) {
|
|
538
|
+
errors.push({
|
|
539
|
+
path: `submodules.${index}`,
|
|
540
|
+
message: `Submodule '${name}' cannot share its parent module's id`,
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
return errors.length > 0 ? { success: false, errors } : null;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* Validate that a SUBMODULE's own manifest is legal as a submodule.
|
|
550
|
+
*
|
|
551
|
+
* Policy function - the caller has already read and schema-validated the
|
|
552
|
+
* submodule's manifest; this decides whether it may be one.
|
|
553
|
+
*
|
|
554
|
+
* @param parent - The declaring parent's manifest
|
|
555
|
+
* @param submoduleName - The name the parent declared, i.e. its directory
|
|
556
|
+
* @param submodule - The submodule's own validated manifest
|
|
557
|
+
* @returns Validation errors if any, null if clean
|
|
558
|
+
*/
|
|
559
|
+
export function validateSubmoduleManifest(
|
|
560
|
+
parent: ModuleManifest,
|
|
561
|
+
submoduleName: string,
|
|
562
|
+
submodule: ModuleManifest,
|
|
563
|
+
): ValidationError | null {
|
|
564
|
+
const errors: Array<{ path: string; message: string }> = [];
|
|
565
|
+
const where = `${SUBMODULES_DIR}/${submoduleName}/manifest.yml`;
|
|
566
|
+
|
|
567
|
+
if (submodule.id !== submoduleName) {
|
|
568
|
+
errors.push({
|
|
569
|
+
path: `${where}#id`,
|
|
570
|
+
message: `Submodule id '${submodule.id}' does not match its directory '${submoduleName}'. A submodule is addressed by the name its parent declares, so the two must agree.`,
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
// D9: celilo has no general rule for resolving a capability name to one of
|
|
575
|
+
// several providers. The two bespoke rules that exist RESOLVE rather than
|
|
576
|
+
// fail — `firewall` builds a chain, `dns_registrar` takes the first row — so
|
|
577
|
+
// N instances registering one capability would silently bind a consumer to
|
|
578
|
+
// whichever was created first. Refused here rather than discovered there.
|
|
579
|
+
if ((submodule.provides?.capabilities ?? []).length > 0) {
|
|
580
|
+
const names = submodule.provides.capabilities.map((c) => c.name).join(', ');
|
|
581
|
+
errors.push({
|
|
582
|
+
path: `${where}#provides.capabilities`,
|
|
583
|
+
message: `A submodule may not provide a capability (declares: ${names}). A capability resolves to ONE provider, and a submodule exists as many instances, so a consumer would bind to whichever instance happened to be created first. Move the capability to '${parent.id}'.`,
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// One level only. D4's instance layout puts every instance flat under
|
|
588
|
+
// `modules/`, and nesting would make a derived id ambiguous about which
|
|
589
|
+
// ancestor owns it.
|
|
590
|
+
if ((submodule.submodules ?? []).length > 0) {
|
|
591
|
+
errors.push({
|
|
592
|
+
path: `${where}#submodules`,
|
|
593
|
+
message: `A submodule may not declare submodules of its own. Ownership is one level deep: '${parent.id}' owns '${submoduleName}', and nothing owns anything below that.`,
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
return errors.length > 0 ? { success: false, errors } : null;
|
|
598
|
+
}
|
|
@@ -171,6 +171,122 @@ version: 1.0.0
|
|
|
171
171
|
});
|
|
172
172
|
});
|
|
173
173
|
|
|
174
|
+
describe('submodules (openspec/changes/submodules D1)', () => {
|
|
175
|
+
const PARENT = `
|
|
176
|
+
celilo_contract: "1.0"
|
|
177
|
+
id: forgejo
|
|
178
|
+
name: Forgejo
|
|
179
|
+
version: 1.0.0
|
|
180
|
+
submodules:
|
|
181
|
+
- runner
|
|
182
|
+
`;
|
|
183
|
+
const SUBMODULE = `
|
|
184
|
+
celilo_contract: "1.0"
|
|
185
|
+
id: runner
|
|
186
|
+
name: Forgejo Runner
|
|
187
|
+
version: 1.0.0
|
|
188
|
+
`;
|
|
189
|
+
|
|
190
|
+
async function writeParent(dirName: string, parent: string, submodule?: string) {
|
|
191
|
+
const moduleDir = join(TEST_FIXTURES_DIR, dirName);
|
|
192
|
+
await mkdir(moduleDir, { recursive: true });
|
|
193
|
+
await writeFile(join(moduleDir, 'manifest.yml'), parent);
|
|
194
|
+
if (submodule !== undefined) {
|
|
195
|
+
const subDir = join(moduleDir, 'submodules', 'runner');
|
|
196
|
+
await mkdir(subDir, { recursive: true });
|
|
197
|
+
await writeFile(join(subDir, 'manifest.yml'), submodule);
|
|
198
|
+
}
|
|
199
|
+
return moduleDir;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
test('accepts a parent whose declared submodule is present and legal', async () => {
|
|
203
|
+
const dir = await writeParent('forgejo-ok', PARENT, SUBMODULE);
|
|
204
|
+
const result = await readModuleManifest(dir);
|
|
205
|
+
expect(result.success).toBe(true);
|
|
206
|
+
if (result.success) expect(result.manifest.submodules).toEqual(['runner']);
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
// The point of validating at PARENT import: an instantiation happens with
|
|
210
|
+
// no operator present, which is the worst moment to learn a manifest is
|
|
211
|
+
// broken.
|
|
212
|
+
test('rejects a parent declaring a submodule that does not exist', async () => {
|
|
213
|
+
const dir = await writeParent('forgejo-missing', PARENT);
|
|
214
|
+
const result = await readModuleManifest(dir);
|
|
215
|
+
expect(result.success).toBe(false);
|
|
216
|
+
if (!result.success) {
|
|
217
|
+
expect(result.error).toContain('submodules/runner/manifest.yml does not exist');
|
|
218
|
+
}
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test('rejects a parent whose submodule provides a capability', async () => {
|
|
222
|
+
const dir = await writeParent(
|
|
223
|
+
'forgejo-provides',
|
|
224
|
+
PARENT,
|
|
225
|
+
`${SUBMODULE}
|
|
226
|
+
provides:
|
|
227
|
+
capabilities:
|
|
228
|
+
- name: source_forge
|
|
229
|
+
version: 1.0.0
|
|
230
|
+
data: {}
|
|
231
|
+
`,
|
|
232
|
+
);
|
|
233
|
+
const result = await readModuleManifest(dir);
|
|
234
|
+
expect(result.success).toBe(false);
|
|
235
|
+
if (!result.success) expect(result.error).toContain('may not provide a capability');
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
test('rejects a submodule whose id disagrees with its directory', async () => {
|
|
239
|
+
const dir = await writeParent(
|
|
240
|
+
'forgejo-mismatch',
|
|
241
|
+
PARENT,
|
|
242
|
+
SUBMODULE.replace('id: runner', 'id: builder'),
|
|
243
|
+
);
|
|
244
|
+
const result = await readModuleManifest(dir);
|
|
245
|
+
expect(result.success).toBe(false);
|
|
246
|
+
if (!result.success) expect(result.error).toContain('does not match its directory');
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
test('a module declaring no submodules is unaffected', async () => {
|
|
250
|
+
const dir = await writeParent(
|
|
251
|
+
'plain-module',
|
|
252
|
+
`
|
|
253
|
+
celilo_contract: "1.0"
|
|
254
|
+
id: plain
|
|
255
|
+
name: Plain
|
|
256
|
+
version: 1.0.0
|
|
257
|
+
`,
|
|
258
|
+
);
|
|
259
|
+
const result = await readModuleManifest(dir);
|
|
260
|
+
expect(result.success).toBe(true);
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
// A submodule reaches the fleet inside its parent's package. The registry
|
|
264
|
+
// path is already closed (submodules are never published separately), so a
|
|
265
|
+
// local path is the only open door and this is what shuts it.
|
|
266
|
+
test('refuses to import a submodule directory on its own, naming the parent', async () => {
|
|
267
|
+
await writeParent('forgejo-direct', PARENT, SUBMODULE);
|
|
268
|
+
const submodulePath = join(TEST_FIXTURES_DIR, 'forgejo-direct', 'submodules', 'runner');
|
|
269
|
+
|
|
270
|
+
const error = validateModuleDirectory(submodulePath);
|
|
271
|
+
|
|
272
|
+
expect(error).not.toBeNull();
|
|
273
|
+
expect(error).toContain("is a submodule of 'forgejo-direct'");
|
|
274
|
+
expect(error).toContain('Import');
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
test('a trailing slash does not defeat the submodule refusal', async () => {
|
|
278
|
+
await writeParent('forgejo-slash', PARENT, SUBMODULE);
|
|
279
|
+
const submodulePath = `${join(TEST_FIXTURES_DIR, 'forgejo-slash', 'submodules', 'runner')}/`;
|
|
280
|
+
|
|
281
|
+
expect(validateModuleDirectory(submodulePath)).toContain('is a submodule of');
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
test('an ordinary module directory is still importable', async () => {
|
|
285
|
+
const dir = await writeParent('ordinary', PARENT, SUBMODULE);
|
|
286
|
+
expect(validateModuleDirectory(dir)).toBeNull();
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
|
|
174
290
|
describe('copyModuleFiles', () => {
|
|
175
291
|
test('should copy all files from source to target', async () => {
|
|
176
292
|
const sourceDir = join(TEST_FIXTURES_DIR, 'source');
|
package/src/module/import.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { execSync } from 'node:child_process';
|
|
2
2
|
import { existsSync, statSync, writeFileSync } from 'node:fs';
|
|
3
3
|
import { copyFile, mkdir, readFile, readdir } from 'node:fs/promises';
|
|
4
|
-
import { join, relative } from 'node:path';
|
|
4
|
+
import { basename, dirname, join, relative } from 'node:path';
|
|
5
5
|
import { eq } from 'drizzle-orm';
|
|
6
6
|
import { z } from 'zod';
|
|
7
7
|
import { getWellKnownCapability, isWellKnown } from '../capabilities/well-known';
|
|
@@ -12,12 +12,15 @@ import { capabilities, moduleIntegrity, modules } from '../db/schema';
|
|
|
12
12
|
import type { NewModule, NewModuleIntegrity } from '../db/schema';
|
|
13
13
|
import { type ModuleManifest, getSingularSystemSpec } from '../manifest/schema';
|
|
14
14
|
import {
|
|
15
|
+
SUBMODULES_DIR,
|
|
15
16
|
validateCapabilityNames,
|
|
16
17
|
validateDeriveFromSources,
|
|
17
18
|
validateHookContract,
|
|
18
19
|
validateManifest,
|
|
19
20
|
validatePrivilegedCapabilities,
|
|
20
21
|
validateProvidesNoCrossCapabilityRefs,
|
|
22
|
+
validateSubmoduleDeclaration,
|
|
23
|
+
validateSubmoduleManifest,
|
|
21
24
|
validateVariableSources,
|
|
22
25
|
validateZoneRequirements,
|
|
23
26
|
} from '../manifest/validate';
|
|
@@ -93,6 +96,17 @@ export function validateModuleDirectory(sourcePath: string): string | null {
|
|
|
93
96
|
return `Module directory does not exist: ${sourcePath}`;
|
|
94
97
|
}
|
|
95
98
|
|
|
99
|
+
// A submodule is not a module (design D1). It reaches the fleet inside its
|
|
100
|
+
// parent's package and exists only as instances the parent creates, so
|
|
101
|
+
// importing one on its own would produce a top-level module that was never
|
|
102
|
+
// written to be singular. The registry path is already closed — submodules
|
|
103
|
+
// are never published separately — so a local path is the only open door.
|
|
104
|
+
const parentDir = dirname(sourcePath.replace(/\/+$/, ''));
|
|
105
|
+
if (basename(parentDir) === SUBMODULES_DIR) {
|
|
106
|
+
const owner = basename(dirname(parentDir));
|
|
107
|
+
return `'${basename(sourcePath)}' is a submodule of '${owner}', not a module. Import '${owner}' instead — its package carries this submodule, and only '${owner}' can instantiate it.`;
|
|
108
|
+
}
|
|
109
|
+
|
|
96
110
|
// Check it's a directory
|
|
97
111
|
const stats = statSync(sourcePath);
|
|
98
112
|
if (!stats.isDirectory()) {
|
|
@@ -208,12 +222,70 @@ export async function readModuleManifest(
|
|
|
208
222
|
};
|
|
209
223
|
}
|
|
210
224
|
|
|
225
|
+
const submoduleCheck = await validateDeclaredSubmodules(sourcePath, validationResult.data);
|
|
226
|
+
if (submoduleCheck) {
|
|
227
|
+
return { success: false, error: submoduleCheck };
|
|
228
|
+
}
|
|
229
|
+
|
|
211
230
|
return {
|
|
212
231
|
success: true,
|
|
213
232
|
manifest: validationResult.data,
|
|
214
233
|
};
|
|
215
234
|
}
|
|
216
235
|
|
|
236
|
+
/**
|
|
237
|
+
* Read and validate every submodule a parent declares.
|
|
238
|
+
*
|
|
239
|
+
* Runs at PARENT IMPORT so a broken submodule fails the parent, rather than
|
|
240
|
+
* surfacing at the first instantiation — which happens with no operator
|
|
241
|
+
* present and is the worst possible moment to learn a manifest is wrong
|
|
242
|
+
* (design D1).
|
|
243
|
+
*
|
|
244
|
+
* @param sourcePath - The parent's module directory
|
|
245
|
+
* @param parent - The parent's validated manifest
|
|
246
|
+
* @returns An error message if any submodule is missing or illegal, null if clean
|
|
247
|
+
*/
|
|
248
|
+
async function validateDeclaredSubmodules(
|
|
249
|
+
sourcePath: string,
|
|
250
|
+
parent: ModuleManifest,
|
|
251
|
+
): Promise<string | null> {
|
|
252
|
+
const declarationCheck = validateSubmoduleDeclaration(parent);
|
|
253
|
+
if (declarationCheck) {
|
|
254
|
+
const errorMessages = declarationCheck.errors.map((e) => `${e.path}: ${e.message}`).join('\n');
|
|
255
|
+
return `Submodule declaration validation failed:\n${errorMessages}`;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
for (const name of parent.submodules ?? []) {
|
|
259
|
+
const submodulePath = join(sourcePath, SUBMODULES_DIR, name);
|
|
260
|
+
const manifestPath = join(submodulePath, 'manifest.yml');
|
|
261
|
+
|
|
262
|
+
if (!existsSync(manifestPath)) {
|
|
263
|
+
return `Module '${parent.id}' declares submodule '${name}', but ${SUBMODULES_DIR}/${name}/manifest.yml does not exist.`;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
let yamlContent: string;
|
|
267
|
+
try {
|
|
268
|
+
yamlContent = await readFile(manifestPath, 'utf-8');
|
|
269
|
+
} catch (error) {
|
|
270
|
+
return `Failed to read ${SUBMODULES_DIR}/${name}/manifest.yml: ${error instanceof Error ? error.message : 'Unknown error'}`;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const parsed = validateManifest(yamlContent);
|
|
274
|
+
if (!parsed.success) {
|
|
275
|
+
const errorMessages = parsed.errors.map((e) => `${e.path}: ${e.message}`).join(', ');
|
|
276
|
+
return `Submodule '${name}' has an invalid manifest: ${errorMessages}`;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const submoduleCheck = validateSubmoduleManifest(parent, name, parsed.data);
|
|
280
|
+
if (submoduleCheck) {
|
|
281
|
+
const errorMessages = submoduleCheck.errors.map((e) => `${e.path}: ${e.message}`).join('\n');
|
|
282
|
+
return `Submodule '${name}' validation failed:\n${errorMessages}`;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
|
|
217
289
|
/**
|
|
218
290
|
* Copy module files to target directory
|
|
219
291
|
*
|
|
@@ -2,9 +2,15 @@ import { existsSync } from 'node:fs';
|
|
|
2
2
|
import { readdir } from 'node:fs/promises';
|
|
3
3
|
import { join, relative } from 'node:path';
|
|
4
4
|
import { eq } from 'drizzle-orm';
|
|
5
|
-
import { getDb } from '../../db/client';
|
|
5
|
+
import { type DbClient, getDb } from '../../db/client';
|
|
6
6
|
import { moduleIntegrity, modules } from '../../db/schema';
|
|
7
7
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
8
|
+
import {
|
|
9
|
+
type InstanceRecord,
|
|
10
|
+
loadInstance,
|
|
11
|
+
submoduleSourcePath,
|
|
12
|
+
verifyInstanceLinks,
|
|
13
|
+
} from '../../services/module-instances';
|
|
8
14
|
import { computeFileChecksum } from './checksum';
|
|
9
15
|
import type { IntegrityViolation } from './extract';
|
|
10
16
|
import { compareVerbatimRoleAssets, readVerbatimRoleAssets } from './generated-plane';
|
|
@@ -49,6 +55,21 @@ async function scanDirectory(dir: string, baseDir: string): Promise<string[]> {
|
|
|
49
55
|
const fullPath = join(dir, entry.name);
|
|
50
56
|
const relativePath = relative(baseDir, fullPath);
|
|
51
57
|
|
|
58
|
+
// A symlink is SKIPPED, deliberately and in writing.
|
|
59
|
+
//
|
|
60
|
+
// An instance's authored source is symlinked in from its parent's
|
|
61
|
+
// `submodules/` tree (openspec/changes/submodules D4), so those bytes are
|
|
62
|
+
// the PARENT's and are covered by the parent's baseline. This module has no
|
|
63
|
+
// claim to make over them, and following the link would have it claim
|
|
64
|
+
// integrity over bytes whose checksums belong to somebody else.
|
|
65
|
+
//
|
|
66
|
+
// This already happened, by accident: `Dirent.isDirectory()` and
|
|
67
|
+
// `isFile()` are BOTH false for a symlink, so links fell through both
|
|
68
|
+
// branches below and vanished. Right answer, no reason. Stating it here
|
|
69
|
+
// means a later change that makes the walk follow links has to decide
|
|
70
|
+
// about instances on purpose rather than reverse this silently.
|
|
71
|
+
if (entry.isSymbolicLink()) continue;
|
|
72
|
+
|
|
52
73
|
if (entry.isDirectory()) {
|
|
53
74
|
// Prune whole derived subtrees rather than walking them. `generated/`
|
|
54
75
|
// alone carries terraform provider binaries.
|
|
@@ -91,6 +112,18 @@ export async function auditModule(
|
|
|
91
112
|
};
|
|
92
113
|
}
|
|
93
114
|
|
|
115
|
+
// An instance has no integrity row of its own and never will
|
|
116
|
+
// (openspec/changes/submodules D4). Its authored bytes are its
|
|
117
|
+
// submodule's, its submodule's are its parent's, and the parent already
|
|
118
|
+
// carries a baseline covering `submodules/**`. So it resolves THROUGH to
|
|
119
|
+
// that rather than being excluded: excluding it would report nothing for
|
|
120
|
+
// forty rows, and erroring would report `No integrity data found` forty
|
|
121
|
+
// times for a fleet that is entirely healthy.
|
|
122
|
+
const instance = loadInstance(moduleId, db);
|
|
123
|
+
if (instance) {
|
|
124
|
+
return await auditInstance(instance, module.sourcePath, db, options);
|
|
125
|
+
}
|
|
126
|
+
|
|
94
127
|
// Get integrity data
|
|
95
128
|
const integrity = db
|
|
96
129
|
.select()
|
|
@@ -242,3 +275,72 @@ export async function auditModule(
|
|
|
242
275
|
};
|
|
243
276
|
}
|
|
244
277
|
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Audit an instance.
|
|
281
|
+
*
|
|
282
|
+
* Two questions, and only the second is the instance's own.
|
|
283
|
+
*
|
|
284
|
+
* Its BYTES belong to its parent, so the integrity verdict is the parent's:
|
|
285
|
+
* `submodules/**` classifies `package`, so the parent's baseline already covers
|
|
286
|
+
* every file an instance runs. Re-checksumming them here would be asking the
|
|
287
|
+
* same question a second time and inventing a second answer that can disagree.
|
|
288
|
+
*
|
|
289
|
+
* Its LINKS are its own, and nothing else in celilo checks them. A link
|
|
290
|
+
* repointed at another module's source is an instance running somebody else's
|
|
291
|
+
* code while every checksum in the fleet still reconciles, which is exactly the
|
|
292
|
+
* kind of quiet wrong answer the integrity plane exists to prevent.
|
|
293
|
+
*/
|
|
294
|
+
async function auditInstance(
|
|
295
|
+
instance: InstanceRecord,
|
|
296
|
+
instancePath: string,
|
|
297
|
+
db: DbClient,
|
|
298
|
+
options: AuditOptions,
|
|
299
|
+
): Promise<AuditResult> {
|
|
300
|
+
const violations: IntegrityViolation[] = [];
|
|
301
|
+
|
|
302
|
+
const parent = db.select().from(modules).where(eq(modules.id, instance.parentId)).get();
|
|
303
|
+
if (!parent) {
|
|
304
|
+
// The FK makes this unreachable through SQL, so reaching it means the row
|
|
305
|
+
// was written around the schema. Reported rather than assumed away.
|
|
306
|
+
return {
|
|
307
|
+
success: false,
|
|
308
|
+
moduleId: instance.moduleId,
|
|
309
|
+
violations,
|
|
310
|
+
error: `Instance '${instance.moduleId}' names parent '${instance.parentId}', which does not exist.`,
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const submodulePath = submoduleSourcePath(parent.sourcePath, instance.submodule);
|
|
315
|
+
for (const link of await verifyInstanceLinks({ instancePath, submodulePath })) {
|
|
316
|
+
violations.push({
|
|
317
|
+
type: 'modified',
|
|
318
|
+
path: link.entry,
|
|
319
|
+
message:
|
|
320
|
+
link.actual === null
|
|
321
|
+
? `Link '${link.entry}' is missing or is not a link. It should point at ${link.expected}. Redeploy this instance to rebuild its links.`
|
|
322
|
+
: `Link '${link.entry}' points at ${link.actual}, not at ${link.expected}. This instance is running source it does not own.`,
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// The parent's verdict, carried rather than recomputed. A parent whose
|
|
327
|
+
// baseline is stale or whose files are modified means every instance under it
|
|
328
|
+
// is running unverified code, and saying so here is what stops an operator
|
|
329
|
+
// reading a clean instance row as evidence.
|
|
330
|
+
const parentResult = await auditModule(instance.parentId, db, options);
|
|
331
|
+
if (!parentResult.success) {
|
|
332
|
+
violations.push({
|
|
333
|
+
type: 'stale-baseline',
|
|
334
|
+
path: `${instance.parentId}`,
|
|
335
|
+
message: `The bytes this instance runs belong to '${instance.parentId}', whose own audit is not clean (${parentResult.violations.length} violation(s)${parentResult.error ? `: ${parentResult.error}` : ''}). Fix the parent; this instance cannot be verified independently.`,
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
return {
|
|
340
|
+
success: violations.length === 0,
|
|
341
|
+
moduleId: instance.moduleId,
|
|
342
|
+
violations,
|
|
343
|
+
moduleVersion: parent.version,
|
|
344
|
+
baselineVersion: parentResult.baselineVersion,
|
|
345
|
+
};
|
|
346
|
+
}
|
|
@@ -102,3 +102,39 @@ describe('classifyModulePath: composes with includeNodeModulesPath', () => {
|
|
|
102
102
|
expect(classifyModulePath('node_modules/@celilo/cli/index.js')).toBe('unknown');
|
|
103
103
|
});
|
|
104
104
|
});
|
|
105
|
+
|
|
106
|
+
describe('a submodule tree obeys the same rules as a module tree', () => {
|
|
107
|
+
// openspec/changes/submodules D1: a submodule declares everything an ordinary
|
|
108
|
+
// module declares, so it must be CLASSIFIED like one too. Every rule keys on
|
|
109
|
+
// segments[0], so without the recursion a nested tree gets different answers
|
|
110
|
+
// than the same file one level up.
|
|
111
|
+
test('authored source under a submodule is package', () => {
|
|
112
|
+
expect(classifyModulePath('submodules/runner/manifest.yml')).toBe('package');
|
|
113
|
+
expect(classifyModulePath('submodules/runner/ansible/site.yml')).toBe('package');
|
|
114
|
+
expect(classifyModulePath('submodules/runner/scripts/hook.ts')).toBe('package');
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("a submodule's e2e tree is excluded, exactly as a module's is", () => {
|
|
118
|
+
expect(classifyModulePath('e2e/thing.ts')).toBe('unknown');
|
|
119
|
+
expect(classifyModulePath('submodules/runner/e2e/thing.ts')).toBe('unknown');
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("a submodule's derived paths are derived", () => {
|
|
123
|
+
expect(classifyModulePath('submodules/runner/generated/terraform/x.tfstate')).toBe('derived');
|
|
124
|
+
expect(classifyModulePath('submodules/runner/celilo/types.d.ts')).toBe('derived');
|
|
125
|
+
expect(classifyModulePath('submodules/runner/cookies.json')).toBe('derived');
|
|
126
|
+
expect(classifyModulePath('submodules/runner/screenshots/a.png')).toBe('derived');
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test('source-tree noise under a submodule is still a finding', () => {
|
|
130
|
+
expect(classifyModulePath('submodules/runner/.DS_Store')).toBe('unknown');
|
|
131
|
+
expect(classifyModulePath('submodules/runner/tsconfig.json')).toBe('unknown');
|
|
132
|
+
expect(classifyModulePath('submodules/runner/a.test.ts')).toBe('unknown');
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
// The directories themselves are shipped bytes and belong in the baseline.
|
|
136
|
+
test('the submodules directory itself is package', () => {
|
|
137
|
+
expect(classifyModulePath('submodules')).toBe('package');
|
|
138
|
+
expect(classifyModulePath('submodules/runner')).toBe('package');
|
|
139
|
+
});
|
|
140
|
+
});
|