@celilo/cli 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +3 -1
- package/CELILO_SUBSYSTEMS.md +7 -1
- package/MODULE_PRIMITIVES.md +6 -1
- package/drizzle/0027_dns_internal_records_consumer_cascade.sql +43 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/src/capabilities/lookup.ts +39 -29
- package/src/capabilities/secret-ref.test.ts +24 -0
- package/src/capabilities/secret-validation.ts +50 -0
- package/src/capabilities/validation.test.ts +238 -2
- package/src/capabilities/validation.ts +67 -1
- package/src/cli/commands/alerts-sweep.ts +18 -0
- package/src/cli/commands/module-remove.ts +34 -2
- package/src/cli/commands/module-update.test.ts +149 -2
- package/src/cli/commands/module-update.ts +113 -25
- package/src/cli/commands/service-set-credentials.test.ts +108 -0
- package/src/cli/commands/service-set-credentials.ts +115 -0
- package/src/cli/commands/system-migrate.ts +6 -4
- package/src/cli/completion.ts +16 -1
- package/src/cli/index.ts +9 -0
- package/src/db/client.ts +10 -8
- package/src/db/dns-internal-cascade-migration.test.ts +184 -0
- package/src/db/migrate.test.ts +147 -0
- package/src/db/migrate.ts +69 -1
- package/src/db/schema.ts +21 -4
- package/src/hooks/capability-loader.test.ts +55 -0
- package/src/hooks/capability-loader.ts +16 -1
- package/src/manifest/template-validator.test.ts +47 -0
- package/src/manifest/template-validator.ts +18 -1
- package/src/module/import.ts +39 -6
- package/src/policy/capability-shape-baseline.ts +88 -0
- package/src/policy/capability-shape-drift.test.ts +162 -0
- package/src/policy/capability-shape.ts +117 -0
- package/src/policy/dns-aspect-coverage.test.ts +100 -0
- package/src/policy/module-business-baseline.ts +32 -18
- package/src/services/alerting/monitors.ts +54 -2
- package/src/services/alerting/sweep-runner.ts +38 -1
- package/src/services/capability-table-rows.test.ts +191 -0
- package/src/services/capability-table-rows.ts +103 -0
- package/src/services/consumer-cleanup.ts +18 -10
- package/src/services/container-service.test.ts +34 -0
- package/src/services/container-service.ts +44 -0
- package/src/services/deployed-systems.test.ts +101 -0
- package/src/services/deployed-systems.ts +43 -11
- package/src/services/dns-internal-records.test.ts +72 -1
- package/src/services/dns-provider-backfill.ts +30 -0
- package/src/services/fleet-checks.test.ts +26 -0
- package/src/services/fleet-checks.ts +11 -1
- package/src/services/module-deploy.ts +88 -41
- package/src/services/module-validator/capability-versions.test.ts +6 -1
- package/src/services/port-forwards.test.ts +6 -2
- package/src/services/port-forwards.ts +0 -11
- package/src/services/provider-arrival.test.ts +241 -0
- package/src/services/provider-arrival.ts +213 -0
- package/src/services/trusted-sources.ts +0 -5
- package/src/templates/generator.test.ts +35 -0
- package/src/templates/generator.ts +29 -1
- package/src/variables/context.test.ts +63 -0
- package/src/variables/context.ts +85 -12
- package/src/variables/declarative-derivation.test.ts +47 -8
- package/src/variables/declarative-derivation.ts +6 -4
- package/src/variables/lxc-nameserver.test.ts +144 -0
- package/src/services/public-web-republish.test.ts +0 -189
- package/src/services/public-web-republish.ts +0 -84
|
@@ -214,7 +214,59 @@ export function ensureInboundSubscriber(bus: SubscriberRegistrar): void {
|
|
|
214
214
|
});
|
|
215
215
|
}
|
|
216
216
|
|
|
217
|
-
|
|
218
|
-
|
|
217
|
+
/** A live alert about to be deleted along with the monitor that owns it. */
|
|
218
|
+
export interface DroppedAlert {
|
|
219
|
+
key: string;
|
|
220
|
+
message: string;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Delete a monitor, and report the live alerts that go with it.
|
|
225
|
+
*
|
|
226
|
+
* The alerts are DELETED, not resolved. `alerts.monitorId` is
|
|
227
|
+
* `on delete cascade`, so the row goes the moment the monitor does. This used
|
|
228
|
+
* to call `resolveMonitorAlerts` first; that write was unreachable — the very
|
|
229
|
+
* next statement dropped the same rows, and nothing read them in between, so
|
|
230
|
+
* the resolved state existed for the duration of one statement. Deleted rather
|
|
231
|
+
* than restored, because resolving properly would need the alert rows to
|
|
232
|
+
* outlive their monitor, which the FK forbids and which no reader wants:
|
|
233
|
+
* `resolvedAt` is only ever read as a liveness predicate (`ack.ts`), never
|
|
234
|
+
* reported. Do not put the call back without changing the FK first.
|
|
235
|
+
*
|
|
236
|
+
* What IS worth keeping is what was lost, which is why the live alerts are
|
|
237
|
+
* returned. A monitor dropped while holding a firing alert takes a real
|
|
238
|
+
* failure and the coverage of it away together, and a caller that only names
|
|
239
|
+
* the monitor cannot tell an operator what stopped being watched.
|
|
240
|
+
*/
|
|
241
|
+
export function deleteMonitor(db: DbClient, monitorId: string): DroppedAlert[] {
|
|
242
|
+
const dropped = db
|
|
243
|
+
.select({ key: alerts.key, message: alerts.message })
|
|
244
|
+
.from(alerts)
|
|
245
|
+
.where(and(eq(alerts.monitorId, monitorId), isNotNull(alerts.activeKey)))
|
|
246
|
+
.all();
|
|
247
|
+
|
|
219
248
|
db.delete(monitors).where(eq(monitors.id, monitorId)).run();
|
|
249
|
+
return dropped;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Drop the `module_hook` monitor belonging to a module that is going away.
|
|
254
|
+
*
|
|
255
|
+
* Called from the removal path rather than expressed as a foreign key, because
|
|
256
|
+
* `monitors.target` cannot carry one: it holds a module id for `module_hook`
|
|
257
|
+
* and an audit check name for `builtin_check`, and SQLite has no conditional
|
|
258
|
+
* reference. Splitting the table to get the constraint would need a synthetic
|
|
259
|
+
* monitor identity for `alerts.monitorId` — the trade the file header already
|
|
260
|
+
* weighs and declines.
|
|
261
|
+
*
|
|
262
|
+
* Deleting cascades the monitor's alerts away (`alerts.monitorId` is
|
|
263
|
+
* `on delete cascade`), which is what releases anything they were suppressing.
|
|
264
|
+
* They are deleted, not resolved — see `deleteMonitor`. Returns what was live
|
|
265
|
+
* so the caller can say what stopped being watched, or null if the module had
|
|
266
|
+
* no monitor.
|
|
267
|
+
*/
|
|
268
|
+
export function deleteMonitorForModule(db: DbClient, moduleId: string): DroppedAlert[] | null {
|
|
269
|
+
const monitor = findMonitor(db, 'module_hook', moduleId);
|
|
270
|
+
if (!monitor) return null;
|
|
271
|
+
return deleteMonitor(db, monitor.id);
|
|
220
272
|
}
|
|
@@ -23,6 +23,7 @@ import type { DbClient } from '../../db/client';
|
|
|
23
23
|
import type { Alert, Monitor } from '../../db/schema';
|
|
24
24
|
import { MONITOR_INTERVAL_FLOOR_MINUTES } from '../cadence';
|
|
25
25
|
import { isScheduled, loadModuleHealthCadences } from './health-cadence';
|
|
26
|
+
import { type DroppedAlert, deleteMonitor } from './monitors';
|
|
26
27
|
import type { NotifyDeps, NotifyOutcome } from './notifier';
|
|
27
28
|
import { deliverDeferred, notifyAlert } from './notifier';
|
|
28
29
|
import { type MonitorRunDeps, runOneMonitor } from './run-monitor';
|
|
@@ -102,6 +103,19 @@ export interface SweepReport {
|
|
|
102
103
|
* former, because nothing ever reached the daemon.
|
|
103
104
|
*/
|
|
104
105
|
failures: string[];
|
|
106
|
+
/**
|
|
107
|
+
* Modules whose `module_hook` monitor was dropped because the module itself
|
|
108
|
+
* is gone, and the live alerts each one took with it.
|
|
109
|
+
*
|
|
110
|
+
* The alerts are DELETED by the monitor's cascade, not resolved, so a firing
|
|
111
|
+
* check and the coverage of it disappear in the same instant with nothing
|
|
112
|
+
* else recording either. A bare module name is not enough to act on: the row
|
|
113
|
+
* that mattered on the fleet was holding `Cannot reach router: Router login
|
|
114
|
+
* failed`, and `1 stranded-dropped (greenwave)` would not have told anyone
|
|
115
|
+
* that a router had stopped being checked. Same reason `noPolicy`, `skipped`
|
|
116
|
+
* and `failures` all name their subject rather than counting it.
|
|
117
|
+
*/
|
|
118
|
+
strandedDropped: { moduleId: string; alerts: DroppedAlert[] }[];
|
|
105
119
|
}
|
|
106
120
|
|
|
107
121
|
/**
|
|
@@ -129,6 +143,7 @@ export async function runSweep(
|
|
|
129
143
|
noPolicy: [],
|
|
130
144
|
skipped: [],
|
|
131
145
|
failures: [],
|
|
146
|
+
strandedDropped: [],
|
|
132
147
|
};
|
|
133
148
|
|
|
134
149
|
// 1. Run due monitors.
|
|
@@ -139,8 +154,30 @@ export async function runSweep(
|
|
|
139
154
|
// manifest could never reach an existing install (design.md D2/D8). A
|
|
140
155
|
// `builtin_check` has no module and no manifest, so its row is the config.
|
|
141
156
|
const cadences = loadModuleHealthCadences(db);
|
|
157
|
+
|
|
158
|
+
// Reconcile away monitors whose module is gone, before anything tries to run
|
|
159
|
+
// them. `loadModuleHealthCadences` holds every module, so a miss here means
|
|
160
|
+
// no module row — and a `module_hook` monitor without one is unschedulable
|
|
161
|
+
// forever: its cadence resolves to null, `isScheduled` says false, and the
|
|
162
|
+
// sweep never selects it again. Anything it left firing would sit there with
|
|
163
|
+
// no path back to `resolved`, suppressing every alert it is an ancestor of
|
|
164
|
+
// (celilo#1029).
|
|
165
|
+
//
|
|
166
|
+
// The removal path deletes the monitor itself, so this only fires for rows a
|
|
167
|
+
// celilo without that fix left behind, or a removal that died between the two
|
|
168
|
+
// deletes. Same stance `setMonitorEnabled` already takes: a monitor that will
|
|
169
|
+
// never report again must not hold live alerts.
|
|
170
|
+
const stranded = monitors.filter((m) => m.kind === 'module_hook' && !cadences.has(m.target));
|
|
171
|
+
for (const monitor of stranded) {
|
|
172
|
+
report.strandedDropped.push({
|
|
173
|
+
moduleId: monitor.target,
|
|
174
|
+
alerts: deleteMonitor(db, monitor.id),
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
const active = stranded.length > 0 ? monitors.filter((m) => !stranded.includes(m)) : monitors;
|
|
178
|
+
|
|
142
179
|
const due = selectDueMonitors(
|
|
143
|
-
|
|
180
|
+
active.map((m) => {
|
|
144
181
|
if (m.kind !== 'module_hook') {
|
|
145
182
|
return {
|
|
146
183
|
id: m.id,
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ONE consumer-removal rule, driven by the declarations
|
|
3
|
+
* (openspec/changes/capability-owned-tables task 2.4).
|
|
4
|
+
*
|
|
5
|
+
* Before this, "the row dies with its consumer" had five implementations for one
|
|
6
|
+
* rule, and two of them were functions core imported BY NAME from
|
|
7
|
+
* `consumer-cleanup.ts` — the file whose whole purpose is deleting exactly that
|
|
8
|
+
* kind of per-capability special-casing.
|
|
9
|
+
*
|
|
10
|
+
* The property worth protecting is not "port forwards get deleted". It is that
|
|
11
|
+
* core clears EVERY declared table without naming any of them, so declaring a
|
|
12
|
+
* new one is sufficient to have it cleaned up.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
16
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
17
|
+
import { tmpdir } from 'node:os';
|
|
18
|
+
import { join } from 'node:path';
|
|
19
|
+
import { allDeclaredTables } from '@celilo/capabilities';
|
|
20
|
+
import type { DbClient } from '../db/client';
|
|
21
|
+
import { dnsInternalRecords, modules, portForwards, trustedSources, webRoutes } from '../db/schema';
|
|
22
|
+
import { setupTestDatabase } from '../test-utils/setup-test-db';
|
|
23
|
+
import { deleteClaimedRows, planClaimedRowDeletion } from './capability-table-rows';
|
|
24
|
+
|
|
25
|
+
const FW = '192.168.0.254';
|
|
26
|
+
|
|
27
|
+
describe('planClaimedRowDeletion', () => {
|
|
28
|
+
/**
|
|
29
|
+
* The anti-regression that matters. If someone declares a table and this plan
|
|
30
|
+
* does not grow, their rows outlive the consumer silently — which is the whole
|
|
31
|
+
* failure the declaration exists to make impossible.
|
|
32
|
+
*/
|
|
33
|
+
it('covers every declared table, so declaring one is enough to have it cleared', () => {
|
|
34
|
+
const planned = new Set(planClaimedRowDeletion().map((t) => t.table));
|
|
35
|
+
const declared = allDeclaredTables().map((d) => d.declaration.table);
|
|
36
|
+
|
|
37
|
+
expect(declared.length).toBeGreaterThan(0);
|
|
38
|
+
for (const table of declared) expect(planned.has(table)).toBe(true);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("carries each table's OWN claim column, which is spelled three ways", () => {
|
|
42
|
+
const byTable = new Map(planClaimedRowDeletion().map((t) => [t.table, t.claimColumn]));
|
|
43
|
+
|
|
44
|
+
expect(byTable.get('web_routes')).toBe('module_id');
|
|
45
|
+
expect(byTable.get('port_forwards')).toBe('registered_by');
|
|
46
|
+
expect(byTable.get('trusted_sources')).toBe('registered_by');
|
|
47
|
+
expect(byTable.get('dns_internal_records')).toBe('consumer_module_id');
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
describe('deleteClaimedRows', () => {
|
|
52
|
+
let dir: string;
|
|
53
|
+
let db: DbClient;
|
|
54
|
+
|
|
55
|
+
beforeEach(async () => {
|
|
56
|
+
dir = mkdtempSync(join(tmpdir(), 'claimed-'));
|
|
57
|
+
const dbPath = join(dir, 'celilo.db');
|
|
58
|
+
process.env.CELILO_DB_PATH = dbPath;
|
|
59
|
+
db = await setupTestDatabase(dbPath);
|
|
60
|
+
|
|
61
|
+
for (const id of ['caddy', 'forgejo', 'technitium']) {
|
|
62
|
+
db.insert(modules)
|
|
63
|
+
.values({
|
|
64
|
+
id,
|
|
65
|
+
name: id,
|
|
66
|
+
version: '1.0.0',
|
|
67
|
+
state: 'VERIFIED',
|
|
68
|
+
sourcePath: `/tmp/${id}`,
|
|
69
|
+
manifestData: {},
|
|
70
|
+
})
|
|
71
|
+
.run();
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
afterEach(() => {
|
|
76
|
+
db.$client.close();
|
|
77
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
78
|
+
rmSync(dir, { recursive: true, force: true });
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
function seed(consumer: string, hostname: string, port: number, subnet: string) {
|
|
82
|
+
db.insert(webRoutes)
|
|
83
|
+
.values({
|
|
84
|
+
slug: `${consumer}-slug`,
|
|
85
|
+
moduleId: consumer,
|
|
86
|
+
type: 'reverse_proxy',
|
|
87
|
+
path: `/${consumer}`,
|
|
88
|
+
hostname,
|
|
89
|
+
})
|
|
90
|
+
.run();
|
|
91
|
+
db.insert(portForwards)
|
|
92
|
+
.values({
|
|
93
|
+
firewallIp: FW,
|
|
94
|
+
internalIp: '10.0.20.5',
|
|
95
|
+
port,
|
|
96
|
+
protocol: 'TCP',
|
|
97
|
+
registeredBy: consumer,
|
|
98
|
+
})
|
|
99
|
+
.run();
|
|
100
|
+
db.insert(trustedSources).values({ firewallIp: FW, subnet, registeredBy: consumer }).run();
|
|
101
|
+
db.insert(dnsInternalRecords)
|
|
102
|
+
.values({
|
|
103
|
+
providerModuleId: 'technitium',
|
|
104
|
+
consumerModuleId: consumer,
|
|
105
|
+
host: hostname,
|
|
106
|
+
ip: '10.0.20.5',
|
|
107
|
+
})
|
|
108
|
+
.run();
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
it('clears the departing consumer from all four declared tables at once', () => {
|
|
112
|
+
seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
|
|
113
|
+
|
|
114
|
+
const cleared = deleteClaimedRows(db, 'caddy');
|
|
115
|
+
|
|
116
|
+
expect(cleared.map((c) => c.table).sort()).toEqual([
|
|
117
|
+
'dns_internal_records',
|
|
118
|
+
'port_forwards',
|
|
119
|
+
'trusted_sources',
|
|
120
|
+
'web_routes',
|
|
121
|
+
]);
|
|
122
|
+
expect(db.select().from(webRoutes).all()).toHaveLength(0);
|
|
123
|
+
expect(db.select().from(portForwards).all()).toHaveLength(0);
|
|
124
|
+
expect(db.select().from(trustedSources).all()).toHaveLength(0);
|
|
125
|
+
expect(db.select().from(dnsInternalRecords).all()).toHaveLength(0);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* The refcount case, the bug most likely to ship silently. Two consumers each
|
|
130
|
+
* hold their own row, and one leaving must not withdraw what the other still
|
|
131
|
+
* needs.
|
|
132
|
+
*/
|
|
133
|
+
it("leaves another consumer's rows untouched", () => {
|
|
134
|
+
seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
|
|
135
|
+
seed('forgejo', 'b.example.org', 2222, '10.2.0.0/24');
|
|
136
|
+
|
|
137
|
+
deleteClaimedRows(db, 'caddy');
|
|
138
|
+
|
|
139
|
+
expect(
|
|
140
|
+
db
|
|
141
|
+
.select()
|
|
142
|
+
.from(webRoutes)
|
|
143
|
+
.all()
|
|
144
|
+
.map((r) => r.moduleId),
|
|
145
|
+
).toEqual(['forgejo']);
|
|
146
|
+
expect(
|
|
147
|
+
db
|
|
148
|
+
.select()
|
|
149
|
+
.from(portForwards)
|
|
150
|
+
.all()
|
|
151
|
+
.map((r) => r.registeredBy),
|
|
152
|
+
).toEqual(['forgejo']);
|
|
153
|
+
expect(
|
|
154
|
+
db
|
|
155
|
+
.select()
|
|
156
|
+
.from(trustedSources)
|
|
157
|
+
.all()
|
|
158
|
+
.map((r) => r.registeredBy),
|
|
159
|
+
).toEqual(['forgejo']);
|
|
160
|
+
expect(
|
|
161
|
+
db
|
|
162
|
+
.select()
|
|
163
|
+
.from(dnsInternalRecords)
|
|
164
|
+
.all()
|
|
165
|
+
.map((r) => r.consumerModuleId),
|
|
166
|
+
).toEqual(['forgejo']);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* `dns_internal_records` is claimed by its CONSUMER, never its provider. The
|
|
171
|
+
* declaration cannot express anything else, which is what makes celilo#1010
|
|
172
|
+
* unrepresentable — but the runtime rule has to agree, or the declaration is
|
|
173
|
+
* decoration.
|
|
174
|
+
*/
|
|
175
|
+
it('does not treat the PROVIDER as the claimant of a dns_internal record', () => {
|
|
176
|
+
seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
|
|
177
|
+
|
|
178
|
+
deleteClaimedRows(db, 'technitium');
|
|
179
|
+
|
|
180
|
+
expect(db.select().from(dnsInternalRecords).all()).toHaveLength(1);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it('is a no-op for a module that claimed nothing', () => {
|
|
184
|
+
seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
|
|
185
|
+
|
|
186
|
+
deleteClaimedRows(db, 'forgejo');
|
|
187
|
+
|
|
188
|
+
expect(db.select().from(webRoutes).all()).toHaveLength(1);
|
|
189
|
+
expect(db.select().from(portForwards).all()).toHaveLength(1);
|
|
190
|
+
});
|
|
191
|
+
});
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Acting on every capability-declared table without naming any of them
|
|
3
|
+
* (openspec/changes/capability-owned-tables, design D3).
|
|
4
|
+
*
|
|
5
|
+
* "This row dies with its consumer" had five implementations for one rule: an FK
|
|
6
|
+
* cascade for `web_routes`, two functions core imported BY NAME from
|
|
7
|
+
* `consumer-cleanup.ts` for `port_forwards` and `trusted_sources`, a second
|
|
8
|
+
* table pruned on read for `dns_registrations`, a cascade on both provider and
|
|
9
|
+
* consumer for `dns_internal_records`, and a hook rewriting a JSON blob for the
|
|
10
|
+
* rest. `consumer-cleanup.ts` exists specifically to delete that kind of
|
|
11
|
+
* per-capability special-casing. It succeeded for the hook dispatch and then
|
|
12
|
+
* grew two hardcoded imports for the rows, because the rows had no generic rule.
|
|
13
|
+
*
|
|
14
|
+
* This is the generic rule. The declaration names the claim column, so core no
|
|
15
|
+
* longer has to know that three tables spell it `module_id`, `registered_by` and
|
|
16
|
+
* `consumer_module_id`.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { CAPABILITY_DECLARED_TABLES, allDeclaredTables } from '@celilo/capabilities';
|
|
20
|
+
import { type Column, eq, is } from 'drizzle-orm';
|
|
21
|
+
import { SQLiteTable, getTableConfig } from 'drizzle-orm/sqlite-core';
|
|
22
|
+
import type { DbClient } from '../db/client';
|
|
23
|
+
import * as dbSchema from '../db/schema';
|
|
24
|
+
|
|
25
|
+
export interface ClaimedRowTarget {
|
|
26
|
+
capability: string;
|
|
27
|
+
/** Physical table name. */
|
|
28
|
+
table: string;
|
|
29
|
+
/** The column holding the consumer's module id. */
|
|
30
|
+
claimColumn: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Physical table name → the drizzle table object, for the tables celilo can
|
|
35
|
+
* actually act on. Built by walking the schema rather than by a hand-written
|
|
36
|
+
* map, so a table cannot be declared and then silently unreachable.
|
|
37
|
+
*/
|
|
38
|
+
function schemaTables(): Map<string, SQLiteTable> {
|
|
39
|
+
const out = new Map<string, SQLiteTable>();
|
|
40
|
+
for (const value of Object.values(dbSchema)) {
|
|
41
|
+
if (!is(value, SQLiteTable)) continue;
|
|
42
|
+
out.set(getTableConfig(value).name, value);
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* What a consumer's removal must clear. Pure — the list is a property of the
|
|
49
|
+
* declarations, so it is worth being able to assert without a database.
|
|
50
|
+
*
|
|
51
|
+
* A declaration naming a table that does not exist is skipped here rather than
|
|
52
|
+
* throwing. `policy/capability-shape-drift.test.ts` Scan A is what fails on
|
|
53
|
+
* that, loudly and at build time; a removal is the wrong moment to discover it,
|
|
54
|
+
* and refusing to clear the OTHER tables because one declaration is wrong would
|
|
55
|
+
* strand real rows.
|
|
56
|
+
*/
|
|
57
|
+
export function planClaimedRowDeletion(): ClaimedRowTarget[] {
|
|
58
|
+
const tables = schemaTables();
|
|
59
|
+
return allDeclaredTables()
|
|
60
|
+
.filter(({ declaration }) => tables.has(declaration.table))
|
|
61
|
+
.map(({ capability, declaration }) => ({
|
|
62
|
+
capability,
|
|
63
|
+
table: declaration.table,
|
|
64
|
+
claimColumn: declaration.claim.column,
|
|
65
|
+
}))
|
|
66
|
+
.sort((a, b) => a.table.localeCompare(b.table));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Delete every declared row this consumer claimed.
|
|
71
|
+
*
|
|
72
|
+
* Called AFTER every provider has been told the consumer is going (its D4): a
|
|
73
|
+
* provider's `on_consumer_removed` is a full converge, and converging from a set
|
|
74
|
+
* the rows had already been removed from would withdraw more than the departing
|
|
75
|
+
* consumer's share.
|
|
76
|
+
*
|
|
77
|
+
* `web_routes` and `dns_internal_records` also carry an FK cascade, so their
|
|
78
|
+
* rows would go anyway once the `modules` row is deleted. Doing it here as well
|
|
79
|
+
* is deliberate and not merely harmless: it makes the ORDER explicit rather than
|
|
80
|
+
* a consequence of when some other statement happens to run, and it is the same
|
|
81
|
+
* point in the sequence at which `port_forwards` and `trusted_sources` were
|
|
82
|
+
* already being cleared by hand.
|
|
83
|
+
*/
|
|
84
|
+
export function deleteClaimedRows(db: DbClient, consumer: string): ClaimedRowTarget[] {
|
|
85
|
+
const tables = schemaTables();
|
|
86
|
+
const cleared: ClaimedRowTarget[] = [];
|
|
87
|
+
for (const target of planClaimedRowDeletion()) {
|
|
88
|
+
const table = tables.get(target.table);
|
|
89
|
+
if (!table) continue;
|
|
90
|
+
const column = getTableConfig(table).columns.find((c) => c.name === target.claimColumn);
|
|
91
|
+
if (!column) continue;
|
|
92
|
+
db.delete(table)
|
|
93
|
+
.where(eq(column as Column, consumer))
|
|
94
|
+
.run();
|
|
95
|
+
cleared.push(target);
|
|
96
|
+
}
|
|
97
|
+
return cleared;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Capability names that declare at least one table — for logs and reports. */
|
|
101
|
+
export function capabilitiesWithDeclaredTables(): string[] {
|
|
102
|
+
return Object.keys(CAPABILITY_DECLARED_TABLES).sort();
|
|
103
|
+
}
|
|
@@ -23,18 +23,19 @@ import { capabilities, modules } from '../db/schema';
|
|
|
23
23
|
import { type RunNamedHookResult, runNamedHook } from '../hooks/run-named-hook';
|
|
24
24
|
import type { HookLogger } from '../hooks/types';
|
|
25
25
|
import type { ModuleManifest } from '../manifest/schema';
|
|
26
|
-
import {
|
|
27
|
-
import { deleteTrustedSourcesForModule } from './trusted-sources';
|
|
26
|
+
import { deleteClaimedRows } from './capability-table-rows';
|
|
28
27
|
|
|
29
28
|
/**
|
|
30
29
|
* States in which a module has never resolved a capability and therefore holds
|
|
31
30
|
* nothing minted on anyone's behalf. Capabilities are registered at IMPORT, not
|
|
32
31
|
* deploy, so the `capabilities` table routinely names providers that were never
|
|
33
32
|
* deployed. The same predicate `remove-guard.ts` uses to decide a module is not
|
|
34
|
-
* a dependent — the guard
|
|
35
|
-
*
|
|
33
|
+
* a dependent — the guard, the cleanup and the provider-arrival backfill must
|
|
34
|
+
* keep ONE definition of a live module (D8). Exported rather than re-spelled:
|
|
35
|
+
* `module-remove.ts` had its own copy of the literal, and `provider-arrival.ts`
|
|
36
|
+
* would have been a third.
|
|
36
37
|
*/
|
|
37
|
-
const PRE_DEPLOY_STATES = new Set(['IMPORTED', 'VALIDATED', 'CONFIGURED']);
|
|
38
|
+
export const PRE_DEPLOY_STATES = new Set(['IMPORTED', 'VALIDATED', 'CONFIGURED']);
|
|
38
39
|
|
|
39
40
|
export type CleanupSkipReason = 'paused' | 'not-deployed';
|
|
40
41
|
|
|
@@ -211,11 +212,18 @@ export async function runConsumerCleanup(
|
|
|
211
212
|
);
|
|
212
213
|
}
|
|
213
214
|
|
|
214
|
-
// The rows, once every provider has converged without them (D4).
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
|
|
218
|
-
deleteTrustedSourcesForModule
|
|
215
|
+
// The rows, once every provider has converged without them (D4).
|
|
216
|
+
//
|
|
217
|
+
// Driven by the capability declarations rather than by name. This used to be
|
|
218
|
+
// two hardcoded imports, `deletePortForwardsForModule` and
|
|
219
|
+
// `deleteTrustedSourcesForModule`, which is the shape of the whole problem
|
|
220
|
+
// openspec/changes/capability-owned-tables addresses: this file exists to
|
|
221
|
+
// delete per-capability special-casing, it succeeded for the hook dispatch,
|
|
222
|
+
// and then grew two capability-named calls for the rows because the rows had
|
|
223
|
+
// no generic rule. Now they have one, and core no longer has to know that the
|
|
224
|
+
// claim column is spelled `module_id` on one table, `registered_by` on two and
|
|
225
|
+
// `consumer_module_id` on a fourth.
|
|
226
|
+
deleteClaimedRows(db, consumer);
|
|
219
227
|
|
|
220
228
|
return result;
|
|
221
229
|
}
|
|
@@ -17,6 +17,8 @@ import {
|
|
|
17
17
|
getServiceCredentials,
|
|
18
18
|
listContainerServices,
|
|
19
19
|
removeContainerService,
|
|
20
|
+
updateServiceCredentials,
|
|
21
|
+
updateVerificationStatus,
|
|
20
22
|
} from './container-service';
|
|
21
23
|
|
|
22
24
|
describe('container-service', () => {
|
|
@@ -270,6 +272,38 @@ describe('container-service', () => {
|
|
|
270
272
|
/Container service not found/,
|
|
271
273
|
);
|
|
272
274
|
});
|
|
275
|
+
|
|
276
|
+
it('re-encrypts replacements and clears stale verification state', async () => {
|
|
277
|
+
const service = await addContainerService({
|
|
278
|
+
name: 'Moving Proxmox',
|
|
279
|
+
providerName: 'proxmox',
|
|
280
|
+
zones: ['internal'],
|
|
281
|
+
providerConfig: {},
|
|
282
|
+
apiCredentials: {
|
|
283
|
+
api_url: 'https://192.168.0.50:8006',
|
|
284
|
+
api_token_id: 'root@pam!celilo',
|
|
285
|
+
api_token_secret: 'existing-secret',
|
|
286
|
+
},
|
|
287
|
+
});
|
|
288
|
+
await updateVerificationStatus(service.id, { success: true, message: 'Connected' });
|
|
289
|
+
|
|
290
|
+
await updateServiceCredentials(service.id, {
|
|
291
|
+
api_url: 'https://10.77.20.50:8006',
|
|
292
|
+
api_token_id: 'root@pam!celilo',
|
|
293
|
+
api_token_secret: 'existing-secret',
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
const credentials = await getServiceCredentials(service.id);
|
|
297
|
+
const updated = await getContainerService(service.id);
|
|
298
|
+
expect(credentials).toEqual({
|
|
299
|
+
api_url: 'https://10.77.20.50:8006',
|
|
300
|
+
api_token_id: 'root@pam!celilo',
|
|
301
|
+
api_token_secret: 'existing-secret',
|
|
302
|
+
});
|
|
303
|
+
expect(updated?.verified).toBe(false);
|
|
304
|
+
expect(updated?.verifiedAt).toBeNull();
|
|
305
|
+
expect(updated?.verificationError).toBeNull();
|
|
306
|
+
});
|
|
273
307
|
});
|
|
274
308
|
|
|
275
309
|
describe('removeContainerService', () => {
|
|
@@ -251,6 +251,50 @@ export async function getServiceCredentials(serviceId: string): Promise<ServiceC
|
|
|
251
251
|
);
|
|
252
252
|
}
|
|
253
253
|
|
|
254
|
+
/**
|
|
255
|
+
* Replace a container service's encrypted API credentials.
|
|
256
|
+
*
|
|
257
|
+
* Credentials identify the remote provider endpoint as well as the principal
|
|
258
|
+
* used there, so changing either invalidates the previous verification result.
|
|
259
|
+
* Callers should explicitly re-run service verification after this update.
|
|
260
|
+
*/
|
|
261
|
+
export async function updateServiceCredentials(
|
|
262
|
+
id: string,
|
|
263
|
+
credentials: ServiceCredentials,
|
|
264
|
+
): Promise<void> {
|
|
265
|
+
const service = await getContainerService(id);
|
|
266
|
+
if (!service) {
|
|
267
|
+
throw new Error(`Container service not found: ${id}`);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const validated =
|
|
271
|
+
service.providerName === 'proxmox'
|
|
272
|
+
? ProxmoxCredentialsSchema.parse(credentials)
|
|
273
|
+
: service.providerName === 'digitalocean'
|
|
274
|
+
? DigitalOceanCredentialsSchema.parse(credentials)
|
|
275
|
+
: null;
|
|
276
|
+
|
|
277
|
+
if (!validated) {
|
|
278
|
+
throw new Error(
|
|
279
|
+
`Unsupported provider for credential validation: ${service.providerName}. Supported providers: proxmox, digitalocean`,
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const masterKey = await getOrCreateMasterKey();
|
|
284
|
+
const encrypted = encryptSecret(JSON.stringify(validated), masterKey);
|
|
285
|
+
|
|
286
|
+
await getDb()
|
|
287
|
+
.update(containerServices)
|
|
288
|
+
.set({
|
|
289
|
+
apiCredentialsEncrypted: JSON.stringify(encrypted),
|
|
290
|
+
verified: false,
|
|
291
|
+
verifiedAt: null,
|
|
292
|
+
verificationError: null,
|
|
293
|
+
updatedAt: new Date(),
|
|
294
|
+
})
|
|
295
|
+
.where(eq(containerServices.id, id));
|
|
296
|
+
}
|
|
297
|
+
|
|
254
298
|
/**
|
|
255
299
|
* List container services with optional filters
|
|
256
300
|
*/
|
|
@@ -266,6 +266,57 @@ describe('backfillModuleSystems', () => {
|
|
|
266
266
|
});
|
|
267
267
|
});
|
|
268
268
|
|
|
269
|
+
test('backfill records a local machine by its interface in the resolved zone', () => {
|
|
270
|
+
db.insert(machines)
|
|
271
|
+
.values({
|
|
272
|
+
id: 'm-local-bf',
|
|
273
|
+
hostname: 'celilo-mgr',
|
|
274
|
+
// Local-execution sentinel, not the machine's network identity.
|
|
275
|
+
ipAddress: '127.0.0.1',
|
|
276
|
+
sshUser: 'jem',
|
|
277
|
+
sshKeyEncrypted: JSON.stringify({ encryptedValue: '', iv: '', authTag: '' }),
|
|
278
|
+
hardware: { cpu_cores: 4, memory_mb: 8192, disk_gb: 64 },
|
|
279
|
+
zone: 'secure-mgmt',
|
|
280
|
+
interfaces: [
|
|
281
|
+
{ name: 'en0', ipAddress: '192.168.0.32', zone: 'external' },
|
|
282
|
+
{ name: 'ens18', ipAddress: '10.77.20.32', zone: 'secure-mgmt' },
|
|
283
|
+
],
|
|
284
|
+
})
|
|
285
|
+
.run();
|
|
286
|
+
db.insert(modules)
|
|
287
|
+
.values({
|
|
288
|
+
id: 'celilo-mgmt',
|
|
289
|
+
name: 'celilo-mgmt',
|
|
290
|
+
version: '1.0.0',
|
|
291
|
+
manifestData: { requires: { system: { zone: 'internal' } } },
|
|
292
|
+
sourcePath: '/tmp/celilo-mgmt',
|
|
293
|
+
state: 'VERIFIED',
|
|
294
|
+
})
|
|
295
|
+
.run();
|
|
296
|
+
db.insert(moduleInfrastructure)
|
|
297
|
+
.values({
|
|
298
|
+
id: 'infra-celilo-mgmt-local',
|
|
299
|
+
moduleId: 'celilo-mgmt',
|
|
300
|
+
infrastructureType: 'machine',
|
|
301
|
+
machineId: 'm-local-bf',
|
|
302
|
+
})
|
|
303
|
+
.run();
|
|
304
|
+
db.insert(moduleConfigs)
|
|
305
|
+
.values({
|
|
306
|
+
moduleId: 'celilo-mgmt',
|
|
307
|
+
key: 'hostname',
|
|
308
|
+
value: 'celilo-mgr',
|
|
309
|
+
valueJson: '"celilo-mgr"',
|
|
310
|
+
})
|
|
311
|
+
.run();
|
|
312
|
+
|
|
313
|
+
expect(backfillModuleSystems(db)).toEqual(['celilo-mgmt']);
|
|
314
|
+
expect(getModuleSystems('celilo-mgmt', db)[0]).toMatchObject({
|
|
315
|
+
zone: 'secure-mgmt',
|
|
316
|
+
ipv4_address: '10.77.20.32',
|
|
317
|
+
});
|
|
318
|
+
});
|
|
319
|
+
|
|
269
320
|
// The control-plane case. celilo-mgmt declares `internal` (it bootstraps before
|
|
270
321
|
// any firewall exists) but in a segmented fleet the operator earmarks a box on
|
|
271
322
|
// `secure-mgmt`. What gets RECORDED must be where the system actually is, since
|
|
@@ -318,6 +369,56 @@ describe('backfillModuleSystems', () => {
|
|
|
318
369
|
expect(getModuleSystems('celilo-mgmt', db)[0]).toMatchObject({ zone: 'secure-mgmt' });
|
|
319
370
|
});
|
|
320
371
|
|
|
372
|
+
test('records a local machine by its interface in the resolved zone', async () => {
|
|
373
|
+
db.insert(machines)
|
|
374
|
+
.values({
|
|
375
|
+
id: 'm-local',
|
|
376
|
+
hostname: 'celilo-mgr',
|
|
377
|
+
// Local-execution sentinel, not the machine's network identity.
|
|
378
|
+
ipAddress: '127.0.0.1',
|
|
379
|
+
sshUser: 'jem',
|
|
380
|
+
sshKeyEncrypted: JSON.stringify({ encryptedValue: '', iv: '', authTag: '' }),
|
|
381
|
+
hardware: { cpu_cores: 4, memory_mb: 8192, disk_gb: 64 },
|
|
382
|
+
zone: 'secure-mgmt',
|
|
383
|
+
earmarkedModule: 'celilo-mgmt',
|
|
384
|
+
interfaces: [
|
|
385
|
+
{ name: 'en0', ipAddress: '192.168.0.32', zone: 'external' },
|
|
386
|
+
{ name: 'ens18', ipAddress: '10.77.20.32', zone: 'secure-mgmt' },
|
|
387
|
+
],
|
|
388
|
+
})
|
|
389
|
+
.run();
|
|
390
|
+
db.insert(modules)
|
|
391
|
+
.values({
|
|
392
|
+
id: 'celilo-mgmt',
|
|
393
|
+
name: 'celilo-mgmt',
|
|
394
|
+
version: '1.0.0',
|
|
395
|
+
manifestData: { requires: { system: { zone: 'internal' } } },
|
|
396
|
+
sourcePath: '/tmp/celilo-mgmt',
|
|
397
|
+
state: 'VERIFIED',
|
|
398
|
+
})
|
|
399
|
+
.run();
|
|
400
|
+
db.insert(moduleConfigs)
|
|
401
|
+
.values({
|
|
402
|
+
moduleId: 'celilo-mgmt',
|
|
403
|
+
key: 'hostname',
|
|
404
|
+
value: 'celilo-mgr',
|
|
405
|
+
valueJson: '"celilo-mgr"',
|
|
406
|
+
})
|
|
407
|
+
.run();
|
|
408
|
+
|
|
409
|
+
const recorded = await recordDeployedSystemForModule(
|
|
410
|
+
'celilo-mgmt',
|
|
411
|
+
{ requires: { system: { zone: 'internal' } } } as ModuleManifest,
|
|
412
|
+
{ type: 'machine', machineId: 'm-local' },
|
|
413
|
+
db,
|
|
414
|
+
);
|
|
415
|
+
|
|
416
|
+
expect(recorded[0]).toMatchObject({
|
|
417
|
+
zone: 'secure-mgmt',
|
|
418
|
+
ipv4_address: '10.77.20.32',
|
|
419
|
+
});
|
|
420
|
+
});
|
|
421
|
+
|
|
321
422
|
test('skips an API-only module (no declared systems)', () => {
|
|
322
423
|
ensureProxmoxService(db);
|
|
323
424
|
db.insert(modules)
|