@celilo/cli 0.27.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +16 -0
- package/CELILO_SUBSYSTEMS.md +5 -2
- package/drizzle/0025_port_forward_owner.sql +29 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +2 -2
- package/src/cli/commands/module-remove.ts +26 -23
- package/src/cli/commands/system-audit.ts +5 -1
- package/src/cli/commands/system-update.ts +10 -2
- package/src/db/schema.ts +24 -6
- package/src/hooks/capability-loader.ts +59 -13
- package/src/hooks/define-hook.test.ts +0 -6
- package/src/hooks/executor.ts +2 -1
- package/src/hooks/types.ts +9 -17
- package/src/manifest/contracts/index.ts +20 -0
- package/src/manifest/contracts/v1.ts +33 -1
- package/src/manifest/schema.ts +48 -58
- package/src/services/audit/undeployed-modules.ts +18 -1
- package/src/services/consumer-cleanup.test.ts +347 -0
- package/src/services/consumer-cleanup.ts +244 -0
- package/src/services/module-validator/index.test.ts +9 -0
- package/src/services/port-forwards.test.ts +93 -40
- package/src/services/port-forwards.ts +74 -48
- package/src/services/trusted-sources.test.ts +52 -13
- package/src/services/trusted-sources.ts +25 -15
- package/src/templates/generator.ts +46 -28
- package/src/templates/{dns-ingress-ip.test.ts → ingress-ip.test.ts} +38 -22
- package/src/test-utils/cli-context.ts +15 -2
- package/src/services/web-route-cleanup.test.ts +0 -250
- package/src/services/web-route-cleanup.ts +0 -144
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Telling every provider that one of its consumers is leaving
|
|
3
|
+
* (openspec/changes/consumer-removal-cleanup).
|
|
4
|
+
*
|
|
5
|
+
* A capability is two-sided. The consumer asks, the provider mints something in
|
|
6
|
+
* its own world — a site block in caddy's Caddyfile, a DNAT rule in a ruleset,
|
|
7
|
+
* an OIDC client at authentik — and removal only ever touched one side of it.
|
|
8
|
+
* The FK cascade made that worse rather than better: the registry row
|
|
9
|
+
* disappeared, so the provider's next converge had no way to learn the thing
|
|
10
|
+
* had ever existed. The registry went quiet and the machine kept serving.
|
|
11
|
+
*
|
|
12
|
+
* This is the generic path that replaces `web-route-cleanup.ts`, which did the
|
|
13
|
+
* same job for exactly one capability, called by name from core.
|
|
14
|
+
*
|
|
15
|
+
* Split plan/execute (Rule 10.4) because the interesting decisions — which
|
|
16
|
+
* providers, which are skipped and why — are pure, and the part that isn't is
|
|
17
|
+
* just "run each hook and record what happened".
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { eq } from 'drizzle-orm';
|
|
21
|
+
import type { DbClient } from '../db/client';
|
|
22
|
+
import { capabilities, modules } from '../db/schema';
|
|
23
|
+
import { type RunNamedHookResult, runNamedHook } from '../hooks/run-named-hook';
|
|
24
|
+
import type { HookLogger } from '../hooks/types';
|
|
25
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
26
|
+
import { deletePortForwardsForModule } from './port-forwards';
|
|
27
|
+
import { deleteTrustedSourcesForModule } from './trusted-sources';
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* States in which a module has never resolved a capability and therefore holds
|
|
31
|
+
* nothing minted on anyone's behalf. Capabilities are registered at IMPORT, not
|
|
32
|
+
* deploy, so the `capabilities` table routinely names providers that were never
|
|
33
|
+
* deployed. The same predicate `remove-guard.ts` uses to decide a module is not
|
|
34
|
+
* a dependent — the guard and the cleanup must keep ONE definition of a live
|
|
35
|
+
* provider (D8).
|
|
36
|
+
*/
|
|
37
|
+
const PRE_DEPLOY_STATES = new Set(['IMPORTED', 'VALIDATED', 'CONFIGURED']);
|
|
38
|
+
|
|
39
|
+
export type CleanupSkipReason = 'paused' | 'not-deployed';
|
|
40
|
+
|
|
41
|
+
export interface CleanupTarget {
|
|
42
|
+
/** The provider module to notify. */
|
|
43
|
+
providerId: string;
|
|
44
|
+
/** Which of its capabilities the departing consumer used — for the log line. */
|
|
45
|
+
capabilityNames: string[];
|
|
46
|
+
/** Set when the provider will NOT be notified. */
|
|
47
|
+
skip?: CleanupSkipReason;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface ProviderRow {
|
|
51
|
+
moduleId: string;
|
|
52
|
+
capabilityName: string;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface ProviderState {
|
|
56
|
+
moduleId: string;
|
|
57
|
+
state: string;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Which providers must be told that `consumer` is going away.
|
|
62
|
+
*
|
|
63
|
+
* Pure. Sorted by provider id so dispatch order is deterministic and a failure
|
|
64
|
+
* is reproducible.
|
|
65
|
+
*
|
|
66
|
+
* ONE ENTRY PER PROVIDER, not per capability (D1): a provider can hold two
|
|
67
|
+
* capabilities the same consumer used, and dispatching per capability would run
|
|
68
|
+
* the same withdrawal twice. But MANY providers per capability (D3) —
|
|
69
|
+
* `capabilities` has no uniqueness on the name, and `firewall` deliberately has
|
|
70
|
+
* several rows (an edge provider plus inner layers). Every one is told.
|
|
71
|
+
*
|
|
72
|
+
* The provider is never its own consumer: a module that both provides and
|
|
73
|
+
* requires a capability would otherwise be asked to withdraw its own state as
|
|
74
|
+
* it is being removed, which its `on_uninstall` already owns.
|
|
75
|
+
*/
|
|
76
|
+
export function planConsumerCleanup(
|
|
77
|
+
consumer: string,
|
|
78
|
+
manifest: ModuleManifest,
|
|
79
|
+
providerRows: ProviderRow[],
|
|
80
|
+
providerStates: ProviderState[],
|
|
81
|
+
): CleanupTarget[] {
|
|
82
|
+
// `requires` AND `optional` — the same set `remove-guard.ts` counts as a
|
|
83
|
+
// dependency edge. A capability consumed optionally still minted state.
|
|
84
|
+
const consumed = new Set([
|
|
85
|
+
...(manifest.requires?.capabilities ?? []).map((c) => c.name),
|
|
86
|
+
...(manifest.optional?.capabilities ?? []).map((c) => c.name),
|
|
87
|
+
]);
|
|
88
|
+
if (consumed.size === 0) return [];
|
|
89
|
+
|
|
90
|
+
const stateOf = new Map(providerStates.map((s) => [s.moduleId, s.state]));
|
|
91
|
+
const byProvider = new Map<string, Set<string>>();
|
|
92
|
+
|
|
93
|
+
for (const row of providerRows) {
|
|
94
|
+
if (!consumed.has(row.capabilityName)) continue;
|
|
95
|
+
if (row.moduleId === consumer) continue;
|
|
96
|
+
const names = byProvider.get(row.moduleId) ?? new Set<string>();
|
|
97
|
+
names.add(row.capabilityName);
|
|
98
|
+
byProvider.set(row.moduleId, names);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return [...byProvider.entries()]
|
|
102
|
+
.map(([providerId, names]): CleanupTarget => {
|
|
103
|
+
const state = stateOf.get(providerId);
|
|
104
|
+
const capabilityNames = [...names].sort();
|
|
105
|
+
if (state === 'PAUSED') return { providerId, capabilityNames, skip: 'paused' };
|
|
106
|
+
if (state === undefined || PRE_DEPLOY_STATES.has(state)) {
|
|
107
|
+
return { providerId, capabilityNames, skip: 'not-deployed' };
|
|
108
|
+
}
|
|
109
|
+
return { providerId, capabilityNames };
|
|
110
|
+
})
|
|
111
|
+
.sort((a, b) => a.providerId.localeCompare(b.providerId));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Read the plan's inputs out of the DB. */
|
|
115
|
+
export function loadConsumerCleanupPlan(
|
|
116
|
+
consumer: string,
|
|
117
|
+
manifest: ModuleManifest,
|
|
118
|
+
db: DbClient,
|
|
119
|
+
): CleanupTarget[] {
|
|
120
|
+
const providerRows = db
|
|
121
|
+
.select({ moduleId: capabilities.moduleId, capabilityName: capabilities.capabilityName })
|
|
122
|
+
.from(capabilities)
|
|
123
|
+
.all();
|
|
124
|
+
const providerStates = db
|
|
125
|
+
.select({ moduleId: modules.id, state: modules.state })
|
|
126
|
+
.from(modules)
|
|
127
|
+
.all();
|
|
128
|
+
return planConsumerCleanup(consumer, manifest, providerRows, providerStates);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export interface CleanupFailure {
|
|
132
|
+
providerId: string;
|
|
133
|
+
error: string;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export interface ConsumerCleanupResult {
|
|
137
|
+
notified: string[];
|
|
138
|
+
skipped: Array<{ providerId: string; reason: CleanupSkipReason }>;
|
|
139
|
+
failures: CleanupFailure[];
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Run the plan. Sequential, deterministic, and it CONTINUES PAST A FAILURE
|
|
144
|
+
* (D13) — every provider is told even if an earlier one threw. Stopping early
|
|
145
|
+
* would leave MORE providers holding state for a module that is about to
|
|
146
|
+
* disappear, which is the failure this exists to fix.
|
|
147
|
+
*
|
|
148
|
+
* A failed withdrawal NEVER blocks the removal (D6). The record is an ERRORed
|
|
149
|
+
* PROVIDER, not a refused removal: the hook is a full converge, so a failure
|
|
150
|
+
* does not mean "it failed to forget one thing" — the provider may have
|
|
151
|
+
* re-rendered its state without the departing consumer AND without everything
|
|
152
|
+
* else. ERROR is the honest label for a provider whose state is now unknown,
|
|
153
|
+
* and `audit/undeployed-modules.ts` already turns it into a `blocked` finding.
|
|
154
|
+
* Nothing refuses to USE an ERRORed module, so the provider keeps serving while
|
|
155
|
+
* carrying the flag.
|
|
156
|
+
*/
|
|
157
|
+
export async function runConsumerCleanup(
|
|
158
|
+
consumer: string,
|
|
159
|
+
plan: CleanupTarget[],
|
|
160
|
+
db: DbClient,
|
|
161
|
+
logger: HookLogger,
|
|
162
|
+
/** Injectable hook runner — tests drive the failure and paused paths through it. */
|
|
163
|
+
runHook: (providerId: string) => Promise<RunNamedHookResult> = (providerId) =>
|
|
164
|
+
runNamedHook(providerId, 'on_consumer_removed', db, logger, { inputs: { consumer } }),
|
|
165
|
+
): Promise<ConsumerCleanupResult> {
|
|
166
|
+
const result: ConsumerCleanupResult = { notified: [], skipped: [], failures: [] };
|
|
167
|
+
|
|
168
|
+
for (const target of plan) {
|
|
169
|
+
if (target.skip) {
|
|
170
|
+
result.skipped.push({ providerId: target.providerId, reason: target.skip });
|
|
171
|
+
// Reported, not silent (D7/D8). Nothing was attempted, so nothing is
|
|
172
|
+
// unknown and the provider is NOT marked ERROR.
|
|
173
|
+
if (target.skip === 'paused') {
|
|
174
|
+
logger.warn(
|
|
175
|
+
`${target.providerId} is paused, so it was not told that '${consumer}' is gone — it is still holding whatever it minted for it (${target.capabilityNames.join(', ')}). Unpause and redeploy it to reconcile.`,
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const hookResult = await runHook(target.providerId);
|
|
182
|
+
|
|
183
|
+
// A module paused BETWEEN the plan and this dispatch. `runNamedHook` reports
|
|
184
|
+
// that as success, so taking it at face value would log a withdrawal that
|
|
185
|
+
// did not happen — which is the silence this whole change exists to end.
|
|
186
|
+
// The window is small (one CLI process) and the consequence of trusting it
|
|
187
|
+
// is not, so it is read rather than assumed.
|
|
188
|
+
if (hookResult.skippedPaused) {
|
|
189
|
+
result.skipped.push({ providerId: target.providerId, reason: 'paused' });
|
|
190
|
+
logger.warn(
|
|
191
|
+
`${target.providerId} was paused while '${consumer}' was being removed, so it was not told — it is still holding whatever it minted for it (${target.capabilityNames.join(', ')}). Unpause and redeploy it to reconcile.`,
|
|
192
|
+
);
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (hookResult.success) {
|
|
197
|
+
// `notDefined` means the provider declares no such hook — it mints
|
|
198
|
+
// nothing per consumer, and the dispatch succeeding is the right answer.
|
|
199
|
+
if (!hookResult.notDefined) {
|
|
200
|
+
result.notified.push(target.providerId);
|
|
201
|
+
logger.info(`${target.providerId} withdrew what it held for '${consumer}'`);
|
|
202
|
+
}
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const error = hookResult.error ?? 'unknown error';
|
|
207
|
+
result.failures.push({ providerId: target.providerId, error });
|
|
208
|
+
markProviderErrored(target.providerId, consumer, error, db);
|
|
209
|
+
logger.warn(
|
|
210
|
+
`${target.providerId} failed to withdraw what it held for '${consumer}' and is now marked ERROR: ${error}`,
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// The rows, once every provider has converged without them (D4). `web_routes`
|
|
215
|
+
// dies with the `modules` row via its FK cascade; these two carry a plain
|
|
216
|
+
// `registered_by` text column and do not.
|
|
217
|
+
deletePortForwardsForModule(db, consumer);
|
|
218
|
+
deleteTrustedSourcesForModule(db, consumer);
|
|
219
|
+
|
|
220
|
+
return result;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Mark the PROVIDER — not the module being removed — as ERROR, naming the
|
|
225
|
+
* departing consumer and the raw hook error.
|
|
226
|
+
*
|
|
227
|
+
* The consumer is in `errorMessage` because the audit finding is read long
|
|
228
|
+
* after the removal, by someone with no reason to connect the two.
|
|
229
|
+
*/
|
|
230
|
+
function markProviderErrored(
|
|
231
|
+
providerId: string,
|
|
232
|
+
consumer: string,
|
|
233
|
+
error: string,
|
|
234
|
+
db: DbClient,
|
|
235
|
+
): void {
|
|
236
|
+
db.update(modules)
|
|
237
|
+
.set({
|
|
238
|
+
state: 'ERROR',
|
|
239
|
+
errorMessage: `Failed to withdraw state held for removed consumer '${consumer}': ${error}`,
|
|
240
|
+
updatedAt: new Date(),
|
|
241
|
+
})
|
|
242
|
+
.where(eq(modules.id, providerId))
|
|
243
|
+
.run();
|
|
244
|
+
}
|
|
@@ -15,6 +15,15 @@ const REPO_ROOT = resolve(__dirname, '../../../../..');
|
|
|
15
15
|
const CADDY_MODULE_PATH = resolve(REPO_ROOT, 'modules/caddy');
|
|
16
16
|
|
|
17
17
|
describe('runChecks (orchestrator)', () => {
|
|
18
|
+
// celilo#804: this test timed out at 60000ms on a docs-only CI run yet
|
|
19
|
+
// measures ~500ms end-to-end locally (all 3 tests in this file, combined).
|
|
20
|
+
// The work here is a handful of synchronous `git`/`spawnSync` calls
|
|
21
|
+
// (checkGitHygiene) with nothing to cut without dropping the git_hygiene
|
|
22
|
+
// check this test exists to exercise — so this was a saturated CI runner
|
|
23
|
+
// stalling those spawns, not this suite sitting close to its budget.
|
|
24
|
+
// Raising the timeout further wouldn't shrink that margin (a real stall
|
|
25
|
+
// would blow through any budget); left at the file's default (60000ms,
|
|
26
|
+
// ~100x the measured runtime) rather than bumped without evidence it helps.
|
|
18
27
|
test('healthy in-tree module produces all-ok report', async () => {
|
|
19
28
|
const checks = await runChecks(CADDY_MODULE_PATH, {
|
|
20
29
|
noBuild: true,
|
|
@@ -3,11 +3,14 @@ import { mkdtempSync, rmSync } from 'node:fs';
|
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import type { PortForwardStore } from '@celilo/capabilities';
|
|
6
|
+
import { eq } from 'drizzle-orm';
|
|
6
7
|
import type { DbClient } from '../db/client';
|
|
8
|
+
import { portForwards } from '../db/schema';
|
|
7
9
|
import { setupTestDatabase } from '../test-utils/setup-test-db';
|
|
8
|
-
import { buildPortForwardStore } from './port-forwards';
|
|
10
|
+
import { buildPortForwardStore, deletePortForwardsForModule } from './port-forwards';
|
|
9
11
|
|
|
10
12
|
const FW = '192.168.0.254';
|
|
13
|
+
const CADDY = { internalIp: '10.0.20.5', protocol: 'TCP' as const, description: 'caddy' };
|
|
11
14
|
|
|
12
15
|
describe('port-forward store', () => {
|
|
13
16
|
let dir: string;
|
|
@@ -19,7 +22,7 @@ describe('port-forward store', () => {
|
|
|
19
22
|
const dbPath = join(dir, 'celilo.db');
|
|
20
23
|
process.env.CELILO_DB_PATH = dbPath;
|
|
21
24
|
db = await setupTestDatabase(dbPath);
|
|
22
|
-
store = buildPortForwardStore(db);
|
|
25
|
+
store = buildPortForwardStore(db, 'caddy');
|
|
23
26
|
});
|
|
24
27
|
afterEach(() => {
|
|
25
28
|
db.$client.close();
|
|
@@ -31,62 +34,112 @@ describe('port-forward store', () => {
|
|
|
31
34
|
}
|
|
32
35
|
});
|
|
33
36
|
|
|
34
|
-
it('
|
|
35
|
-
store.
|
|
36
|
-
store.add(FW, {
|
|
37
|
-
internalIp: '10.0.20.42',
|
|
38
|
-
port: 2222,
|
|
39
|
-
protocol: 'TCP',
|
|
40
|
-
description: 'forgejo',
|
|
41
|
-
});
|
|
37
|
+
it('declares and lists forwards for a firewall', () => {
|
|
38
|
+
store.replace(FW, CADDY, [80, 443]);
|
|
42
39
|
const forwards = store.list(FW);
|
|
43
40
|
expect(forwards).toHaveLength(2);
|
|
44
|
-
expect(forwards.map((f) => f.port).sort((a, b) => a - b)).toEqual([
|
|
41
|
+
expect(forwards.map((f) => f.port).sort((a, b) => a - b)).toEqual([80, 443]);
|
|
45
42
|
});
|
|
46
43
|
|
|
47
|
-
it('
|
|
48
|
-
store.
|
|
49
|
-
store.
|
|
44
|
+
it('stamps registeredBy from the bound consumer, and no caller can supply it', () => {
|
|
45
|
+
store.replace(FW, CADDY, [443]);
|
|
46
|
+
expect(store.list(FW)[0].registeredBy).toBe('caddy');
|
|
47
|
+
// The write takes only (firewall, target, ports) — there is no parameter an
|
|
48
|
+
// owner could be passed through, which is the point rather than an omission.
|
|
49
|
+
expect(store.replace.length).toBe(3);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('replace is idempotent — same declaration twice → one row per port', () => {
|
|
53
|
+
store.replace(FW, { ...CADDY, description: 'v1' }, [443]);
|
|
54
|
+
store.replace(FW, { ...CADDY, description: 'v2' }, [443]);
|
|
50
55
|
const forwards = store.list(FW);
|
|
51
56
|
expect(forwards).toHaveLength(1);
|
|
52
57
|
expect(forwards[0].description).toBe('v2');
|
|
53
58
|
});
|
|
54
59
|
|
|
55
60
|
it('scopes forwards by firewallIp', () => {
|
|
56
|
-
store.
|
|
57
|
-
store.
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
});
|
|
61
|
+
store.replace(FW, CADDY, [443]);
|
|
62
|
+
store.replace(
|
|
63
|
+
'10.0.30.254',
|
|
64
|
+
{ internalIp: '10.0.30.5', protocol: 'UDP', description: 'b' },
|
|
65
|
+
[53],
|
|
66
|
+
);
|
|
63
67
|
expect(store.list(FW)).toHaveLength(1);
|
|
64
68
|
expect(store.list('10.0.30.254')).toHaveLength(1);
|
|
65
69
|
});
|
|
66
70
|
|
|
67
|
-
it('remove deletes exactly the tuple and leaves others', () => {
|
|
68
|
-
store.add(FW, { internalIp: '10.0.20.5', port: 443, protocol: 'TCP', description: 'x' });
|
|
69
|
-
store.add(FW, { internalIp: '10.0.20.5', port: 80, protocol: 'TCP', description: 'y' });
|
|
70
|
-
store.remove(FW, '10.0.20.5', 443, 'TCP');
|
|
71
|
-
const forwards = store.list(FW);
|
|
72
|
-
expect(forwards).toHaveLength(1);
|
|
73
|
-
expect(forwards[0].port).toBe(80);
|
|
74
|
-
});
|
|
75
|
-
|
|
76
71
|
it('ingressIp forwards are distinct from the public (NULL) ones', () => {
|
|
77
|
-
|
|
78
|
-
store.
|
|
79
|
-
|
|
80
|
-
port: 53,
|
|
81
|
-
protocol: 'UDP',
|
|
82
|
-
ingressIp: '10.0.10.53',
|
|
83
|
-
description: 'ingress',
|
|
84
|
-
});
|
|
72
|
+
const target = { internalIp: '10.0.20.5', protocol: 'UDP' as const, description: 'dns' };
|
|
73
|
+
store.replace(FW, target, [53]);
|
|
74
|
+
store.replace(FW, { ...target, ingressIp: '10.0.10.53' }, [53]);
|
|
85
75
|
expect(store.list(FW)).toHaveLength(2);
|
|
86
|
-
//
|
|
87
|
-
store.
|
|
76
|
+
// Re-declaring the public (NULL-ingress) set as empty leaves the ingress one.
|
|
77
|
+
store.replace(FW, target, []);
|
|
88
78
|
const forwards = store.list(FW);
|
|
89
79
|
expect(forwards).toHaveLength(1);
|
|
90
80
|
expect(forwards[0].ingressIp).toBe('10.0.10.53');
|
|
91
81
|
});
|
|
82
|
+
|
|
83
|
+
// D5b / celilo#855. Before this, `exposeService` upserted per port and nothing
|
|
84
|
+
// ever removed a forward a consumer stopped wanting: a module that exposed
|
|
85
|
+
// :8080 and redeployed exposing :9090 kept both, forever.
|
|
86
|
+
it('re-declaring with a shorter port list drops the ports left out', () => {
|
|
87
|
+
store.replace(FW, CADDY, [80, 443, 8080]);
|
|
88
|
+
store.replace(FW, CADDY, [80, 443]);
|
|
89
|
+
expect(
|
|
90
|
+
store
|
|
91
|
+
.list(FW)
|
|
92
|
+
.map((f) => f.port)
|
|
93
|
+
.sort((a, b) => a - b),
|
|
94
|
+
).toEqual([80, 443]);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it('a consumer narrowing its declaration withdraws no other module rows', () => {
|
|
98
|
+
const forgejo = buildPortForwardStore(db, 'forgejo');
|
|
99
|
+
store.replace(FW, CADDY, [80, 443]);
|
|
100
|
+
forgejo.replace(FW, { internalIp: '10.0.20.42', protocol: 'TCP', description: 'git' }, [2222]);
|
|
101
|
+
|
|
102
|
+
store.replace(FW, CADDY, [443]);
|
|
103
|
+
|
|
104
|
+
expect(store.list(FW).filter((f) => f.registeredBy === 'forgejo')).toHaveLength(1);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
// D5a, the refcount case — the bug most likely to ship silently. The owner is
|
|
108
|
+
// IN the unique index, so two consumers of the same forward are two rows and
|
|
109
|
+
// one leaving does not delete a rule the other still needs.
|
|
110
|
+
it('two consumers of the SAME forward are two rows, and one leaving leaves the other', () => {
|
|
111
|
+
const other = buildPortForwardStore(db, 'greenwave-app');
|
|
112
|
+
store.replace(FW, CADDY, [443]);
|
|
113
|
+
other.replace(FW, { ...CADDY, description: 'also 443' }, [443]);
|
|
114
|
+
expect(store.list(FW)).toHaveLength(2);
|
|
115
|
+
|
|
116
|
+
deletePortForwardsForModule(db, 'caddy');
|
|
117
|
+
|
|
118
|
+
const left = store.list(FW);
|
|
119
|
+
expect(left).toHaveLength(1);
|
|
120
|
+
expect(left[0].registeredBy).toBe('greenwave-app');
|
|
121
|
+
expect(left[0].port).toBe(443);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
// Migration 0025 leaves pre-existing rows with an empty owner so the fleet
|
|
125
|
+
// keeps serving. They are adopted the first time their owner re-declares the
|
|
126
|
+
// same target, which is the only moment we can safely say who owns them.
|
|
127
|
+
it('adopts an unattributed legacy row when its owner re-declares the target', () => {
|
|
128
|
+
db.insert(portForwards)
|
|
129
|
+
.values({
|
|
130
|
+
firewallIp: FW,
|
|
131
|
+
internalIp: CADDY.internalIp,
|
|
132
|
+
port: 443,
|
|
133
|
+
protocol: 'TCP',
|
|
134
|
+
description: 'legacy',
|
|
135
|
+
registeredBy: '',
|
|
136
|
+
})
|
|
137
|
+
.run();
|
|
138
|
+
|
|
139
|
+
store.replace(FW, CADDY, [443]);
|
|
140
|
+
|
|
141
|
+
const rows = db.select().from(portForwards).where(eq(portForwards.firewallIp, FW)).all();
|
|
142
|
+
expect(rows).toHaveLength(1);
|
|
143
|
+
expect(rows[0].registeredBy).toBe('caddy');
|
|
144
|
+
});
|
|
92
145
|
});
|
|
@@ -2,15 +2,20 @@
|
|
|
2
2
|
* Port-forward registry — the DB-backed `PortForwardStore` (openspec/changes/unified-management-no-ssh/proposal.md).
|
|
3
3
|
*
|
|
4
4
|
* The shared-core desired-state store for the `firewall` capability. The
|
|
5
|
-
* capability-loader constructs one of these
|
|
6
|
-
* provider factory, so `exposeService
|
|
7
|
-
*
|
|
8
|
-
* `list(firewallIp)
|
|
9
|
-
* "read the box back with `iptables -L`" as the
|
|
5
|
+
* capability-loader constructs one of these bound to the CONSUMING module and
|
|
6
|
+
* injects it into the firewall provider factory, so `exposeService` becomes a
|
|
7
|
+
* declaration of that consumer's complete port set and the provider's converge
|
|
8
|
+
* renders the whole ruleset from `list(firewallIp)`, applying it atomically
|
|
9
|
+
* (`iptables-restore`). Replaces "read the box back with `iptables -L`" as the
|
|
10
|
+
* source of truth.
|
|
11
|
+
*
|
|
12
|
+
* `registeredBy` is stamped HERE, never accepted from a caller — the same rule
|
|
13
|
+
* `buildTrustedSourceStore` follows, so a forward cannot be attributed to the
|
|
14
|
+
* wrong module (openspec/changes/consumer-removal-cleanup, D2).
|
|
10
15
|
*/
|
|
11
16
|
|
|
12
|
-
import type { PortForward, PortForwardStore,
|
|
13
|
-
import { and, eq, isNull } from 'drizzle-orm';
|
|
17
|
+
import type { PortForward, PortForwardStore, PortForwardTarget } from '@celilo/capabilities';
|
|
18
|
+
import { and, eq, isNull, or } from 'drizzle-orm';
|
|
14
19
|
import type { DbClient } from '../db/client';
|
|
15
20
|
import { portForwards } from '../db/schema';
|
|
16
21
|
|
|
@@ -19,7 +24,37 @@ function ingressMatch(ingressIp: string | undefined) {
|
|
|
19
24
|
return ingressIp ? eq(portForwards.ingressIp, ingressIp) : isNull(portForwards.ingressIp);
|
|
20
25
|
}
|
|
21
26
|
|
|
22
|
-
|
|
27
|
+
/**
|
|
28
|
+
* Every row this consumer owns for one target, plus the UNATTRIBUTED rows for
|
|
29
|
+
* the same target.
|
|
30
|
+
*
|
|
31
|
+
* The unattributed half is the migration's other end: rows written before
|
|
32
|
+
* `registered_by` existed carry `''`, and a consumer re-declaring the target
|
|
33
|
+
* they describe is the one moment we can safely say who owns them — one module
|
|
34
|
+
* owns a backend IP. Without this they would render forever with no owner to
|
|
35
|
+
* withdraw them.
|
|
36
|
+
*/
|
|
37
|
+
function ownedOrUnattributed(firewallIp: string, target: PortForwardTarget, consumer: string) {
|
|
38
|
+
return and(
|
|
39
|
+
eq(portForwards.firewallIp, firewallIp),
|
|
40
|
+
eq(portForwards.internalIp, target.internalIp),
|
|
41
|
+
eq(portForwards.protocol, target.protocol),
|
|
42
|
+
ingressMatch(target.ingressIp),
|
|
43
|
+
or(eq(portForwards.registeredBy, consumer), eq(portForwards.registeredBy, '')),
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Build the store bound to the module that will call through the capability.
|
|
49
|
+
*
|
|
50
|
+
* ponytail: a consumer that stops exposing a target ENTIRELY — its backend IP
|
|
51
|
+
* changes, or it drops a host — leaves rows for the old target, because no call
|
|
52
|
+
* arrives to declare that target's set empty. Bounded: the rows die when the
|
|
53
|
+
* module is removed, and a redeploy onto a new host is the only way to reach
|
|
54
|
+
* it. Closing it needs a sweep against `getModuleSystems`, which is not
|
|
55
|
+
* obviously worth its own failure mode.
|
|
56
|
+
*/
|
|
57
|
+
export function buildPortForwardStore(db: DbClient, registeredBy: string): PortForwardStore {
|
|
23
58
|
return {
|
|
24
59
|
list(firewallIp: string): PortForward[] {
|
|
25
60
|
return db
|
|
@@ -33,54 +68,45 @@ export function buildPortForwardStore(db: DbClient): PortForwardStore {
|
|
|
33
68
|
protocol: r.protocol,
|
|
34
69
|
ingressIp: r.ingressIp ?? undefined,
|
|
35
70
|
description: r.description,
|
|
71
|
+
registeredBy: r.registeredBy,
|
|
36
72
|
}));
|
|
37
73
|
},
|
|
38
74
|
|
|
39
|
-
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
//
|
|
75
|
+
replace(firewallIp: string, target: PortForwardTarget, ports: number[]): void {
|
|
76
|
+
// Delete-then-insert over the consumer's WHOLE set for this target, not
|
|
77
|
+
// per port: that is what makes a redeploy exposing a shorter port list
|
|
78
|
+
// withdraw the ports it left out (celilo#855). Scoped to this consumer,
|
|
79
|
+
// so another module's forward for the same target is untouched.
|
|
43
80
|
db.delete(portForwards)
|
|
44
|
-
.where(
|
|
45
|
-
and(
|
|
46
|
-
eq(portForwards.firewallIp, firewallIp),
|
|
47
|
-
eq(portForwards.internalIp, forward.internalIp),
|
|
48
|
-
eq(portForwards.port, forward.port),
|
|
49
|
-
eq(portForwards.protocol, forward.protocol),
|
|
50
|
-
ingressMatch(forward.ingressIp),
|
|
51
|
-
),
|
|
52
|
-
)
|
|
81
|
+
.where(ownedOrUnattributed(firewallIp, target, registeredBy))
|
|
53
82
|
.run();
|
|
54
|
-
db.insert(portForwards)
|
|
55
|
-
.values({
|
|
56
|
-
firewallIp,
|
|
57
|
-
internalIp: forward.internalIp,
|
|
58
|
-
port: forward.port,
|
|
59
|
-
protocol: forward.protocol,
|
|
60
|
-
ingressIp: forward.ingressIp ?? null,
|
|
61
|
-
description: forward.description,
|
|
62
|
-
})
|
|
63
|
-
.run();
|
|
64
|
-
},
|
|
65
83
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
eq(portForwards.protocol, protocol),
|
|
80
|
-
ingressMatch(ingressIp),
|
|
81
|
-
),
|
|
84
|
+
if (ports.length === 0) return;
|
|
85
|
+
|
|
86
|
+
db.insert(portForwards)
|
|
87
|
+
.values(
|
|
88
|
+
ports.map((port) => ({
|
|
89
|
+
firewallIp,
|
|
90
|
+
internalIp: target.internalIp,
|
|
91
|
+
port,
|
|
92
|
+
protocol: target.protocol,
|
|
93
|
+
ingressIp: target.ingressIp ?? null,
|
|
94
|
+
description: target.description,
|
|
95
|
+
registeredBy,
|
|
96
|
+
})),
|
|
82
97
|
)
|
|
83
98
|
.run();
|
|
84
99
|
},
|
|
85
100
|
};
|
|
86
101
|
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Drop every forward a departing module owns, across every firewall.
|
|
105
|
+
*
|
|
106
|
+
* `registered_by` is plain text rather than a FK, so unlike `web_routes` these
|
|
107
|
+
* rows do NOT die with the `modules` row — core deletes them explicitly, AFTER
|
|
108
|
+
* the provider has converged without them (D4).
|
|
109
|
+
*/
|
|
110
|
+
export function deletePortForwardsForModule(db: DbClient, moduleId: string): void {
|
|
111
|
+
db.delete(portForwards).where(eq(portForwards.registeredBy, moduleId)).run();
|
|
112
|
+
}
|