@celilo/cli 4.1.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/drizzle/0034_private_routes.sql +41 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +2 -2
- package/src/api/sessions.test.ts +1 -1
- package/src/capabilities/registration.test.ts +11 -11
- package/src/capabilities/secrets.test.ts +18 -18
- package/src/capabilities/validation.test.ts +24 -24
- package/src/cli/cli.test.ts +3 -3
- package/src/cli/commands/module-changeset.test.ts +1 -1
- package/src/cli/commands/module-generate.test.ts +2 -2
- package/src/cli/commands/module-import-aspect.test.ts +2 -2
- package/src/cli/commands/module-update.test.ts +14 -14
- package/src/cli/commands/module-upgrade-gate.test.ts +26 -1
- package/src/cli/commands/module-upgrade.test.ts +3 -3
- package/src/cli/commands/module-upgrade.ts +12 -0
- package/src/cli/commands/module-version.test.ts +1 -1
- package/src/cli/index.ts +22 -2
- package/src/cli/restore-migration-failure.test.ts +1 -1
- package/src/cli/stdout-pipe-flush.test.ts +35 -0
- package/src/db/private-routes.test.ts +159 -0
- package/src/db/schema.ts +49 -0
- package/src/hooks/capability-loader.ts +179 -12
- package/src/hooks/executor.test.ts +13 -13
- package/src/hooks/hook-state-dir.test.ts +2 -2
- package/src/hooks/test-fixtures/store-backed.ts +5 -3
- package/src/manifest/contracts/index.ts +28 -7
- package/src/manifest/contracts/v2.ts +32 -0
- package/src/manifest/icon-schema.test.ts +1 -1
- package/src/manifest/schema.ts +13 -2
- package/src/manifest/template-validator.test.ts +1 -1
- package/src/manifest/validate-privileged.test.ts +1 -1
- package/src/manifest/validate.test.ts +68 -29
- package/src/manifest/validate.ts +14 -1
- package/src/module/import.test.ts +7 -7
- package/src/module/packaging/build.test.ts +1 -1
- package/src/packaging/nfpm-depends.test.ts +42 -0
- package/src/policy/capability-shape-baseline.ts +7 -1
- package/src/policy/fixture-capability-coverage.test.ts +5 -5
- package/src/policy/module-business-baseline.ts +33 -4
- package/src/services/aspect-approvals.test.ts +2 -2
- package/src/services/aspect-reconcile.test.ts +2 -2
- package/src/services/aspect-runner.test.ts +4 -4
- package/src/services/audit/backups.test.ts +1 -1
- package/src/services/audit/capability-abi.test.ts +1 -1
- package/src/services/audit/module-configs.test.ts +1 -1
- package/src/services/build-bus/hook-dispatch-executor.test.ts +3 -3
- package/src/services/bus-interview-park.test.ts +1 -1
- package/src/services/bus-secret-flow.test.ts +1 -1
- package/src/services/capability-table-rows.test.ts +20 -2
- package/src/services/celilo-events.test.ts +80 -6
- package/src/services/celilo-events.ts +115 -12
- package/src/services/celilo-mgmt-hooks.test.ts +6 -3
- package/src/services/cross-module-read.test.ts +2 -2
- package/src/services/deploy-validation.test.ts +6 -6
- package/src/services/dns-provider-backfill.test.ts +2 -2
- package/src/services/dns-registrations.test.ts +1 -1
- package/src/services/fleet-checks.test.ts +2 -2
- package/src/services/health-runner.test.ts +1 -1
- package/src/services/infrastructure-variable-resolver.test.ts +11 -11
- package/src/services/instance-ops.test.ts +2 -2
- package/src/services/module-deploy.dns-repoint.test.ts +1 -1
- package/src/services/module-pause-quiescence.test.ts +1 -1
- package/src/services/module-pause.test.ts +1 -1
- package/src/services/module-subscriptions.test.ts +2 -2
- package/src/services/module-types-generator.test.ts +1 -1
- package/src/services/module-validator/contract-version.test.ts +3 -3
- package/src/services/network-ensure.test.ts +1 -1
- package/src/services/proxmox-reconcile.test.ts +2 -2
- package/src/services/remove-guard.test.ts +1 -1
- package/src/services/update/dep-graph.test.ts +1 -1
- package/src/services/update/orchestrator.test.ts +1 -1
- package/src/variables/capability-self-ref.test.ts +2 -2
- package/src/variables/computed/computed-integration.test.ts +1 -1
- package/src/variables/declarative-derivation.test.ts +19 -19
|
@@ -21,7 +21,7 @@ function makeModule(
|
|
|
21
21
|
id,
|
|
22
22
|
name: id,
|
|
23
23
|
version: '1.0.0',
|
|
24
|
-
celilo_contract: '
|
|
24
|
+
celilo_contract: '2.0',
|
|
25
25
|
variables: { owns: variables, imports: [] },
|
|
26
26
|
} as unknown as ModuleManifest;
|
|
27
27
|
// Default state INSTALLED so existing tests continue to expect
|
|
@@ -121,7 +121,7 @@ describe('the dispatcher dispatches on_upstream_publish through the ordinary exe
|
|
|
121
121
|
id: 'upstream-fixture',
|
|
122
122
|
name: 'upstream-fixture',
|
|
123
123
|
version: '1.0.0',
|
|
124
|
-
celilo_contract: '
|
|
124
|
+
celilo_contract: '2.0',
|
|
125
125
|
provides: { capabilities: [] },
|
|
126
126
|
requires: { capabilities: [] },
|
|
127
127
|
hooks: { on_upstream_publish: [hookEntry] },
|
|
@@ -178,7 +178,7 @@ describe('the dispatcher dispatches on_upstream_publish through the ordinary exe
|
|
|
178
178
|
id: 'upstream-fixture',
|
|
179
179
|
name: 'upstream-fixture',
|
|
180
180
|
version: '1.0.0',
|
|
181
|
-
celilo_contract: '
|
|
181
|
+
celilo_contract: '2.0',
|
|
182
182
|
provides: { capabilities: [] },
|
|
183
183
|
requires: { capabilities: [] },
|
|
184
184
|
hooks: { on_upstream_publish: [hookEntry] },
|
|
@@ -232,7 +232,7 @@ test('a failing hook run lands in the build_bus_hook_runs ledger', async () => {
|
|
|
232
232
|
id: 'upstream-fixture',
|
|
233
233
|
name: 'upstream-fixture',
|
|
234
234
|
version: '1.0.0',
|
|
235
|
-
celilo_contract: '
|
|
235
|
+
celilo_contract: '2.0',
|
|
236
236
|
provides: { capabilities: [] },
|
|
237
237
|
requires: { capabilities: [] },
|
|
238
238
|
hooks: { on_upstream_publish: [hookEntry] },
|
|
@@ -71,7 +71,7 @@ beforeEach(() => {
|
|
|
71
71
|
name: 'iptables',
|
|
72
72
|
sourcePath: join(dir, 'installed'),
|
|
73
73
|
version: '1.0.2+9',
|
|
74
|
-
manifestData: { celilo_contract: '
|
|
74
|
+
manifestData: { celilo_contract: '2.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
|
|
75
75
|
})
|
|
76
76
|
.run();
|
|
77
77
|
|
|
@@ -18,7 +18,14 @@ import { tmpdir } from 'node:os';
|
|
|
18
18
|
import { join } from 'node:path';
|
|
19
19
|
import { allDeclaredTables } from '@celilo/capabilities';
|
|
20
20
|
import type { DbClient } from '../db/client';
|
|
21
|
-
import {
|
|
21
|
+
import {
|
|
22
|
+
dnsInternalRecords,
|
|
23
|
+
modules,
|
|
24
|
+
portForwards,
|
|
25
|
+
privateRoutes,
|
|
26
|
+
trustedSources,
|
|
27
|
+
webRoutes,
|
|
28
|
+
} from '../db/schema';
|
|
22
29
|
import { setupTestDatabaseAt } from '../test-utils/database';
|
|
23
30
|
import { resetTestDbPath } from '../test-utils/db-path';
|
|
24
31
|
import { deleteClaimedRows, planClaimedRowDeletion } from './capability-table-rows';
|
|
@@ -89,6 +96,15 @@ describe('deleteClaimedRows', () => {
|
|
|
89
96
|
hostname,
|
|
90
97
|
})
|
|
91
98
|
.run();
|
|
99
|
+
db.insert(privateRoutes)
|
|
100
|
+
.values({
|
|
101
|
+
slug: `${consumer}-private-slug`,
|
|
102
|
+
moduleId: consumer,
|
|
103
|
+
type: 'reverse_proxy',
|
|
104
|
+
path: `/${consumer}`,
|
|
105
|
+
hostname: `private-${hostname}`,
|
|
106
|
+
})
|
|
107
|
+
.run();
|
|
92
108
|
db.insert(portForwards)
|
|
93
109
|
.values({
|
|
94
110
|
firewallIp: FW,
|
|
@@ -109,7 +125,7 @@ describe('deleteClaimedRows', () => {
|
|
|
109
125
|
.run();
|
|
110
126
|
}
|
|
111
127
|
|
|
112
|
-
it('clears the departing consumer from
|
|
128
|
+
it('clears the departing consumer from every declared table at once', () => {
|
|
113
129
|
seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
|
|
114
130
|
|
|
115
131
|
const cleared = deleteClaimedRows(db, 'caddy');
|
|
@@ -117,10 +133,12 @@ describe('deleteClaimedRows', () => {
|
|
|
117
133
|
expect(cleared.map((c) => c.table).sort()).toEqual([
|
|
118
134
|
'dns_internal_records',
|
|
119
135
|
'port_forwards',
|
|
136
|
+
'private_routes',
|
|
120
137
|
'trusted_sources',
|
|
121
138
|
'web_routes',
|
|
122
139
|
]);
|
|
123
140
|
expect(db.select().from(webRoutes).all()).toHaveLength(0);
|
|
141
|
+
expect(db.select().from(privateRoutes).all()).toHaveLength(0);
|
|
124
142
|
expect(db.select().from(portForwards).all()).toHaveLength(0);
|
|
125
143
|
expect(db.select().from(trustedSources).all()).toHaveLength(0);
|
|
126
144
|
expect(db.select().from(dnsInternalRecords).all()).toHaveLength(0);
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
emitWebRoutesChanged,
|
|
17
17
|
emitWebRoutesChangedAndWait,
|
|
18
18
|
routesChangedHighWater,
|
|
19
|
+
subscriberBudgetMs,
|
|
19
20
|
waitForRouteReconcile,
|
|
20
21
|
} from './celilo-events';
|
|
21
22
|
|
|
@@ -171,12 +172,13 @@ describe('celilo lifecycle events', () => {
|
|
|
171
172
|
});
|
|
172
173
|
|
|
173
174
|
describe('deploy-waits for web-route reconcile (ISS-0035)', () => {
|
|
174
|
-
function subscribeReconciler(): void {
|
|
175
|
+
function subscribeReconciler(budget?: { maxAttempts: number; timeoutMs: number }): void {
|
|
175
176
|
const bus = openBus({ dbPath, events: defineEvents({}) });
|
|
176
177
|
bus.subscribe({
|
|
177
178
|
name: 'caddy.reconcile-web-routes',
|
|
178
179
|
pattern: 'public_web.routes_changed',
|
|
179
180
|
handler: 'unused',
|
|
181
|
+
...budget,
|
|
180
182
|
});
|
|
181
183
|
bus.close();
|
|
182
184
|
}
|
|
@@ -208,6 +210,16 @@ describe('celilo lifecycle events', () => {
|
|
|
208
210
|
bus.close();
|
|
209
211
|
}
|
|
210
212
|
|
|
213
|
+
function settleAllDeliveries(): void {
|
|
214
|
+
const bus = openBus({ dbPath, events: defineEvents({}) });
|
|
215
|
+
const sub = bus.getSubscriberByName('caddy.reconcile-web-routes');
|
|
216
|
+
if (!sub) throw new Error('expected a subscriber');
|
|
217
|
+
for (const event of bus.recentEvents({ type: 'public_web.routes_changed', limit: 100 })) {
|
|
218
|
+
bus.markSucceeded({ eventId: event.id, subscriberId: sub.id });
|
|
219
|
+
}
|
|
220
|
+
bus.close();
|
|
221
|
+
}
|
|
222
|
+
|
|
211
223
|
/** One failed attempt that will be retried — the delivery stays pending. */
|
|
212
224
|
function recordRetryingFailure(message: string): void {
|
|
213
225
|
const bus = openBus({ dbPath, events: defineEvents({}) });
|
|
@@ -231,7 +243,7 @@ describe('celilo lifecycle events', () => {
|
|
|
231
243
|
it('returns immediately when the deploy changed no routes', async () => {
|
|
232
244
|
const since = routesChangedHighWater();
|
|
233
245
|
const result = await waitForRouteReconcile(since, { timeoutMs: 1000 });
|
|
234
|
-
expect(result).
|
|
246
|
+
expect(result).toMatchObject({
|
|
235
247
|
events: 0,
|
|
236
248
|
succeeded: 0,
|
|
237
249
|
failed: 0,
|
|
@@ -247,7 +259,7 @@ describe('celilo lifecycle events', () => {
|
|
|
247
259
|
emitWebRoutesChanged('celilo-website');
|
|
248
260
|
// No heartbeat stamped → health() is no_dispatcher.
|
|
249
261
|
const result = await waitForRouteReconcile(since, { timeoutMs: 5000, pollMs: 50 });
|
|
250
|
-
expect(result).
|
|
262
|
+
expect(result).toMatchObject({
|
|
251
263
|
events: 1,
|
|
252
264
|
succeeded: 0,
|
|
253
265
|
failed: 0,
|
|
@@ -262,7 +274,7 @@ describe('celilo lifecycle events', () => {
|
|
|
262
274
|
const since = routesChangedHighWater();
|
|
263
275
|
emitWebRoutesChanged('celilo-website');
|
|
264
276
|
const result = await waitForRouteReconcile(since, { timeoutMs: 1000, pollMs: 50 });
|
|
265
|
-
expect(result).
|
|
277
|
+
expect(result).toMatchObject({
|
|
266
278
|
events: 1,
|
|
267
279
|
succeeded: 0,
|
|
268
280
|
failed: 0,
|
|
@@ -310,7 +322,7 @@ describe('celilo lifecycle events', () => {
|
|
|
310
322
|
settleDelivery('succeed');
|
|
311
323
|
|
|
312
324
|
const result = await waitForRouteReconcile(since, { timeoutMs: 1000, pollMs: 50 });
|
|
313
|
-
expect(result).
|
|
325
|
+
expect(result).toMatchObject({
|
|
314
326
|
events: 1,
|
|
315
327
|
succeeded: 1,
|
|
316
328
|
failed: 0,
|
|
@@ -333,6 +345,68 @@ describe('celilo lifecycle events', () => {
|
|
|
333
345
|
expect(result.timedOut).toBe(false);
|
|
334
346
|
});
|
|
335
347
|
|
|
348
|
+
/**
|
|
349
|
+
* celilo#1416. The waiter's deadline was the constant 90s while caddy's
|
|
350
|
+
* subscription is `timeout_ms: 90000, max_attempts: 3` — so the ENTIRE wait
|
|
351
|
+
* equalled ONE attempt and a successful retry could never be observed. The
|
|
352
|
+
* budget must come from the subscriber's own registration.
|
|
353
|
+
*/
|
|
354
|
+
it('budgets for the subscriber whole retry allowance, not one attempt', async () => {
|
|
355
|
+
markDispatcherAlive();
|
|
356
|
+
subscribeReconciler({ maxAttempts: 3, timeoutMs: 90_000 });
|
|
357
|
+
const since = routesChangedHighWater();
|
|
358
|
+
emitWebRoutesChanged('celilo-website');
|
|
359
|
+
settleDelivery('succeed');
|
|
360
|
+
|
|
361
|
+
const result = await waitForRouteReconcile(since, { pollMs: 50 });
|
|
362
|
+
expect(result.succeeded).toBe(1);
|
|
363
|
+
expect(result.budgetMs).toBeGreaterThanOrEqual(subscriberBudgetMs(3, 90_000));
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
/** A deploy emitting N change events waits for N of them, not for one. */
|
|
367
|
+
it('scales the budget with the number of change events', async () => {
|
|
368
|
+
markDispatcherAlive();
|
|
369
|
+
subscribeReconciler({ maxAttempts: 2, timeoutMs: 60_000 });
|
|
370
|
+
const since = routesChangedHighWater();
|
|
371
|
+
emitWebRoutesChanged('celilo-website');
|
|
372
|
+
emitWebRoutesChanged('celilo-website');
|
|
373
|
+
settleAllDeliveries();
|
|
374
|
+
|
|
375
|
+
const result = await waitForRouteReconcile(since, { pollMs: 50 });
|
|
376
|
+
expect(result.events).toBe(2);
|
|
377
|
+
expect(result.budgetMs).toBeGreaterThanOrEqual(2 * subscriberBudgetMs(2, 60_000));
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
/** The end-to-end shape: one failed attempt, then a success, reports SUCCESS. */
|
|
381
|
+
it('reports success when the provider fails once and then succeeds', async () => {
|
|
382
|
+
markDispatcherAlive();
|
|
383
|
+
subscribeReconciler({ maxAttempts: 3, timeoutMs: 400 });
|
|
384
|
+
const since = routesChangedHighWater();
|
|
385
|
+
emitWebRoutesChanged('celilo-website');
|
|
386
|
+
recordRetryingFailure('transient boom');
|
|
387
|
+
|
|
388
|
+
const pending = waitForRouteReconcile(since, { pollMs: 50 });
|
|
389
|
+
await new Promise((r) => setTimeout(r, 120));
|
|
390
|
+
settleDelivery('succeed');
|
|
391
|
+
|
|
392
|
+
const result = await pending;
|
|
393
|
+
expect(result.timedOut).toBe(false);
|
|
394
|
+
expect(result.succeeded).toBe(1);
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
/** An expiry with deliveries in flight is not "nobody consumed it". */
|
|
398
|
+
it('separates deliveries still running from settled failures on expiry', async () => {
|
|
399
|
+
markDispatcherAlive();
|
|
400
|
+
subscribeReconciler();
|
|
401
|
+
const since = routesChangedHighWater();
|
|
402
|
+
emitWebRoutesChanged('celilo-website');
|
|
403
|
+
|
|
404
|
+
const result = await waitForRouteReconcile(since, { timeoutMs: 300, pollMs: 50 });
|
|
405
|
+
expect(result.timedOut).toBe(true);
|
|
406
|
+
expect(result.stillRunning).toBe(1);
|
|
407
|
+
expect(result.failed).toBe(0);
|
|
408
|
+
});
|
|
409
|
+
|
|
336
410
|
it('emitWebRoutesChangedAndWait emits then waits for the reconcile to settle', async () => {
|
|
337
411
|
markDispatcherAlive();
|
|
338
412
|
subscribeReconciler();
|
|
@@ -346,7 +420,7 @@ describe('celilo lifecycle events', () => {
|
|
|
346
420
|
settleDelivery('succeed');
|
|
347
421
|
|
|
348
422
|
const result = await pending;
|
|
349
|
-
expect(result).
|
|
423
|
+
expect(result).toMatchObject({
|
|
350
424
|
events: 1,
|
|
351
425
|
succeeded: 1,
|
|
352
426
|
failed: 0,
|
|
@@ -184,12 +184,36 @@ export function emitSystemDestroyed(payload: SystemDestroyedPayload): void {
|
|
|
184
184
|
* Best-effort — a failed emit never breaks register_route; the route is already
|
|
185
185
|
* persisted in web_routes and the provider's next deploy reconciles it anyway.
|
|
186
186
|
*/
|
|
187
|
-
export function emitWebRoutesChanged(
|
|
188
|
-
|
|
187
|
+
export function emitWebRoutesChanged(
|
|
188
|
+
triggeredBy: string,
|
|
189
|
+
type: RoutesChangedType = ROUTES_CHANGED_TYPE,
|
|
190
|
+
): void {
|
|
191
|
+
emitBest(type, { triggeredBy });
|
|
189
192
|
}
|
|
190
193
|
|
|
191
194
|
const ROUTES_CHANGED_TYPE = 'public_web.routes_changed';
|
|
192
195
|
|
|
196
|
+
/**
|
|
197
|
+
* The private ingress's twin of {@link ROUTES_CHANGED_TYPE}
|
|
198
|
+
* (`providers-converge-declared-state` slice 2). The two ingresses are
|
|
199
|
+
* deliberately SEPARATE event types rather than one shared signal: a private
|
|
200
|
+
* route lives in `private_routes`, a public one in `web_routes` (celilo#846),
|
|
201
|
+
* and waking the wrong provider would make it re-render from a table it does
|
|
202
|
+
* not serve.
|
|
203
|
+
*/
|
|
204
|
+
const PRIVATE_ROUTES_CHANGED_TYPE = 'private_web.routes_changed';
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Which ingress's route table changed. Every function below defaults to the
|
|
208
|
+
* public type, so existing callers are unaffected by construction.
|
|
209
|
+
*/
|
|
210
|
+
export type RoutesChangedType = typeof ROUTES_CHANGED_TYPE | typeof PRIVATE_ROUTES_CHANGED_TYPE;
|
|
211
|
+
|
|
212
|
+
/** Emit `private_web.routes_changed` — the private ingress's coarse signal. */
|
|
213
|
+
export function emitPrivateRoutesChanged(triggeredBy: string): void {
|
|
214
|
+
emitWebRoutesChanged(triggeredBy, PRIVATE_ROUTES_CHANGED_TYPE);
|
|
215
|
+
}
|
|
216
|
+
|
|
193
217
|
/**
|
|
194
218
|
* Emit `public_web.routes_changed` and block until the provider's reconcile
|
|
195
219
|
* delivery settles (ISS-0035). This is what `register_route` / `unregister_routes`
|
|
@@ -205,10 +229,30 @@ const ROUTES_CHANGED_TYPE = 'public_web.routes_changed';
|
|
|
205
229
|
export async function emitWebRoutesChangedAndWait(
|
|
206
230
|
triggeredBy: string,
|
|
207
231
|
opts?: { timeoutMs?: number; pollMs?: number },
|
|
232
|
+
type: RoutesChangedType = ROUTES_CHANGED_TYPE,
|
|
208
233
|
): Promise<RouteReconcileWaitResult> {
|
|
209
|
-
const since = routesChangedHighWater();
|
|
210
|
-
emitWebRoutesChanged(triggeredBy);
|
|
211
|
-
return waitForRouteReconcile(since, opts);
|
|
234
|
+
const since = routesChangedHighWater(type);
|
|
235
|
+
emitWebRoutesChanged(triggeredBy, type);
|
|
236
|
+
return waitForRouteReconcile(since, opts, type);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* The private ingress's twin of {@link emitWebRoutesChangedAndWait}
|
|
241
|
+
* (`providers-converge-declared-state` slice 2, D7).
|
|
242
|
+
*
|
|
243
|
+
* `registerReverseProxy` awaits THIS, so it still returns only once the route
|
|
244
|
+
* is actually live — the synchronous guarantee the in-call reconcile used to
|
|
245
|
+
* give, now provider-agnostic and on one path with the deploy.
|
|
246
|
+
*
|
|
247
|
+
* ⚠️ Callers must honour `noDispatcher` (ISS-0081): it means the event was
|
|
248
|
+
* persisted but never DELIVERED, so the route is NOT live. Treating it as a
|
|
249
|
+
* benign zero is the silent-success shape this change exists to remove.
|
|
250
|
+
*/
|
|
251
|
+
export async function emitPrivateRoutesChangedAndWait(
|
|
252
|
+
triggeredBy: string,
|
|
253
|
+
opts?: { timeoutMs?: number; pollMs?: number },
|
|
254
|
+
): Promise<RouteReconcileWaitResult> {
|
|
255
|
+
return emitWebRoutesChangedAndWait(triggeredBy, opts, PRIVATE_ROUTES_CHANGED_TYPE);
|
|
212
256
|
}
|
|
213
257
|
|
|
214
258
|
/**
|
|
@@ -240,6 +284,19 @@ export interface RouteReconcileWaitResult {
|
|
|
240
284
|
* failure, not a benign "0 deliveries" success.
|
|
241
285
|
*/
|
|
242
286
|
noDispatcher: boolean;
|
|
287
|
+
/**
|
|
288
|
+
* Deliveries still `pending`/`running` when the deadline hit. Non-zero means
|
|
289
|
+
* the provider had NOT given up — its retries were still in flight — which is
|
|
290
|
+
* a different thing from a failure and must not be reported as one
|
|
291
|
+
* (celilo#1416: "0 ok, 0 failed of 1" read as "nobody consumed it").
|
|
292
|
+
*/
|
|
293
|
+
stillRunning: number;
|
|
294
|
+
/**
|
|
295
|
+
* The deadline actually used, in ms. Derived from the subscribers' own retry
|
|
296
|
+
* budgets unless the caller passed `timeoutMs`, so the message can say what
|
|
297
|
+
* it waited for rather than quoting a constant.
|
|
298
|
+
*/
|
|
299
|
+
budgetMs: number;
|
|
243
300
|
}
|
|
244
301
|
|
|
245
302
|
/**
|
|
@@ -247,11 +304,11 @@ export interface RouteReconcileWaitResult {
|
|
|
247
304
|
* exist yet. A deploy captures this before it runs so {@link waitForRouteReconcile}
|
|
248
305
|
* can tell which route-change events the deploy itself produced.
|
|
249
306
|
*/
|
|
250
|
-
export function routesChangedHighWater(): number {
|
|
307
|
+
export function routesChangedHighWater(type: RoutesChangedType = ROUTES_CHANGED_TYPE): number {
|
|
251
308
|
let bus: ReturnType<typeof openBus> | undefined;
|
|
252
309
|
try {
|
|
253
310
|
bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
|
|
254
|
-
return bus.recentEvents({ type
|
|
311
|
+
return bus.recentEvents({ type, limit: 1 })[0]?.id ?? 0;
|
|
255
312
|
} catch (err) {
|
|
256
313
|
const msg = err instanceof Error ? err.message : String(err);
|
|
257
314
|
console.warn(`[celilo] failed to read routes-changed high-water: ${msg}`);
|
|
@@ -261,6 +318,27 @@ export function routesChangedHighWater(): number {
|
|
|
261
318
|
}
|
|
262
319
|
}
|
|
263
320
|
|
|
321
|
+
/**
|
|
322
|
+
* A subscriber's whole retry budget: every attempt's handler timeout plus the
|
|
323
|
+
* dispatcher's exponential backoff between them (1s, 2s, 4s..., capped at 5m —
|
|
324
|
+
* `backoffMs` in the event bus's dispatcher).
|
|
325
|
+
*
|
|
326
|
+
* Exported so the manifest gate that keeps consumer hook timeouts above this
|
|
327
|
+
* can compute the same number from the same formula rather than a second copy
|
|
328
|
+
* of it (celilo#1416).
|
|
329
|
+
*/
|
|
330
|
+
export function subscriberBudgetMs(maxAttempts: number, timeoutMs: number): number {
|
|
331
|
+
const attempts = Math.max(1, maxAttempts);
|
|
332
|
+
let budget = attempts * timeoutMs;
|
|
333
|
+
for (let attempt = 1; attempt < attempts; attempt++) {
|
|
334
|
+
budget += Math.min(1000 * 2 ** (attempt - 1), 5 * 60 * 1000);
|
|
335
|
+
}
|
|
336
|
+
return budget;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** Slack over the derived budget, so the wait outlives the last attempt. */
|
|
340
|
+
const RECONCILE_BUDGET_SLACK_MS = 5_000;
|
|
341
|
+
|
|
264
342
|
function sleep(ms: number): Promise<void> {
|
|
265
343
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
266
344
|
}
|
|
@@ -284,10 +362,15 @@ function sleep(ms: number): Promise<void> {
|
|
|
284
362
|
export async function waitForRouteReconcile(
|
|
285
363
|
sinceEventId: number,
|
|
286
364
|
opts: { timeoutMs?: number; pollMs?: number } = {},
|
|
365
|
+
type: RoutesChangedType = ROUTES_CHANGED_TYPE,
|
|
287
366
|
): Promise<RouteReconcileWaitResult> {
|
|
288
|
-
const timeoutMs = opts.timeoutMs ?? 90_000;
|
|
289
367
|
const pollMs = opts.pollMs ?? 250;
|
|
290
|
-
|
|
368
|
+
// The deadline is DERIVED below, once we know which subscribers are on the
|
|
369
|
+
// hook and what retry budget each was registered with. A constant here is
|
|
370
|
+
// what celilo#1416 was: 90s, exactly ONE attempt of a subscriber configured
|
|
371
|
+
// for three, so a successful retry could never be observed.
|
|
372
|
+
let budgetMs = opts.timeoutMs ?? 0;
|
|
373
|
+
let deadline = opts.timeoutMs !== undefined ? Date.now() + opts.timeoutMs : 0;
|
|
291
374
|
|
|
292
375
|
let bus: ReturnType<typeof openBus> | undefined;
|
|
293
376
|
try {
|
|
@@ -296,9 +379,7 @@ export async function waitForRouteReconcile(
|
|
|
296
379
|
|
|
297
380
|
// The deploy's emits already happened, so the set of route-change events
|
|
298
381
|
// is fixed; only their deliveries transition as the dispatcher works.
|
|
299
|
-
const events = b
|
|
300
|
-
.recentEvents({ type: ROUTES_CHANGED_TYPE, limit: 100 })
|
|
301
|
-
.filter((e) => e.id > sinceEventId);
|
|
382
|
+
const events = b.recentEvents({ type, limit: 100 }).filter((e) => e.id > sinceEventId);
|
|
302
383
|
|
|
303
384
|
if (events.length === 0) {
|
|
304
385
|
return {
|
|
@@ -308,6 +389,8 @@ export async function waitForRouteReconcile(
|
|
|
308
389
|
timedOut: false,
|
|
309
390
|
lastError: null,
|
|
310
391
|
noDispatcher: false,
|
|
392
|
+
stillRunning: 0,
|
|
393
|
+
budgetMs,
|
|
311
394
|
};
|
|
312
395
|
}
|
|
313
396
|
|
|
@@ -332,6 +415,8 @@ export async function waitForRouteReconcile(
|
|
|
332
415
|
timedOut: false,
|
|
333
416
|
lastError: null,
|
|
334
417
|
noDispatcher: true,
|
|
418
|
+
stillRunning: 0,
|
|
419
|
+
budgetMs,
|
|
335
420
|
};
|
|
336
421
|
}
|
|
337
422
|
|
|
@@ -342,6 +427,20 @@ export async function waitForRouteReconcile(
|
|
|
342
427
|
(d) => d.status === 'pending' || d.status === 'running',
|
|
343
428
|
).length;
|
|
344
429
|
|
|
430
|
+
if (deadline === 0) {
|
|
431
|
+
// Every subscriber gets its whole registered retry budget, and a deploy
|
|
432
|
+
// that emitted N change events is waiting on N of them — budgeting for
|
|
433
|
+
// one is how a two-event deploy charged both against a single 90s wall.
|
|
434
|
+
const subscriberIds = [...new Set(deliveries.map((d) => d.subscriberId))];
|
|
435
|
+
const worst = subscriberIds.reduce((max, id) => {
|
|
436
|
+
const sub = b.getSubscriber(id);
|
|
437
|
+
if (!sub) return max;
|
|
438
|
+
return Math.max(max, subscriberBudgetMs(sub.maxAttempts, sub.timeoutMs));
|
|
439
|
+
}, 0);
|
|
440
|
+
budgetMs = worst * events.length + RECONCILE_BUDGET_SLACK_MS;
|
|
441
|
+
deadline = Date.now() + budgetMs;
|
|
442
|
+
}
|
|
443
|
+
|
|
345
444
|
if (pending === 0 || Date.now() >= deadline) {
|
|
346
445
|
return {
|
|
347
446
|
events: events.length,
|
|
@@ -350,6 +449,8 @@ export async function waitForRouteReconcile(
|
|
|
350
449
|
timedOut: pending > 0,
|
|
351
450
|
lastError: deliveries.find((d) => d.lastError)?.lastError ?? null,
|
|
352
451
|
noDispatcher: false,
|
|
452
|
+
stillRunning: pending,
|
|
453
|
+
budgetMs,
|
|
353
454
|
};
|
|
354
455
|
}
|
|
355
456
|
|
|
@@ -365,6 +466,8 @@ export async function waitForRouteReconcile(
|
|
|
365
466
|
timedOut: false,
|
|
366
467
|
lastError: null,
|
|
367
468
|
noDispatcher: false,
|
|
469
|
+
stillRunning: 0,
|
|
470
|
+
budgetMs,
|
|
368
471
|
};
|
|
369
472
|
} finally {
|
|
370
473
|
bus?.close();
|
|
@@ -5,6 +5,7 @@ import { join } from 'node:path';
|
|
|
5
5
|
import { skipIntegration } from '../test-utils/integration-guard';
|
|
6
6
|
|
|
7
7
|
import type { HookContext } from '@celilo/capabilities';
|
|
8
|
+
import { configStore, secretStore } from '../hooks/test-fixtures/store-backed';
|
|
8
9
|
|
|
9
10
|
// Hooks under test — imported via the scripts dir adjacent to the
|
|
10
11
|
// modules/celilo-mgmt/manifest.yml. The dynamic import keeps the test
|
|
@@ -46,7 +47,7 @@ function buildContext(extras: Record<string, unknown>): HookContext {
|
|
|
46
47
|
const { logger } = buildLogger();
|
|
47
48
|
return {
|
|
48
49
|
logger,
|
|
49
|
-
config: {
|
|
50
|
+
config: configStore({
|
|
50
51
|
db_path: '', // populated by caller via env
|
|
51
52
|
hostname: 'test-mgmt',
|
|
52
53
|
target_ip: '127.0.0.1',
|
|
@@ -61,9 +62,11 @@ function buildContext(extras: Record<string, unknown>): HookContext {
|
|
|
61
62
|
ssh_public_key: '',
|
|
62
63
|
install_docker: false,
|
|
63
64
|
install_terraform: false,
|
|
64
|
-
}
|
|
65
|
-
secrets:
|
|
65
|
+
}),
|
|
66
|
+
secrets: secretStore(),
|
|
66
67
|
capabilities: {},
|
|
68
|
+
systems: [],
|
|
69
|
+
stateDir: '',
|
|
67
70
|
debug: false,
|
|
68
71
|
screenshotDir: '',
|
|
69
72
|
...extras,
|
|
@@ -25,7 +25,7 @@ import {
|
|
|
25
25
|
|
|
26
26
|
function buildManifest(caps: Array<{ name: string; version: string }>): ModuleManifest {
|
|
27
27
|
return {
|
|
28
|
-
celilo_contract: '
|
|
28
|
+
celilo_contract: '2.0',
|
|
29
29
|
id: 'm',
|
|
30
30
|
name: 'm',
|
|
31
31
|
version: '0.0.1',
|
|
@@ -88,7 +88,7 @@ describe('materializeCrossModuleRoot', () => {
|
|
|
88
88
|
name: id,
|
|
89
89
|
version,
|
|
90
90
|
sourcePath: join(dir, 'sources', id),
|
|
91
|
-
manifestData: { id, name: id, version, celilo_contract: '
|
|
91
|
+
manifestData: { id, name: id, version, celilo_contract: '2.0', requires: {} } as Record<
|
|
92
92
|
string,
|
|
93
93
|
unknown
|
|
94
94
|
>,
|
|
@@ -31,7 +31,7 @@ import {
|
|
|
31
31
|
|
|
32
32
|
function manifestWithStringMapSecret(): ModuleManifest {
|
|
33
33
|
return {
|
|
34
|
-
celilo_contract: '
|
|
34
|
+
celilo_contract: '2.0',
|
|
35
35
|
id: 'testmod',
|
|
36
36
|
name: 'Test Module',
|
|
37
37
|
version: '1.0.0',
|
|
@@ -54,7 +54,7 @@ function manifestWithStringMapSecret(): ModuleManifest {
|
|
|
54
54
|
|
|
55
55
|
function manifestWithPlainStringSecret(): ModuleManifest {
|
|
56
56
|
return {
|
|
57
|
-
celilo_contract: '
|
|
57
|
+
celilo_contract: '2.0',
|
|
58
58
|
id: 'testmod',
|
|
59
59
|
name: 'Test Module',
|
|
60
60
|
version: '1.0.0',
|
|
@@ -140,7 +140,7 @@ describe('findMissingRequiredVariables (deploy path)', () => {
|
|
|
140
140
|
// for the interview, and not as a deploy-blocking error.
|
|
141
141
|
test('a source: hook variable is never reported missing, even required and unwritten', async () => {
|
|
142
142
|
const manifest = {
|
|
143
|
-
celilo_contract: '
|
|
143
|
+
celilo_contract: '2.0',
|
|
144
144
|
variables: {
|
|
145
145
|
owns: [
|
|
146
146
|
{
|
|
@@ -341,7 +341,7 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
|
|
|
341
341
|
|
|
342
342
|
function manifestWithArtifacts(artifacts: string[]): ModuleManifest {
|
|
343
343
|
return {
|
|
344
|
-
celilo_contract: '
|
|
344
|
+
celilo_contract: '2.0',
|
|
345
345
|
id: 'buildmod',
|
|
346
346
|
name: 'Build Module',
|
|
347
347
|
version: '1.0.0',
|
|
@@ -351,7 +351,7 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
|
|
|
351
351
|
}
|
|
352
352
|
|
|
353
353
|
test('returns [] when the module declares no build section', () => {
|
|
354
|
-
const manifest = { celilo_contract: '
|
|
354
|
+
const manifest = { celilo_contract: '2.0', id: 'm', version: '1.0.0' } as ModuleManifest;
|
|
355
355
|
expect(findMissingBuildArtifacts(manifest, dir)).toEqual([]);
|
|
356
356
|
});
|
|
357
357
|
|
|
@@ -391,7 +391,7 @@ describe('validateAndPrepareDeployment (optional capability version gate)', () =
|
|
|
391
391
|
sourcePath: tempDir,
|
|
392
392
|
version: '1.0.0',
|
|
393
393
|
manifestData: {
|
|
394
|
-
celilo_contract: '
|
|
394
|
+
celilo_contract: '2.0',
|
|
395
395
|
id: 'consumer',
|
|
396
396
|
name: 'Consumer',
|
|
397
397
|
version: '1.0.0',
|
|
@@ -42,7 +42,7 @@ describe('backfillWebRouteDns (ISS-0029)', () => {
|
|
|
42
42
|
id,
|
|
43
43
|
name: id,
|
|
44
44
|
version: '1.0.0',
|
|
45
|
-
manifestData: { id, name: id, version: '1.0.0', celilo_contract: '
|
|
45
|
+
manifestData: { id, name: id, version: '1.0.0', celilo_contract: '2.0' },
|
|
46
46
|
sourcePath: `/tmp/${id}`,
|
|
47
47
|
})
|
|
48
48
|
.run();
|
|
@@ -73,7 +73,7 @@ describe('backfillWebRouteDns (ISS-0029)', () => {
|
|
|
73
73
|
id: 'caddy',
|
|
74
74
|
name: 'caddy',
|
|
75
75
|
version: '1.0.0',
|
|
76
|
-
manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '
|
|
76
|
+
manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '2.0' },
|
|
77
77
|
sourcePath: '/tmp/caddy',
|
|
78
78
|
})
|
|
79
79
|
.onConflictDoNothing()
|
|
@@ -37,7 +37,7 @@ describe('dns_registrations ledger', () => {
|
|
|
37
37
|
name: id,
|
|
38
38
|
sourcePath: join(tempDir, id),
|
|
39
39
|
version: '1.0.0',
|
|
40
|
-
manifestData: { celilo_contract: '
|
|
40
|
+
manifestData: { celilo_contract: '2.0', id, name: id, version: '1.0.0' },
|
|
41
41
|
})
|
|
42
42
|
.run();
|
|
43
43
|
}
|
|
@@ -70,7 +70,7 @@ function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', system
|
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
const baseManifest = (overrides: Partial<ModuleManifest> = {}): ModuleManifest => ({
|
|
73
|
-
celilo_contract: '
|
|
73
|
+
celilo_contract: '2.0',
|
|
74
74
|
id: 'mod',
|
|
75
75
|
name: 'Mod',
|
|
76
76
|
version: '1.0.0',
|
|
@@ -340,7 +340,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
|
|
|
340
340
|
writeFileSync(
|
|
341
341
|
join(src, 'manifest.yml'),
|
|
342
342
|
[
|
|
343
|
-
'celilo_contract: "
|
|
343
|
+
'celilo_contract: "2.0"',
|
|
344
344
|
`id: ${id}`,
|
|
345
345
|
`name: ${id}`,
|
|
346
346
|
'version: 1.0.0',
|
|
@@ -106,7 +106,7 @@ describe('runModuleHealthCheck — clearing a recorded failure (celilo#1363)', (
|
|
|
106
106
|
state: 'ERROR',
|
|
107
107
|
errorMessage: 'Upgrade to 1.0.0+6 failed to deploy: ssh: connection refused',
|
|
108
108
|
manifestData: {
|
|
109
|
-
celilo_contract: '
|
|
109
|
+
celilo_contract: '2.0',
|
|
110
110
|
id: 'celilo-mgmt',
|
|
111
111
|
name: 'Control plane',
|
|
112
112
|
version: '1.0.0',
|