@celilo/cli 4.1.0 → 5.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/AGENTS.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +2 -0
  3. package/MODULE_PRIMITIVES.md +9 -0
  4. package/drizzle/0034_private_routes.sql +41 -0
  5. package/drizzle/meta/_journal.json +8 -1
  6. package/package.json +2 -2
  7. package/src/api/sessions.test.ts +1 -1
  8. package/src/capabilities/registration.test.ts +11 -11
  9. package/src/capabilities/secrets.test.ts +18 -18
  10. package/src/capabilities/validation.test.ts +24 -24
  11. package/src/cli/cli.test.ts +3 -3
  12. package/src/cli/commands/module-changeset.test.ts +1 -1
  13. package/src/cli/commands/module-generate.test.ts +2 -2
  14. package/src/cli/commands/module-import-aspect.test.ts +2 -2
  15. package/src/cli/commands/module-update.test.ts +14 -14
  16. package/src/cli/commands/module-upgrade-gate.test.ts +26 -1
  17. package/src/cli/commands/module-upgrade.test.ts +3 -3
  18. package/src/cli/commands/module-upgrade.ts +12 -0
  19. package/src/cli/commands/module-version.test.ts +1 -1
  20. package/src/cli/index.ts +22 -2
  21. package/src/cli/restore-migration-failure.test.ts +1 -1
  22. package/src/cli/stdout-pipe-flush.test.ts +35 -0
  23. package/src/db/private-routes.test.ts +159 -0
  24. package/src/db/schema.ts +49 -0
  25. package/src/hooks/capability-loader.ts +179 -12
  26. package/src/hooks/executor.test.ts +13 -13
  27. package/src/hooks/hook-state-dir.test.ts +2 -2
  28. package/src/hooks/mount-set.test.ts +28 -0
  29. package/src/hooks/mount-set.ts +33 -0
  30. package/src/hooks/test-fixtures/store-backed.ts +5 -3
  31. package/src/manifest/contracts/index.ts +28 -7
  32. package/src/manifest/contracts/v2.ts +32 -0
  33. package/src/manifest/icon-schema.test.ts +1 -1
  34. package/src/manifest/schema.ts +27 -2
  35. package/src/manifest/template-validator.test.ts +1 -1
  36. package/src/manifest/validate-privileged.test.ts +1 -1
  37. package/src/manifest/validate.test.ts +68 -29
  38. package/src/manifest/validate.ts +14 -1
  39. package/src/module/import.test.ts +7 -7
  40. package/src/module/packaging/build.test.ts +1 -1
  41. package/src/module/packaging/build.ts +16 -2
  42. package/src/packaging/nfpm-depends.test.ts +42 -0
  43. package/src/policy/capability-shape-baseline.ts +7 -1
  44. package/src/policy/fixture-capability-coverage.test.ts +5 -5
  45. package/src/policy/module-business-baseline.ts +33 -4
  46. package/src/registry/client.test.ts +5 -4
  47. package/src/services/aspect-approvals.test.ts +2 -2
  48. package/src/services/aspect-reconcile.test.ts +2 -2
  49. package/src/services/aspect-runner.test.ts +4 -4
  50. package/src/services/audit/backups.test.ts +1 -1
  51. package/src/services/audit/capability-abi.test.ts +1 -1
  52. package/src/services/audit/module-configs.test.ts +1 -1
  53. package/src/services/build-bus/hook-dispatch-executor.test.ts +3 -3
  54. package/src/services/bus-interview-park.test.ts +1 -1
  55. package/src/services/bus-secret-flow.test.ts +1 -1
  56. package/src/services/capability-table-rows.test.ts +20 -2
  57. package/src/services/celilo-events.test.ts +80 -6
  58. package/src/services/celilo-events.ts +115 -12
  59. package/src/services/celilo-mgmt-hooks.test.ts +6 -3
  60. package/src/services/cross-module-read.test.ts +2 -2
  61. package/src/services/deploy-validation.test.ts +6 -6
  62. package/src/services/dns-provider-backfill.test.ts +2 -2
  63. package/src/services/dns-registrations.test.ts +1 -1
  64. package/src/services/fleet-checks.test.ts +2 -2
  65. package/src/services/health-runner.test.ts +1 -1
  66. package/src/services/infrastructure-variable-resolver.test.ts +11 -11
  67. package/src/services/instance-ops.test.ts +2 -2
  68. package/src/services/module-deploy.dns-repoint.test.ts +1 -1
  69. package/src/services/module-pause-quiescence.test.ts +1 -1
  70. package/src/services/module-pause.test.ts +1 -1
  71. package/src/services/module-subscriptions.test.ts +2 -2
  72. package/src/services/module-types-generator.test.ts +1 -1
  73. package/src/services/module-validator/contract-version.test.ts +3 -3
  74. package/src/services/network-ensure.test.ts +1 -1
  75. package/src/services/proxmox-reconcile.test.ts +2 -2
  76. package/src/services/remove-guard.test.ts +1 -1
  77. package/src/services/update/dep-graph.test.ts +1 -1
  78. package/src/services/update/orchestrator.test.ts +1 -1
  79. package/src/variables/capability-self-ref.test.ts +2 -2
  80. package/src/variables/computed/computed-integration.test.ts +1 -1
  81. package/src/variables/declarative-derivation.test.ts +19 -19
@@ -62,7 +62,7 @@ function seedProvider(opts: {
62
62
  id: opts.moduleId,
63
63
  name: opts.moduleId,
64
64
  version: '1.0.0',
65
- celilo_contract: '1.0',
65
+ celilo_contract: '2.0',
66
66
  base_module_aspect: aspect,
67
67
  },
68
68
  sourcePath: `/tmp/${opts.moduleId}`,
@@ -108,7 +108,7 @@ function seedTargetSystem(opts: {
108
108
  id: opts.moduleId,
109
109
  name: opts.moduleId,
110
110
  version: '1.0.0',
111
- celilo_contract: '1.0',
111
+ celilo_contract: '2.0',
112
112
  },
113
113
  sourcePath: `/tmp/${opts.moduleId}`,
114
114
  })
@@ -85,7 +85,7 @@ function seedLxc(opts: {
85
85
  id: opts.moduleId,
86
86
  name: opts.moduleId,
87
87
  version: '1.0.0',
88
- celilo_contract: '1.0',
88
+ celilo_contract: '2.0',
89
89
  },
90
90
  sourcePath: `/tmp/${opts.moduleId}`,
91
91
  })
@@ -320,7 +320,7 @@ describe('aspect-runner', () => {
320
320
  id: 'knot-unbound-internal',
321
321
  name: 'knot-unbound-internal',
322
322
  version: '1.0.0',
323
- celilo_contract: '1.0',
323
+ celilo_contract: '2.0',
324
324
  },
325
325
  sourcePath: '/tmp/knot',
326
326
  })
@@ -386,7 +386,7 @@ describe('aspect-runner', () => {
386
386
 
387
387
  describe('maybeRunAspectForTrigger', () => {
388
388
  const manifestWithAspect: ModuleManifest = {
389
- celilo_contract: '1.0',
389
+ celilo_contract: '2.0',
390
390
  id: 'knot-unbound-internal',
391
391
  name: 'Knot Unbound Internal',
392
392
  version: '1.0.0',
@@ -413,7 +413,7 @@ describe('aspect-runner', () => {
413
413
  id: opts.id,
414
414
  name: opts.id,
415
415
  version: opts.version,
416
- celilo_contract: '1.0',
416
+ celilo_contract: '2.0',
417
417
  },
418
418
  sourcePath: `/tmp/${opts.id}`,
419
419
  })
@@ -20,7 +20,7 @@ function makeModule(
20
20
  id,
21
21
  name: id,
22
22
  version: '1.0.0',
23
- celilo_contract: '1.0',
23
+ celilo_contract: '2.0',
24
24
  hooks: opts.hasBackupHook ? { on_backup: { script: './backup.ts' } } : {},
25
25
  backup: opts.schedule ? { schedule: opts.schedule } : undefined,
26
26
  } as unknown as ModuleManifest;
@@ -14,7 +14,7 @@ function makeModule(
14
14
  id,
15
15
  name: id,
16
16
  version: '1.0.0',
17
- celilo_contract: '1.0',
17
+ celilo_contract: '2.0',
18
18
  provides: opts.provides
19
19
  ? {
20
20
  capabilities: opts.provides.map((p) => ({ ...p, data: {}, functions: [] })),
@@ -21,7 +21,7 @@ function makeModule(
21
21
  id,
22
22
  name: id,
23
23
  version: '1.0.0',
24
- celilo_contract: '1.0',
24
+ celilo_contract: '2.0',
25
25
  variables: { owns: variables, imports: [] },
26
26
  } as unknown as ModuleManifest;
27
27
  // Default state INSTALLED so existing tests continue to expect
@@ -121,7 +121,7 @@ describe('the dispatcher dispatches on_upstream_publish through the ordinary exe
121
121
  id: 'upstream-fixture',
122
122
  name: 'upstream-fixture',
123
123
  version: '1.0.0',
124
- celilo_contract: '1.0',
124
+ celilo_contract: '2.0',
125
125
  provides: { capabilities: [] },
126
126
  requires: { capabilities: [] },
127
127
  hooks: { on_upstream_publish: [hookEntry] },
@@ -178,7 +178,7 @@ describe('the dispatcher dispatches on_upstream_publish through the ordinary exe
178
178
  id: 'upstream-fixture',
179
179
  name: 'upstream-fixture',
180
180
  version: '1.0.0',
181
- celilo_contract: '1.0',
181
+ celilo_contract: '2.0',
182
182
  provides: { capabilities: [] },
183
183
  requires: { capabilities: [] },
184
184
  hooks: { on_upstream_publish: [hookEntry] },
@@ -232,7 +232,7 @@ test('a failing hook run lands in the build_bus_hook_runs ledger', async () => {
232
232
  id: 'upstream-fixture',
233
233
  name: 'upstream-fixture',
234
234
  version: '1.0.0',
235
- celilo_contract: '1.0',
235
+ celilo_contract: '2.0',
236
236
  provides: { capabilities: [] },
237
237
  requires: { capabilities: [] },
238
238
  hooks: { on_upstream_publish: [hookEntry] },
@@ -71,7 +71,7 @@ beforeEach(() => {
71
71
  name: 'iptables',
72
72
  sourcePath: join(dir, 'installed'),
73
73
  version: '1.0.2+9',
74
- manifestData: { celilo_contract: '1.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
74
+ manifestData: { celilo_contract: '2.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
75
75
  })
76
76
  .run();
77
77
 
@@ -396,7 +396,7 @@ describe('bus-mediated interviewForMissingSecrets', () => {
396
396
  writeFileSync(
397
397
  join(tempDir, 'manifest.yml'),
398
398
  `
399
- celilo_contract: "1.0"
399
+ celilo_contract: "2.0"
400
400
  id: testmod
401
401
  name: Test Module
402
402
  version: 1.0.0
@@ -18,7 +18,14 @@ import { tmpdir } from 'node:os';
18
18
  import { join } from 'node:path';
19
19
  import { allDeclaredTables } from '@celilo/capabilities';
20
20
  import type { DbClient } from '../db/client';
21
- import { dnsInternalRecords, modules, portForwards, trustedSources, webRoutes } from '../db/schema';
21
+ import {
22
+ dnsInternalRecords,
23
+ modules,
24
+ portForwards,
25
+ privateRoutes,
26
+ trustedSources,
27
+ webRoutes,
28
+ } from '../db/schema';
22
29
  import { setupTestDatabaseAt } from '../test-utils/database';
23
30
  import { resetTestDbPath } from '../test-utils/db-path';
24
31
  import { deleteClaimedRows, planClaimedRowDeletion } from './capability-table-rows';
@@ -89,6 +96,15 @@ describe('deleteClaimedRows', () => {
89
96
  hostname,
90
97
  })
91
98
  .run();
99
+ db.insert(privateRoutes)
100
+ .values({
101
+ slug: `${consumer}-private-slug`,
102
+ moduleId: consumer,
103
+ type: 'reverse_proxy',
104
+ path: `/${consumer}`,
105
+ hostname: `private-${hostname}`,
106
+ })
107
+ .run();
92
108
  db.insert(portForwards)
93
109
  .values({
94
110
  firewallIp: FW,
@@ -109,7 +125,7 @@ describe('deleteClaimedRows', () => {
109
125
  .run();
110
126
  }
111
127
 
112
- it('clears the departing consumer from all four declared tables at once', () => {
128
+ it('clears the departing consumer from every declared table at once', () => {
113
129
  seed('caddy', 'a.example.org', 443, '10.1.0.0/24');
114
130
 
115
131
  const cleared = deleteClaimedRows(db, 'caddy');
@@ -117,10 +133,12 @@ describe('deleteClaimedRows', () => {
117
133
  expect(cleared.map((c) => c.table).sort()).toEqual([
118
134
  'dns_internal_records',
119
135
  'port_forwards',
136
+ 'private_routes',
120
137
  'trusted_sources',
121
138
  'web_routes',
122
139
  ]);
123
140
  expect(db.select().from(webRoutes).all()).toHaveLength(0);
141
+ expect(db.select().from(privateRoutes).all()).toHaveLength(0);
124
142
  expect(db.select().from(portForwards).all()).toHaveLength(0);
125
143
  expect(db.select().from(trustedSources).all()).toHaveLength(0);
126
144
  expect(db.select().from(dnsInternalRecords).all()).toHaveLength(0);
@@ -16,6 +16,7 @@ import {
16
16
  emitWebRoutesChanged,
17
17
  emitWebRoutesChangedAndWait,
18
18
  routesChangedHighWater,
19
+ subscriberBudgetMs,
19
20
  waitForRouteReconcile,
20
21
  } from './celilo-events';
21
22
 
@@ -171,12 +172,13 @@ describe('celilo lifecycle events', () => {
171
172
  });
172
173
 
173
174
  describe('deploy-waits for web-route reconcile (ISS-0035)', () => {
174
- function subscribeReconciler(): void {
175
+ function subscribeReconciler(budget?: { maxAttempts: number; timeoutMs: number }): void {
175
176
  const bus = openBus({ dbPath, events: defineEvents({}) });
176
177
  bus.subscribe({
177
178
  name: 'caddy.reconcile-web-routes',
178
179
  pattern: 'public_web.routes_changed',
179
180
  handler: 'unused',
181
+ ...budget,
180
182
  });
181
183
  bus.close();
182
184
  }
@@ -208,6 +210,16 @@ describe('celilo lifecycle events', () => {
208
210
  bus.close();
209
211
  }
210
212
 
213
+ function settleAllDeliveries(): void {
214
+ const bus = openBus({ dbPath, events: defineEvents({}) });
215
+ const sub = bus.getSubscriberByName('caddy.reconcile-web-routes');
216
+ if (!sub) throw new Error('expected a subscriber');
217
+ for (const event of bus.recentEvents({ type: 'public_web.routes_changed', limit: 100 })) {
218
+ bus.markSucceeded({ eventId: event.id, subscriberId: sub.id });
219
+ }
220
+ bus.close();
221
+ }
222
+
211
223
  /** One failed attempt that will be retried — the delivery stays pending. */
212
224
  function recordRetryingFailure(message: string): void {
213
225
  const bus = openBus({ dbPath, events: defineEvents({}) });
@@ -231,7 +243,7 @@ describe('celilo lifecycle events', () => {
231
243
  it('returns immediately when the deploy changed no routes', async () => {
232
244
  const since = routesChangedHighWater();
233
245
  const result = await waitForRouteReconcile(since, { timeoutMs: 1000 });
234
- expect(result).toEqual({
246
+ expect(result).toMatchObject({
235
247
  events: 0,
236
248
  succeeded: 0,
237
249
  failed: 0,
@@ -247,7 +259,7 @@ describe('celilo lifecycle events', () => {
247
259
  emitWebRoutesChanged('celilo-website');
248
260
  // No heartbeat stamped → health() is no_dispatcher.
249
261
  const result = await waitForRouteReconcile(since, { timeoutMs: 5000, pollMs: 50 });
250
- expect(result).toEqual({
262
+ expect(result).toMatchObject({
251
263
  events: 1,
252
264
  succeeded: 0,
253
265
  failed: 0,
@@ -262,7 +274,7 @@ describe('celilo lifecycle events', () => {
262
274
  const since = routesChangedHighWater();
263
275
  emitWebRoutesChanged('celilo-website');
264
276
  const result = await waitForRouteReconcile(since, { timeoutMs: 1000, pollMs: 50 });
265
- expect(result).toEqual({
277
+ expect(result).toMatchObject({
266
278
  events: 1,
267
279
  succeeded: 0,
268
280
  failed: 0,
@@ -310,7 +322,7 @@ describe('celilo lifecycle events', () => {
310
322
  settleDelivery('succeed');
311
323
 
312
324
  const result = await waitForRouteReconcile(since, { timeoutMs: 1000, pollMs: 50 });
313
- expect(result).toEqual({
325
+ expect(result).toMatchObject({
314
326
  events: 1,
315
327
  succeeded: 1,
316
328
  failed: 0,
@@ -333,6 +345,68 @@ describe('celilo lifecycle events', () => {
333
345
  expect(result.timedOut).toBe(false);
334
346
  });
335
347
 
348
+ /**
349
+ * celilo#1416. The waiter's deadline was the constant 90s while caddy's
350
+ * subscription is `timeout_ms: 90000, max_attempts: 3` — so the ENTIRE wait
351
+ * equalled ONE attempt and a successful retry could never be observed. The
352
+ * budget must come from the subscriber's own registration.
353
+ */
354
+ it('budgets for the subscriber whole retry allowance, not one attempt', async () => {
355
+ markDispatcherAlive();
356
+ subscribeReconciler({ maxAttempts: 3, timeoutMs: 90_000 });
357
+ const since = routesChangedHighWater();
358
+ emitWebRoutesChanged('celilo-website');
359
+ settleDelivery('succeed');
360
+
361
+ const result = await waitForRouteReconcile(since, { pollMs: 50 });
362
+ expect(result.succeeded).toBe(1);
363
+ expect(result.budgetMs).toBeGreaterThanOrEqual(subscriberBudgetMs(3, 90_000));
364
+ });
365
+
366
+ /** A deploy emitting N change events waits for N of them, not for one. */
367
+ it('scales the budget with the number of change events', async () => {
368
+ markDispatcherAlive();
369
+ subscribeReconciler({ maxAttempts: 2, timeoutMs: 60_000 });
370
+ const since = routesChangedHighWater();
371
+ emitWebRoutesChanged('celilo-website');
372
+ emitWebRoutesChanged('celilo-website');
373
+ settleAllDeliveries();
374
+
375
+ const result = await waitForRouteReconcile(since, { pollMs: 50 });
376
+ expect(result.events).toBe(2);
377
+ expect(result.budgetMs).toBeGreaterThanOrEqual(2 * subscriberBudgetMs(2, 60_000));
378
+ });
379
+
380
+ /** The end-to-end shape: one failed attempt, then a success, reports SUCCESS. */
381
+ it('reports success when the provider fails once and then succeeds', async () => {
382
+ markDispatcherAlive();
383
+ subscribeReconciler({ maxAttempts: 3, timeoutMs: 400 });
384
+ const since = routesChangedHighWater();
385
+ emitWebRoutesChanged('celilo-website');
386
+ recordRetryingFailure('transient boom');
387
+
388
+ const pending = waitForRouteReconcile(since, { pollMs: 50 });
389
+ await new Promise((r) => setTimeout(r, 120));
390
+ settleDelivery('succeed');
391
+
392
+ const result = await pending;
393
+ expect(result.timedOut).toBe(false);
394
+ expect(result.succeeded).toBe(1);
395
+ });
396
+
397
+ /** An expiry with deliveries in flight is not "nobody consumed it". */
398
+ it('separates deliveries still running from settled failures on expiry', async () => {
399
+ markDispatcherAlive();
400
+ subscribeReconciler();
401
+ const since = routesChangedHighWater();
402
+ emitWebRoutesChanged('celilo-website');
403
+
404
+ const result = await waitForRouteReconcile(since, { timeoutMs: 300, pollMs: 50 });
405
+ expect(result.timedOut).toBe(true);
406
+ expect(result.stillRunning).toBe(1);
407
+ expect(result.failed).toBe(0);
408
+ });
409
+
336
410
  it('emitWebRoutesChangedAndWait emits then waits for the reconcile to settle', async () => {
337
411
  markDispatcherAlive();
338
412
  subscribeReconciler();
@@ -346,7 +420,7 @@ describe('celilo lifecycle events', () => {
346
420
  settleDelivery('succeed');
347
421
 
348
422
  const result = await pending;
349
- expect(result).toEqual({
423
+ expect(result).toMatchObject({
350
424
  events: 1,
351
425
  succeeded: 1,
352
426
  failed: 0,
@@ -184,12 +184,36 @@ export function emitSystemDestroyed(payload: SystemDestroyedPayload): void {
184
184
  * Best-effort — a failed emit never breaks register_route; the route is already
185
185
  * persisted in web_routes and the provider's next deploy reconciles it anyway.
186
186
  */
187
- export function emitWebRoutesChanged(triggeredBy: string): void {
188
- emitBest('public_web.routes_changed', { triggeredBy });
187
+ export function emitWebRoutesChanged(
188
+ triggeredBy: string,
189
+ type: RoutesChangedType = ROUTES_CHANGED_TYPE,
190
+ ): void {
191
+ emitBest(type, { triggeredBy });
189
192
  }
190
193
 
191
194
  const ROUTES_CHANGED_TYPE = 'public_web.routes_changed';
192
195
 
196
+ /**
197
+ * The private ingress's twin of {@link ROUTES_CHANGED_TYPE}
198
+ * (`providers-converge-declared-state` slice 2). The two ingresses are
199
+ * deliberately SEPARATE event types rather than one shared signal: a private
200
+ * route lives in `private_routes`, a public one in `web_routes` (celilo#846),
201
+ * and waking the wrong provider would make it re-render from a table it does
202
+ * not serve.
203
+ */
204
+ const PRIVATE_ROUTES_CHANGED_TYPE = 'private_web.routes_changed';
205
+
206
+ /**
207
+ * Which ingress's route table changed. Every function below defaults to the
208
+ * public type, so existing callers are unaffected by construction.
209
+ */
210
+ export type RoutesChangedType = typeof ROUTES_CHANGED_TYPE | typeof PRIVATE_ROUTES_CHANGED_TYPE;
211
+
212
+ /** Emit `private_web.routes_changed` — the private ingress's coarse signal. */
213
+ export function emitPrivateRoutesChanged(triggeredBy: string): void {
214
+ emitWebRoutesChanged(triggeredBy, PRIVATE_ROUTES_CHANGED_TYPE);
215
+ }
216
+
193
217
  /**
194
218
  * Emit `public_web.routes_changed` and block until the provider's reconcile
195
219
  * delivery settles (ISS-0035). This is what `register_route` / `unregister_routes`
@@ -205,10 +229,30 @@ const ROUTES_CHANGED_TYPE = 'public_web.routes_changed';
205
229
  export async function emitWebRoutesChangedAndWait(
206
230
  triggeredBy: string,
207
231
  opts?: { timeoutMs?: number; pollMs?: number },
232
+ type: RoutesChangedType = ROUTES_CHANGED_TYPE,
208
233
  ): Promise<RouteReconcileWaitResult> {
209
- const since = routesChangedHighWater();
210
- emitWebRoutesChanged(triggeredBy);
211
- return waitForRouteReconcile(since, opts);
234
+ const since = routesChangedHighWater(type);
235
+ emitWebRoutesChanged(triggeredBy, type);
236
+ return waitForRouteReconcile(since, opts, type);
237
+ }
238
+
239
+ /**
240
+ * The private ingress's twin of {@link emitWebRoutesChangedAndWait}
241
+ * (`providers-converge-declared-state` slice 2, D7).
242
+ *
243
+ * `registerReverseProxy` awaits THIS, so it still returns only once the route
244
+ * is actually live — the synchronous guarantee the in-call reconcile used to
245
+ * give, now provider-agnostic and on one path with the deploy.
246
+ *
247
+ * ⚠️ Callers must honour `noDispatcher` (ISS-0081): it means the event was
248
+ * persisted but never DELIVERED, so the route is NOT live. Treating it as a
249
+ * benign zero is the silent-success shape this change exists to remove.
250
+ */
251
+ export async function emitPrivateRoutesChangedAndWait(
252
+ triggeredBy: string,
253
+ opts?: { timeoutMs?: number; pollMs?: number },
254
+ ): Promise<RouteReconcileWaitResult> {
255
+ return emitWebRoutesChangedAndWait(triggeredBy, opts, PRIVATE_ROUTES_CHANGED_TYPE);
212
256
  }
213
257
 
214
258
  /**
@@ -240,6 +284,19 @@ export interface RouteReconcileWaitResult {
240
284
  * failure, not a benign "0 deliveries" success.
241
285
  */
242
286
  noDispatcher: boolean;
287
+ /**
288
+ * Deliveries still `pending`/`running` when the deadline hit. Non-zero means
289
+ * the provider had NOT given up — its retries were still in flight — which is
290
+ * a different thing from a failure and must not be reported as one
291
+ * (celilo#1416: "0 ok, 0 failed of 1" read as "nobody consumed it").
292
+ */
293
+ stillRunning: number;
294
+ /**
295
+ * The deadline actually used, in ms. Derived from the subscribers' own retry
296
+ * budgets unless the caller passed `timeoutMs`, so the message can say what
297
+ * it waited for rather than quoting a constant.
298
+ */
299
+ budgetMs: number;
243
300
  }
244
301
 
245
302
  /**
@@ -247,11 +304,11 @@ export interface RouteReconcileWaitResult {
247
304
  * exist yet. A deploy captures this before it runs so {@link waitForRouteReconcile}
248
305
  * can tell which route-change events the deploy itself produced.
249
306
  */
250
- export function routesChangedHighWater(): number {
307
+ export function routesChangedHighWater(type: RoutesChangedType = ROUTES_CHANGED_TYPE): number {
251
308
  let bus: ReturnType<typeof openBus> | undefined;
252
309
  try {
253
310
  bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
254
- return bus.recentEvents({ type: ROUTES_CHANGED_TYPE, limit: 1 })[0]?.id ?? 0;
311
+ return bus.recentEvents({ type, limit: 1 })[0]?.id ?? 0;
255
312
  } catch (err) {
256
313
  const msg = err instanceof Error ? err.message : String(err);
257
314
  console.warn(`[celilo] failed to read routes-changed high-water: ${msg}`);
@@ -261,6 +318,27 @@ export function routesChangedHighWater(): number {
261
318
  }
262
319
  }
263
320
 
321
+ /**
322
+ * A subscriber's whole retry budget: every attempt's handler timeout plus the
323
+ * dispatcher's exponential backoff between them (1s, 2s, 4s..., capped at 5m —
324
+ * `backoffMs` in the event bus's dispatcher).
325
+ *
326
+ * Exported so the manifest gate that keeps consumer hook timeouts above this
327
+ * can compute the same number from the same formula rather than a second copy
328
+ * of it (celilo#1416).
329
+ */
330
+ export function subscriberBudgetMs(maxAttempts: number, timeoutMs: number): number {
331
+ const attempts = Math.max(1, maxAttempts);
332
+ let budget = attempts * timeoutMs;
333
+ for (let attempt = 1; attempt < attempts; attempt++) {
334
+ budget += Math.min(1000 * 2 ** (attempt - 1), 5 * 60 * 1000);
335
+ }
336
+ return budget;
337
+ }
338
+
339
+ /** Slack over the derived budget, so the wait outlives the last attempt. */
340
+ const RECONCILE_BUDGET_SLACK_MS = 5_000;
341
+
264
342
  function sleep(ms: number): Promise<void> {
265
343
  return new Promise((resolve) => setTimeout(resolve, ms));
266
344
  }
@@ -284,10 +362,15 @@ function sleep(ms: number): Promise<void> {
284
362
  export async function waitForRouteReconcile(
285
363
  sinceEventId: number,
286
364
  opts: { timeoutMs?: number; pollMs?: number } = {},
365
+ type: RoutesChangedType = ROUTES_CHANGED_TYPE,
287
366
  ): Promise<RouteReconcileWaitResult> {
288
- const timeoutMs = opts.timeoutMs ?? 90_000;
289
367
  const pollMs = opts.pollMs ?? 250;
290
- const deadline = Date.now() + timeoutMs;
368
+ // The deadline is DERIVED below, once we know which subscribers are on the
369
+ // hook and what retry budget each was registered with. A constant here is
370
+ // what celilo#1416 was: 90s, exactly ONE attempt of a subscriber configured
371
+ // for three, so a successful retry could never be observed.
372
+ let budgetMs = opts.timeoutMs ?? 0;
373
+ let deadline = opts.timeoutMs !== undefined ? Date.now() + opts.timeoutMs : 0;
291
374
 
292
375
  let bus: ReturnType<typeof openBus> | undefined;
293
376
  try {
@@ -296,9 +379,7 @@ export async function waitForRouteReconcile(
296
379
 
297
380
  // The deploy's emits already happened, so the set of route-change events
298
381
  // is fixed; only their deliveries transition as the dispatcher works.
299
- const events = b
300
- .recentEvents({ type: ROUTES_CHANGED_TYPE, limit: 100 })
301
- .filter((e) => e.id > sinceEventId);
382
+ const events = b.recentEvents({ type, limit: 100 }).filter((e) => e.id > sinceEventId);
302
383
 
303
384
  if (events.length === 0) {
304
385
  return {
@@ -308,6 +389,8 @@ export async function waitForRouteReconcile(
308
389
  timedOut: false,
309
390
  lastError: null,
310
391
  noDispatcher: false,
392
+ stillRunning: 0,
393
+ budgetMs,
311
394
  };
312
395
  }
313
396
 
@@ -332,6 +415,8 @@ export async function waitForRouteReconcile(
332
415
  timedOut: false,
333
416
  lastError: null,
334
417
  noDispatcher: true,
418
+ stillRunning: 0,
419
+ budgetMs,
335
420
  };
336
421
  }
337
422
 
@@ -342,6 +427,20 @@ export async function waitForRouteReconcile(
342
427
  (d) => d.status === 'pending' || d.status === 'running',
343
428
  ).length;
344
429
 
430
+ if (deadline === 0) {
431
+ // Every subscriber gets its whole registered retry budget, and a deploy
432
+ // that emitted N change events is waiting on N of them — budgeting for
433
+ // one is how a two-event deploy charged both against a single 90s wall.
434
+ const subscriberIds = [...new Set(deliveries.map((d) => d.subscriberId))];
435
+ const worst = subscriberIds.reduce((max, id) => {
436
+ const sub = b.getSubscriber(id);
437
+ if (!sub) return max;
438
+ return Math.max(max, subscriberBudgetMs(sub.maxAttempts, sub.timeoutMs));
439
+ }, 0);
440
+ budgetMs = worst * events.length + RECONCILE_BUDGET_SLACK_MS;
441
+ deadline = Date.now() + budgetMs;
442
+ }
443
+
345
444
  if (pending === 0 || Date.now() >= deadline) {
346
445
  return {
347
446
  events: events.length,
@@ -350,6 +449,8 @@ export async function waitForRouteReconcile(
350
449
  timedOut: pending > 0,
351
450
  lastError: deliveries.find((d) => d.lastError)?.lastError ?? null,
352
451
  noDispatcher: false,
452
+ stillRunning: pending,
453
+ budgetMs,
353
454
  };
354
455
  }
355
456
 
@@ -365,6 +466,8 @@ export async function waitForRouteReconcile(
365
466
  timedOut: false,
366
467
  lastError: null,
367
468
  noDispatcher: false,
469
+ stillRunning: 0,
470
+ budgetMs,
368
471
  };
369
472
  } finally {
370
473
  bus?.close();
@@ -5,6 +5,7 @@ import { join } from 'node:path';
5
5
  import { skipIntegration } from '../test-utils/integration-guard';
6
6
 
7
7
  import type { HookContext } from '@celilo/capabilities';
8
+ import { configStore, secretStore } from '../hooks/test-fixtures/store-backed';
8
9
 
9
10
  // Hooks under test — imported via the scripts dir adjacent to the
10
11
  // modules/celilo-mgmt/manifest.yml. The dynamic import keeps the test
@@ -46,7 +47,7 @@ function buildContext(extras: Record<string, unknown>): HookContext {
46
47
  const { logger } = buildLogger();
47
48
  return {
48
49
  logger,
49
- config: {
50
+ config: configStore({
50
51
  db_path: '', // populated by caller via env
51
52
  hostname: 'test-mgmt',
52
53
  target_ip: '127.0.0.1',
@@ -61,9 +62,11 @@ function buildContext(extras: Record<string, unknown>): HookContext {
61
62
  ssh_public_key: '',
62
63
  install_docker: false,
63
64
  install_terraform: false,
64
- } as Record<string, unknown>,
65
- secrets: {},
65
+ }),
66
+ secrets: secretStore(),
66
67
  capabilities: {},
68
+ systems: [],
69
+ stateDir: '',
67
70
  debug: false,
68
71
  screenshotDir: '',
69
72
  ...extras,
@@ -25,7 +25,7 @@ import {
25
25
 
26
26
  function buildManifest(caps: Array<{ name: string; version: string }>): ModuleManifest {
27
27
  return {
28
- celilo_contract: '1.0',
28
+ celilo_contract: '2.0',
29
29
  id: 'm',
30
30
  name: 'm',
31
31
  version: '0.0.1',
@@ -88,7 +88,7 @@ describe('materializeCrossModuleRoot', () => {
88
88
  name: id,
89
89
  version,
90
90
  sourcePath: join(dir, 'sources', id),
91
- manifestData: { id, name: id, version, celilo_contract: '1.0', requires: {} } as Record<
91
+ manifestData: { id, name: id, version, celilo_contract: '2.0', requires: {} } as Record<
92
92
  string,
93
93
  unknown
94
94
  >,
@@ -31,7 +31,7 @@ import {
31
31
 
32
32
  function manifestWithStringMapSecret(): ModuleManifest {
33
33
  return {
34
- celilo_contract: '1.0',
34
+ celilo_contract: '2.0',
35
35
  id: 'testmod',
36
36
  name: 'Test Module',
37
37
  version: '1.0.0',
@@ -54,7 +54,7 @@ function manifestWithStringMapSecret(): ModuleManifest {
54
54
 
55
55
  function manifestWithPlainStringSecret(): ModuleManifest {
56
56
  return {
57
- celilo_contract: '1.0',
57
+ celilo_contract: '2.0',
58
58
  id: 'testmod',
59
59
  name: 'Test Module',
60
60
  version: '1.0.0',
@@ -140,7 +140,7 @@ describe('findMissingRequiredVariables (deploy path)', () => {
140
140
  // for the interview, and not as a deploy-blocking error.
141
141
  test('a source: hook variable is never reported missing, even required and unwritten', async () => {
142
142
  const manifest = {
143
- celilo_contract: '1.0',
143
+ celilo_contract: '2.0',
144
144
  variables: {
145
145
  owns: [
146
146
  {
@@ -341,7 +341,7 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
341
341
 
342
342
  function manifestWithArtifacts(artifacts: string[]): ModuleManifest {
343
343
  return {
344
- celilo_contract: '1.0',
344
+ celilo_contract: '2.0',
345
345
  id: 'buildmod',
346
346
  name: 'Build Module',
347
347
  version: '1.0.0',
@@ -351,7 +351,7 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
351
351
  }
352
352
 
353
353
  test('returns [] when the module declares no build section', () => {
354
- const manifest = { celilo_contract: '1.0', id: 'm', version: '1.0.0' } as ModuleManifest;
354
+ const manifest = { celilo_contract: '2.0', id: 'm', version: '1.0.0' } as ModuleManifest;
355
355
  expect(findMissingBuildArtifacts(manifest, dir)).toEqual([]);
356
356
  });
357
357
 
@@ -391,7 +391,7 @@ describe('validateAndPrepareDeployment (optional capability version gate)', () =
391
391
  sourcePath: tempDir,
392
392
  version: '1.0.0',
393
393
  manifestData: {
394
- celilo_contract: '1.0',
394
+ celilo_contract: '2.0',
395
395
  id: 'consumer',
396
396
  name: 'Consumer',
397
397
  version: '1.0.0',
@@ -42,7 +42,7 @@ describe('backfillWebRouteDns (ISS-0029)', () => {
42
42
  id,
43
43
  name: id,
44
44
  version: '1.0.0',
45
- manifestData: { id, name: id, version: '1.0.0', celilo_contract: '1.0' },
45
+ manifestData: { id, name: id, version: '1.0.0', celilo_contract: '2.0' },
46
46
  sourcePath: `/tmp/${id}`,
47
47
  })
48
48
  .run();
@@ -73,7 +73,7 @@ describe('backfillWebRouteDns (ISS-0029)', () => {
73
73
  id: 'caddy',
74
74
  name: 'caddy',
75
75
  version: '1.0.0',
76
- manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '1.0' },
76
+ manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '2.0' },
77
77
  sourcePath: '/tmp/caddy',
78
78
  })
79
79
  .onConflictDoNothing()