@celilo/cli 4.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/AGENTS.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +1 -0
  3. package/drizzle/0034_private_routes.sql +41 -0
  4. package/drizzle/meta/_journal.json +8 -1
  5. package/package.json +5 -4
  6. package/src/api/sessions.test.ts +1 -1
  7. package/src/capabilities/registration.test.ts +11 -11
  8. package/src/capabilities/secrets.test.ts +18 -18
  9. package/src/capabilities/validation.test.ts +24 -24
  10. package/src/cli/cli.test.ts +3 -3
  11. package/src/cli/command-tree-parser.test.ts +18 -0
  12. package/src/cli/command-tree-parser.ts +12 -0
  13. package/src/cli/commands/api.ts +67 -14
  14. package/src/cli/commands/events.ts +5 -2
  15. package/src/cli/commands/machine-reclassify.ts +79 -0
  16. package/src/cli/commands/module-changeset.test.ts +1 -1
  17. package/src/cli/commands/module-generate.test.ts +2 -2
  18. package/src/cli/commands/module-import-aspect.test.ts +2 -2
  19. package/src/cli/commands/module-update.test.ts +14 -14
  20. package/src/cli/commands/module-upgrade-gate.test.ts +26 -1
  21. package/src/cli/commands/module-upgrade.test.ts +3 -3
  22. package/src/cli/commands/module-upgrade.ts +12 -0
  23. package/src/cli/commands/module-version.test.ts +1 -1
  24. package/src/cli/commands/system-audit.ts +22 -0
  25. package/src/cli/commands/system-reboot.ts +126 -0
  26. package/src/cli/commands/system-update.ts +5 -0
  27. package/src/cli/completion.ts +3 -2
  28. package/src/cli/index.ts +41 -2
  29. package/src/cli/restore-migration-failure.test.ts +1 -1
  30. package/src/cli/stdout-pipe-flush.test.ts +35 -0
  31. package/src/cli/tui/audit-state.ts +4 -0
  32. package/src/db/private-routes.test.ts +159 -0
  33. package/src/db/schema.ts +49 -0
  34. package/src/hooks/broker.ts +31 -0
  35. package/src/hooks/capability-loader.ts +185 -12
  36. package/src/hooks/executor.test.ts +13 -13
  37. package/src/hooks/executor.ts +20 -2
  38. package/src/hooks/hook-state-dir.test.ts +2 -2
  39. package/src/hooks/test-fixtures/slow-capability-hook.ts +28 -0
  40. package/src/hooks/test-fixtures/store-backed.ts +5 -3
  41. package/src/hooks/timeout-names-provider.test.ts +87 -0
  42. package/src/manifest/contracts/index.ts +28 -7
  43. package/src/manifest/contracts/v2.ts +32 -0
  44. package/src/manifest/icon-schema.test.ts +1 -1
  45. package/src/manifest/schema.ts +13 -2
  46. package/src/manifest/template-validator.test.ts +1 -1
  47. package/src/manifest/validate-privileged.test.ts +1 -1
  48. package/src/manifest/validate.test.ts +68 -29
  49. package/src/manifest/validate.ts +14 -1
  50. package/src/module/import.test.ts +7 -7
  51. package/src/module/packaging/build.test.ts +1 -1
  52. package/src/packaging/nfpm-depends.test.ts +42 -0
  53. package/src/policy/capability-shape-baseline.ts +7 -1
  54. package/src/policy/fixture-capability-coverage.test.ts +5 -5
  55. package/src/policy/module-business-baseline.ts +33 -4
  56. package/src/services/api-access.test.ts +44 -0
  57. package/src/services/aspect-approvals.test.ts +2 -2
  58. package/src/services/aspect-reconcile.test.ts +2 -2
  59. package/src/services/aspect-runner.test.ts +4 -4
  60. package/src/services/audit/backups.test.ts +1 -1
  61. package/src/services/audit/capability-abi.test.ts +1 -1
  62. package/src/services/audit/index.test.ts +7 -0
  63. package/src/services/audit/index.ts +15 -0
  64. package/src/services/audit/machine-interface-zones.test.ts +30 -0
  65. package/src/services/audit/machine-interface-zones.ts +43 -0
  66. package/src/services/audit/module-configs.test.ts +1 -1
  67. package/src/services/audit/reboot-pending.test.ts +63 -0
  68. package/src/services/audit/reboot-pending.ts +85 -0
  69. package/src/services/audit/recurrence-gate.test.ts +21 -0
  70. package/src/services/audit/server-bun-pin.test.ts +44 -0
  71. package/src/services/audit/server-bun-pin.ts +95 -0
  72. package/src/services/audit/types.ts +2 -0
  73. package/src/services/build-bus/hook-dispatch-executor.test.ts +3 -3
  74. package/src/services/bus-interview-park.test.ts +1 -1
  75. package/src/services/bus-secret-flow.test.ts +1 -1
  76. package/src/services/capability-compat.test.ts +22 -0
  77. package/src/services/capability-compat.ts +15 -2
  78. package/src/services/capability-table-rows.test.ts +20 -2
  79. package/src/services/celilo-events.test.ts +119 -6
  80. package/src/services/celilo-events.ts +156 -16
  81. package/src/services/celilo-mgmt-hooks.test.ts +6 -3
  82. package/src/services/cross-module-read.test.ts +2 -2
  83. package/src/services/deploy-preflight.ts +12 -3
  84. package/src/services/deploy-validation.test.ts +86 -7
  85. package/src/services/deploy-validation.ts +19 -0
  86. package/src/services/dns-provider-backfill.test.ts +2 -2
  87. package/src/services/dns-registrations.test.ts +1 -1
  88. package/src/services/fleet-checks.test.ts +2 -2
  89. package/src/services/fleet-checks.ts +40 -10
  90. package/src/services/health-runner.test.ts +1 -1
  91. package/src/services/infrastructure-variable-resolver.test.ts +11 -11
  92. package/src/services/instance-ops.test.ts +2 -2
  93. package/src/services/machine-interface-zones.test.ts +73 -0
  94. package/src/services/machine-interface-zones.ts +101 -0
  95. package/src/services/module-deploy.dns-repoint.test.ts +1 -1
  96. package/src/services/module-deploy.stale-provider-guard.test.ts +98 -0
  97. package/src/services/module-deploy.ts +31 -10
  98. package/src/services/module-pause-quiescence.test.ts +1 -1
  99. package/src/services/module-pause.test.ts +1 -1
  100. package/src/services/module-subscriptions.test.ts +2 -2
  101. package/src/services/module-types-generator.test.ts +1 -1
  102. package/src/services/module-validator/contract-version.test.ts +3 -3
  103. package/src/services/network-ensure.test.ts +1 -1
  104. package/src/services/provider-arrival.test.ts +62 -46
  105. package/src/services/provider-arrival.ts +41 -10
  106. package/src/services/proxmox-reconcile.test.ts +2 -2
  107. package/src/services/remove-guard.test.ts +1 -1
  108. package/src/services/system-reboot.test.ts +231 -0
  109. package/src/services/system-reboot.ts +357 -0
  110. package/src/services/update/dep-graph.test.ts +1 -1
  111. package/src/services/update/orchestrator.test.ts +165 -1
  112. package/src/services/update/orchestrator.ts +43 -12
  113. package/src/services/update/types.ts +10 -1
  114. package/src/templates/generator.test.ts +15 -4
  115. package/src/templates/generator.ts +12 -4
  116. package/src/variables/capability-self-ref.test.ts +2 -2
  117. package/src/variables/computed/computed-integration.test.ts +1 -1
  118. package/src/variables/context.ts +14 -1
  119. package/src/variables/declarative-derivation.test.ts +19 -19
@@ -36,6 +36,7 @@ import {
36
36
  NETWORK_ZONES,
37
37
  capabilities,
38
38
  modules,
39
+ privateRoutes,
39
40
  secrets,
40
41
  systemConfig,
41
42
  webRoutes,
@@ -45,7 +46,10 @@ import { decryptSecret } from '../secrets/encryption';
45
46
  import { getOrCreateMasterKey } from '../secrets/master-key';
46
47
  import { buildControlPlaneApi } from '../services/api-principal-enrolment';
47
48
  import { recordCapabilityBinding, withBindingRecord } from '../services/capability-bindings';
48
- import { emitWebRoutesChangedAndWait } from '../services/celilo-events';
49
+ import {
50
+ emitPrivateRoutesChangedAndWait,
51
+ emitWebRoutesChangedAndWait,
52
+ } from '../services/celilo-events';
49
53
  import { CONTROL_PLANE_MODULE_ID, getModuleSystems } from '../services/deployed-systems';
50
54
  import { withDnsInternalLedger } from '../services/dns-internal-records';
51
55
  import { withDnsRegistrationLedger } from '../services/dns-registrations';
@@ -95,22 +99,56 @@ import { loadHookConfigMap } from './load-hook-config';
95
99
  * success anyway, so a consumer could report "ready" while caddy never learned
96
100
  * the hostname — no site block, no cert, TLS internal_error for clients.
97
101
  */
98
- async function requireProviderReconcile(consumingModuleId: string, what: string): Promise<void> {
99
- const reconcile = await emitWebRoutesChangedAndWait(consumingModuleId);
102
+ /**
103
+ * Which ingress a reconcile is being required for. The two differ only in the
104
+ * event type they emit and the names their failures carry, so they share this
105
+ * function rather than each keeping a copy — a private ingress that silently
106
+ * stopped waiting would look exactly like one with nothing to wait for
107
+ * (providers-converge-declared-state, slice 2).
108
+ */
109
+ const INGRESS = {
110
+ public_web: { capability: 'public_web', provider: 'caddy', emit: emitWebRoutesChangedAndWait },
111
+ private_web: {
112
+ capability: 'private_web',
113
+ provider: 'caddy-internal',
114
+ emit: emitPrivateRoutesChangedAndWait,
115
+ },
116
+ } as const;
117
+
118
+ async function requireProviderReconcile(
119
+ consumingModuleId: string,
120
+ what: string,
121
+ ingress: keyof typeof INGRESS = 'public_web',
122
+ ): Promise<void> {
123
+ const { capability, provider, emit } = INGRESS[ingress];
124
+ const reconcile = await emit(consumingModuleId);
100
125
 
101
126
  if (reconcile.noDispatcher) {
102
127
  throw new Error(
103
- `public_web ${what} for ${consumingModuleId} was persisted but NOT delivered to the provider (caddy): no event dispatcher is running, so caddy never reconciled and the hostname has no site block or cert. Run this through \`celilo module deploy\` (which runs the dispatcher) rather than a bare hook.`,
128
+ `${capability} ${what} for ${consumingModuleId} was persisted but NOT delivered to the provider (${provider}): no event dispatcher is running, so ${provider} never reconciled and the hostname has no site block or cert. Run this through \`celilo module deploy\` (which runs the dispatcher) rather than a bare hook.`,
104
129
  );
105
130
  }
131
+ // The handler's own words, when it left any. A crash is retried, so its
132
+ // delivery sits in `pending` with the error already recorded while the
133
+ // deadline runs out — which is how celilo#1401 reported a TIMEOUT for a
134
+ // caddy crash, for days, with `0 ok, 0 failed of 1` and the cause nowhere.
135
+ const because = reconcile.lastError ? ` The provider reported: ${reconcile.lastError}` : '';
136
+
106
137
  if (reconcile.timedOut) {
138
+ // "0 ok, 0 failed" reads as "nobody consumed it", which sends the reader to
139
+ // the dispatcher (celilo#1398/#1399) when the truth is that the provider's
140
+ // deliveries were STILL RUNNING at the deadline (celilo#1416). Say which.
141
+ const inFlight =
142
+ reconcile.stillRunning > 0
143
+ ? `${reconcile.stillRunning} delivery(ies) still running`
144
+ : `${reconcile.succeeded} ok, ${reconcile.failed} failed`;
107
145
  throw new Error(
108
- `public_web reconcile for ${consumingModuleId} (${what}) did not finish within the deadline (${reconcile.succeeded} ok, ${reconcile.failed} failed of ${reconcile.events} change event(s)) — caddy did not confirm the change is live.`,
146
+ `${capability} reconcile for ${consumingModuleId} (${what}) did not finish within its ${Math.round(reconcile.budgetMs / 1000)}s budget (${inFlight} of ${reconcile.events} change event(s)) — ${provider} did not confirm the change is live.${because}`,
109
147
  );
110
148
  }
111
149
  if (reconcile.failed > 0) {
112
150
  throw new Error(
113
- `public_web reconcile for ${consumingModuleId} (${what}): ${reconcile.failed} delivery(ies) failed — caddy could not apply the change, so the hostname is not served as declared.`,
151
+ `${capability} reconcile for ${consumingModuleId} (${what}): ${reconcile.failed} delivery(ies) failed — ${provider} could not apply the change, so the hostname is not served as declared.${because}`,
114
152
  );
115
153
  }
116
154
  // ISS-0087: a change WAS persisted and the dispatcher IS alive, yet ZERO
@@ -120,7 +158,7 @@ async function requireProviderReconcile(consumingModuleId: string, what: string)
120
158
  // changed and nobody served it — a failure.
121
159
  if (reconcile.events > 0 && reconcile.succeeded === 0) {
122
160
  throw new Error(
123
- `public_web ${what} for ${consumingModuleId} changed (${reconcile.events} event(s)) but NO provider reconciled it — caddy has no reconcile_routes subscription on the bus, so the change is persisted yet never served. Ensure a public_web provider (caddy) is deployed and subscribed.`,
161
+ `${capability} ${what} for ${consumingModuleId} changed (${reconcile.events} event(s)) but NO provider reconciled it — ${provider} has no reconcile_routes subscription on the bus, so the change is persisted yet never served. Ensure a ${capability} provider (${provider}) is deployed and subscribed.`,
124
162
  );
125
163
  }
126
164
  }
@@ -169,11 +207,13 @@ export const CAPABILITY_MODULE_MAP: Record<string, { script: string; legacyFacto
169
207
  legacyFactoryName: 'default',
170
208
  },
171
209
  // MODULE-provided, unlike its sibling public_web above, which the framework
172
- // implements (createPublicWeb) against celilo's `web_routes` table. The
173
- // asymmetry is deliberate: a private route stored in `web_routes` would be
174
- // picked up by the PUBLIC caddy, which derives its served hostnames from
175
- // every row of that table — so the provider keeps its own routes and celilo
176
- // core holds no private-ingress business at all (celilo#846).
210
+ // implements (createPublicWeb). The two ingresses are separate TABLES, not
211
+ // separate owners: core owns `private_routes` as it owns `web_routes`
212
+ // (migration 0034), and that separation — not the provider keeping its own
213
+ // config — is what stops the PUBLIC caddy, which derives its served
214
+ // hostnames from every row of `web_routes`, from sweeping a private route
215
+ // into the public ingress (celilo#846). What stays module-side is the
216
+ // RENDER: caddy-internal decides what its Caddyfile says.
177
217
  private_web: {
178
218
  script: 'scripts/private-web-functions.ts',
179
219
  legacyFactoryName: 'default',
@@ -493,6 +533,26 @@ export async function loadCapabilityFunctions(
493
533
  // same seam. A jailed hook cannot write into its own web root, so a
494
534
  // module-provided web provider needs the state overlay to honor it.
495
535
  consumerStateWebRoot: resolveModuleStateWebRoot(consumingModuleId, db),
536
+ // THE PRIVATE INGRESS'S DECLARED ROWS, for the provider that serves
537
+ // them. Scoped to private_web deliberately: these are one
538
+ // capability's declared state, and handing them to every
539
+ // module-provided capability would put the private route table in
540
+ // reach of modules that have no business reading it.
541
+ ...(capName === 'private_web'
542
+ ? {
543
+ privateRoutes: buildPrivateRouteOps(db),
544
+ // The same seam public_web has (see `onRoutesChanged` below):
545
+ // the capability persists the row and hands the converge to
546
+ // the provider's `reconcile_routes` subscription, then AWAITS
547
+ // that delivery so `registerReverseProxy` still returns only
548
+ // once the route is live (design D7). One primitive, two
549
+ // triggers — the subscription is the second, not a
550
+ // replacement for the call.
551
+ onRoutesChanged: async () => {
552
+ await requireProviderReconcile(consumingModuleId, 'route change', 'private_web');
553
+ },
554
+ }
555
+ : {}),
496
556
  });
497
557
  // Stamp here too, not only on the legacy path: a consumer that cannot
498
558
  // get what it needs must be able to name WHICH provider could not give
@@ -811,6 +871,51 @@ export async function loadCapabilityFunctions(
811
871
  debugLog('public_web: not registered in DB, skipping');
812
872
  }
813
873
 
874
+ // THE PRIVATE INGRESS, symmetric with the public one above (design D7 of
875
+ // providers-converge-declared-state). Its provider gets a view of the rows it
876
+ // serves and the converge that places what it renders — the same two things
877
+ // caddy gets, for the same reason, now that its routes are core-owned state
878
+ // rather than its own module-config JSON.
879
+ //
880
+ // Consumers never see either: only the module that PROVIDES private_web.
881
+ const privateWebProviders = db
882
+ .select()
883
+ .from(capabilities)
884
+ .where(eq(capabilities.capabilityName, 'private_web'))
885
+ .all();
886
+ const privateWebProvider = privateWebProviders[0];
887
+ if (privateWebProvider && consumingModuleId === privateWebProvider.moduleId) {
888
+ result.private_routes = buildPrivateRouteOps(db);
889
+ debugLog(`private_routes: injected for provider ${consumingModuleId}`);
890
+
891
+ const providerModuleId = privateWebProvider.moduleId;
892
+ const privateConverge: ProviderConvergeView = {
893
+ // The private ingress serves no consumer-built static sites today: every
894
+ // row it renders is a reverse proxy. Answering honestly rather than
895
+ // throwing keeps the shape identical to the public provider's, so one
896
+ // renderer contract covers both (design D1).
897
+ resolveConsumerSite: async (moduleId: string) => ({
898
+ unavailable: `the private ingress serves no static sites, so '${moduleId}' has none here`,
899
+ }),
900
+ converge: async (artifacts) => {
901
+ const { convergeProviderConfig, resolveStaticContentRetention } = await import(
902
+ '../services/provider-converge'
903
+ );
904
+ const result = await convergeProviderConfig(db, providerModuleId, {
905
+ ...artifacts,
906
+ retention: resolveStaticContentRetention(db, providerModuleId),
907
+ });
908
+ return {
909
+ success: result.success,
910
+ ...(result.error ? { error: result.error } : {}),
911
+ unresolved: result.unresolved,
912
+ };
913
+ },
914
+ };
915
+ result.provider_converge = privateConverge;
916
+ debugLog(`provider_converge: injected for private provider ${consumingModuleId}`);
917
+ }
918
+
814
919
  // Framework-granted, so unlike everything above there is no provider row to
815
920
  // look up and no script to import — celilo IS the management server whose
816
921
  // principals these are (web-ui-console D7b).
@@ -1482,6 +1587,74 @@ async function buildFirewallChain(
1482
1587
  return { chain: currentUpstream, self };
1483
1588
  }
1484
1589
 
1590
+ /**
1591
+ * Route ops for the private_web PROVIDER, over `private_routes`.
1592
+ *
1593
+ * The private ingress's rows are core-owned declared state now, exactly as
1594
+ * `web_routes` is for public_web (design D1 of
1595
+ * providers-converge-declared-state). They used to live in caddy-internal's
1596
+ * `routes` module-config JSON, which is why the two ingresses could not share
1597
+ * plumbing: celilo had nothing to render, converge or audit.
1598
+ *
1599
+ * SEPARATE TABLE, and that is the whole safety property. The public provider
1600
+ * derives the hostnames it serves from every row of ITS table, so a private
1601
+ * route sharing that table is one missing WHERE clause from being served
1602
+ * publicly with a public certificate (celilo#846). Injected only into the
1603
+ * private_web provider's own capability functions, the same way the route view
1604
+ * reaches caddy and no one else.
1605
+ *
1606
+ * Rows go out in the snake_case shape the provider's renderer already speaks,
1607
+ * so the module needs no translation layer and its existing route tests keep
1608
+ * asserting the same objects.
1609
+ */
1610
+ function buildPrivateRouteOps(db: DbClient) {
1611
+ const toRecord = (row: typeof privateRoutes.$inferSelect) => ({
1612
+ module_id: row.moduleId,
1613
+ slug: row.slug,
1614
+ type: row.type,
1615
+ path: row.path,
1616
+ hostname: row.hostname,
1617
+ ...(row.targetHost ? { target_host: row.targetHost } : {}),
1618
+ ...(row.targetPort ? { target_port: row.targetPort } : {}),
1619
+ ...(row.websocket ? { websocket: row.websocket } : {}),
1620
+ });
1621
+
1622
+ return {
1623
+ all: () => db.select().from(privateRoutes).all().map(toRecord),
1624
+ /**
1625
+ * Replace the whole set.
1626
+ *
1627
+ * The provider keeps the full route list in memory for the life of one
1628
+ * hook run and hands it back whole, which is the semantics its config key
1629
+ * had. Honouring that here keeps the change to the module small and the
1630
+ * two halves easy to reason about.
1631
+ *
1632
+ * ponytail: whole-set replace, in one transaction so a reader never sees
1633
+ * an empty ingress. If two providers ever write this table concurrently,
1634
+ * this becomes per-row upsert plus a delete of what the writer dropped.
1635
+ */
1636
+ replaceAll: (rows: ReturnType<typeof toRecord>[]) => {
1637
+ db.transaction((tx) => {
1638
+ tx.delete(privateRoutes).run();
1639
+ for (const row of rows) {
1640
+ tx.insert(privateRoutes)
1641
+ .values({
1642
+ slug: row.slug,
1643
+ moduleId: row.module_id,
1644
+ type: row.type,
1645
+ path: row.path,
1646
+ hostname: row.hostname,
1647
+ targetHost: row.target_host ?? null,
1648
+ targetPort: row.target_port ?? null,
1649
+ websocket: row.websocket ?? false,
1650
+ })
1651
+ .run();
1652
+ }
1653
+ });
1654
+ },
1655
+ };
1656
+ }
1657
+
1485
1658
  /**
1486
1659
  * Build RouteOps for the public_web capability implementation.
1487
1660
  * Wraps database operations so the capability never touches the DB directly.
@@ -529,7 +529,7 @@ describe('Hook Executor', () => {
529
529
  const result = await invokeHook(
530
530
  __dirname,
531
531
  'container_created',
532
- '1.0',
532
+ '2.0',
533
533
  { script: './test-fixtures/artifact-writing-hook.ts', timeout: 10000 },
534
534
  { vps_ip: '10.0.0.5' },
535
535
  {},
@@ -560,7 +560,7 @@ describe('Hook Executor', () => {
560
560
  const result = await invokeHook(
561
561
  __dirname,
562
562
  'container_created',
563
- '1.0',
563
+ '2.0',
564
564
  { script: './test-fixtures/success-hook.ts', timeout: 10000 },
565
565
  { vps_ip: '10.0.0.5' },
566
566
  {},
@@ -585,7 +585,7 @@ describe('Hook Executor', () => {
585
585
  const result = await invokeHook(
586
586
  __dirname,
587
587
  'container_created',
588
- '1.0',
588
+ '2.0',
589
589
  { script: './test-fixtures/success-hook.ts', timeout: 10000 },
590
590
  { vps_ip: '10.0.0.5' },
591
591
  {},
@@ -614,7 +614,7 @@ describe('Hook Executor', () => {
614
614
  const result = await invokeHook(
615
615
  __dirname,
616
616
  'container_created',
617
- '1.0',
617
+ '2.0',
618
618
  definition,
619
619
  { vps_ip: '10.0.0.5' },
620
620
  { username: 'test' },
@@ -641,7 +641,7 @@ describe('Hook Executor', () => {
641
641
  const result = await invokeHook(
642
642
  __dirname,
643
643
  'on_backup',
644
- '1.0',
644
+ '2.0',
645
645
  definition,
646
646
  {},
647
647
  {},
@@ -663,7 +663,7 @@ describe('Hook Executor', () => {
663
663
  const result = await invokeHook(
664
664
  __dirname,
665
665
  'container_created',
666
- '1.0',
666
+ '2.0',
667
667
  definition,
668
668
  {},
669
669
  {},
@@ -686,7 +686,7 @@ describe('Hook Executor', () => {
686
686
  const result = await invokeHook(
687
687
  __dirname,
688
688
  'on_backup',
689
- '1.0',
689
+ '2.0',
690
690
  definition,
691
691
  { backup_dir: '/tmp/backup' },
692
692
  {},
@@ -708,7 +708,7 @@ describe('Hook Executor', () => {
708
708
  const result = await invokeHook(
709
709
  __dirname,
710
710
  'container_created',
711
- '1.0',
711
+ '2.0',
712
712
  definition,
713
713
  {},
714
714
  {},
@@ -743,10 +743,10 @@ describe('Hook Executor', () => {
743
743
  const { logger } = createCapturingLogger();
744
744
  const definition: HookDefinition = { script: './test-fixtures/success-hook.ts' };
745
745
 
746
- const result = await invokeHook(__dirname, 'on_poop', '1.0', definition, {}, {}, {}, logger);
746
+ const result = await invokeHook(__dirname, 'on_poop', '2.0', definition, {}, {}, {}, logger);
747
747
 
748
748
  expect(result.success).toBe(false);
749
- expect(result.error).toContain("Hook 'on_poop' is not part of celilo_contract 1.0");
749
+ expect(result.error).toContain("Hook 'on_poop' is not part of celilo_contract 2.0");
750
750
  });
751
751
 
752
752
  test('pre-flight: fails when a required capability is missing', async () => {
@@ -759,7 +759,7 @@ describe('Hook Executor', () => {
759
759
  const result = await invokeHook(
760
760
  __dirname,
761
761
  'container_created',
762
- '1.0',
762
+ '2.0',
763
763
  definition,
764
764
  { vps_ip: '10.0.0.5' },
765
765
  {},
@@ -786,7 +786,7 @@ describe('Hook Executor', () => {
786
786
  const result = await invokeHook(
787
787
  __dirname,
788
788
  'container_created',
789
- '1.0',
789
+ '2.0',
790
790
  definition,
791
791
  { vps_ip: '10.0.0.5' },
792
792
  {},
@@ -813,7 +813,7 @@ describe('Hook Executor', () => {
813
813
  const result = await invokeHook(
814
814
  __dirname,
815
815
  'container_created',
816
- '1.0',
816
+ '2.0',
817
817
  definition,
818
818
  { vps_ip: '10.0.0.5' },
819
819
  {},
@@ -511,7 +511,14 @@ export async function executeHookScript(
511
511
  setTimeout(() => child.kill('SIGKILL'), SIGKILL_GRACE_MS).unref();
512
512
  };
513
513
 
514
- const totalTimer = setTimeout(() => kill('total'), timeoutMs);
514
+ // Snapshotted inside the timer rather than read at throw time: the child's
515
+ // death rejects its in-flight calls, so by the time we build the message
516
+ // the map that knows the answer has drained itself.
517
+ let pendingCallsAtKill: string[] = [];
518
+ const totalTimer = setTimeout(() => {
519
+ pendingCallsAtKill = broker.pendingCalls();
520
+ kill('total');
521
+ }, timeoutMs);
515
522
  // Debug runs get no idle timer, same as before: an operator stepping
516
523
  // through a browser hook is silent for minutes on purpose.
517
524
  const idleTimer = context.debug
@@ -536,7 +543,18 @@ export async function executeHookScript(
536
543
  // nothing keeps running past this line. That is the difference from the
537
544
  // `Promise.race` this replaces (celilo#1003).
538
545
  if (killedFor === 'total') {
539
- throw new Error(`Hook total timeout exceeded (${Math.round(timeoutMs / 1000)}s)`);
546
+ // Name what it was WAITING ON, not just that it waited. A hook whose
547
+ // budget is consumed by one slow provider used to die as a bare
548
+ // "Hook total timeout exceeded", so the operator had to infer the
549
+ // provider from a capability push line with no matching completion
550
+ // (celilo#1403: bna-yard-sale's 120s on_install blocked on caddy's 75s
551
+ // route reconcile). Captured before `close()`, while the calls are
552
+ // still in flight — `stop()` refuses new calls but clears nothing.
553
+ const waitingOn = pendingCallsAtKill;
554
+ const waitingSuffix = waitingOn.length > 0 ? `, waiting on ${waitingOn.join(', ')}` : '';
555
+ throw new Error(
556
+ `Hook total timeout exceeded (${Math.round(timeoutMs / 1000)}s)${waitingSuffix}`,
557
+ );
540
558
  }
541
559
  if (killedFor === 'idle') {
542
560
  throw new Error(
@@ -62,7 +62,7 @@ describe('celilo#1000: a hook is TOLD where its state directory is', () => {
62
62
  const result = await invokeHook(
63
63
  root,
64
64
  'on_install',
65
- '1.0',
65
+ '2.0',
66
66
  { script: 'hook.ts' },
67
67
  {},
68
68
  {},
@@ -97,7 +97,7 @@ describe('celilo#1000: a hook is TOLD where its state directory is', () => {
97
97
  const result = await invokeHook(
98
98
  root,
99
99
  'on_install',
100
- '1.0',
100
+ '2.0',
101
101
  { script: 'hook.ts' },
102
102
  {},
103
103
  {},
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Test fixture: a hook that blocks inside a capability call it never gets an
3
+ * answer to, and is killed at its own total timeout (celilo#1403).
4
+ *
5
+ * The real shape is a consumer's `on_install` calling
6
+ * `public_web.publishStaticSite`, which blocks on the provider's route
7
+ * reconcile. This fixture reproduces the boundary condition — a call in flight
8
+ * when the kill lands — without a provider.
9
+ *
10
+ * It logs BEFORE the call so the run is not mistaken for an idle timeout, then
11
+ * awaits forever.
12
+ */
13
+
14
+ import { defineHook } from '@celilo/capabilities';
15
+
16
+ interface SlowCapability {
17
+ neverReturns(request: Record<string, unknown>): Promise<unknown>;
18
+ }
19
+
20
+ export default defineHook({
21
+ hook: 'container_created',
22
+ requires: [],
23
+ handler: async (ctx) => {
24
+ const slow = (ctx.capabilities as unknown as Record<string, SlowCapability>).slow_provider;
25
+ ctx.logger.info('calling the provider');
26
+ await slow.neverReturns({});
27
+ },
28
+ });
@@ -1,4 +1,4 @@
1
- import type { HookStore, HookStoreBackedMap } from '@celilo/capabilities';
1
+ import type { HookStore } from '@celilo/capabilities';
2
2
 
3
3
  /**
4
4
  * In-memory hook-owned-state stores for tests that build a HookContext
@@ -16,10 +16,12 @@ export function configStore(
16
16
  return attach(values);
17
17
  }
18
18
 
19
- export function secretStore(values: Record<string, string> = {}): HookStoreBackedMap {
19
+ export function secretStore(
20
+ values: Record<string, string> = {},
21
+ ): Record<string, string> & HookStore {
20
22
  // Downcast: the caller passed Record<string, string>; attach only adds
21
23
  // methods and never writes, so the value type is unchanged.
22
- return attach(values) as HookStoreBackedMap;
24
+ return attach(values) as Record<string, string> & HookStore;
23
25
  }
24
26
 
25
27
  function attach(values: Record<string, unknown>): Record<string, unknown> & HookStore {
@@ -0,0 +1,87 @@
1
+ /**
2
+ * A hook killed at its TOTAL timeout must name what it was waiting on.
3
+ *
4
+ * Measured 2026-09-21 (celilo#1403): bna-yard-sale's `on_install` called
5
+ * `public_web.publishStaticSite`, which blocks on caddy's route reconcile —
6
+ * 70-75s across the fleet's sites. The consumer died at its 120s budget with:
7
+ *
8
+ * [bna-yard-sale:on_install] -> public_web.publishStaticSite
9
+ * [bna-yard-sale:on_install] OK dns_registrar.registerHost
10
+ * [bna-yard-sale:on_install] OK dns_internal.registerRecord
11
+ * [bna-yard-sale:on_install] FAIL Hook total timeout exceeded (120s)
12
+ *
13
+ * Both DNS calls completed; the push line had no matching completion, and that
14
+ * ABSENCE was the only evidence of which provider held it. The error named the
15
+ * hook and never the cause, so the operator had to infer it from a gap in the
16
+ * log — the same class of missing signal the greenwave/axon ISP-router hooks
17
+ * already carry a comment about.
18
+ */
19
+ import { describe, expect, test } from 'bun:test';
20
+ import { mkdtempSync, rmSync } from 'node:fs';
21
+ import { tmpdir } from 'node:os';
22
+ import { join } from 'node:path';
23
+ import { executeHookScript } from './executor';
24
+ import { createCapturingLogger } from './logger';
25
+ import { configStore, secretStore } from './test-fixtures/store-backed';
26
+ import type { HookContext } from './types';
27
+
28
+ const FIXTURES = join(__dirname, 'test-fixtures');
29
+
30
+ /** A capability whose one method never settles — the provider that holds the hook. */
31
+ function hangingCapabilities(): Record<string, unknown> {
32
+ return {
33
+ slow_provider: {
34
+ providerModuleId: 'caddy',
35
+ version: '1.0.0',
36
+ neverReturns: () => new Promise(() => {}),
37
+ },
38
+ };
39
+ }
40
+
41
+ async function runUntilTotalTimeout(): Promise<Error> {
42
+ const dir = mkdtempSync(join(tmpdir(), 'celilo-timeout-'));
43
+ try {
44
+ const context: HookContext = {
45
+ config: configStore(),
46
+ secrets: secretStore(),
47
+ systems: [],
48
+ logger: createCapturingLogger().logger,
49
+ debug: false,
50
+ screenshotDir: dir,
51
+ stateDir: dir,
52
+ capabilities: hangingCapabilities(),
53
+ };
54
+ try {
55
+ await executeHookScript(join(FIXTURES, 'slow-capability-hook.ts'), context, {
56
+ // Total well under idle, so the kill is unambiguously the TOTAL one:
57
+ // a hook blocked in a capability call emits nothing, so a shorter idle
58
+ // timer would win the race and prove a different thing.
59
+ timeoutMs: 3_000,
60
+ idleTimeoutMs: 60_000,
61
+ });
62
+ } catch (error) {
63
+ return error as Error;
64
+ }
65
+ throw new Error('the hook returned; it was supposed to be killed at its total timeout');
66
+ } finally {
67
+ rmSync(dir, { recursive: true, force: true });
68
+ }
69
+ }
70
+
71
+ describe('a total-timeout error names the provider it was waiting on (celilo#1403)', () => {
72
+ test('the run reached the total timeout, not the idle one', async () => {
73
+ const error = await runUntilTotalTimeout();
74
+ // Anchor the subject: an idle kill would exercise none of this.
75
+ expect(error.message).toContain('Hook total timeout exceeded');
76
+ expect(error.message).not.toContain('idle');
77
+ }, 20_000);
78
+
79
+ test('the message names the provider, the capability and the method', async () => {
80
+ const error = await runUntilTotalTimeout();
81
+ expect(error.message).toContain('waiting on');
82
+ // The provider MODULE is the operationally useful half — it is what the
83
+ // operator goes and looks at. `caddy` comes from `providerModuleId`.
84
+ expect(error.message).toContain('caddy');
85
+ expect(error.message).toContain('slow_provider.neverReturns');
86
+ }, 20_000);
87
+ });
@@ -5,13 +5,15 @@
5
5
  * manifest's `celilo_contract` field selects which contract applies; the
6
6
  * registry is the single source of truth for what each version promises.
7
7
  *
8
- * To mint a new contract version: add a new file (e.g. `./v2.ts`), import its
9
- * contract object here, and register it in `CONTRACTS`. Old contracts stay
10
- * registered indefinitely so legacy modules continue to validate.
8
+ * To mint a new contract version: add a new file (e.g. `./v3.ts`), import its
9
+ * contract object here, and register it in `CONTRACTS`. A version whose
10
+ * modules still work stays registered; a version whose modules would now FAIL
11
+ * SILENTLY moves to `RETIRED_CONTRACT_VERSIONS` with the reason, so the
12
+ * validator refuses it by name rather than dropping state at deploy time.
11
13
  */
12
14
 
13
- import { V1_CONTRACT } from './v1';
14
15
  import type { ContractHookSignature, ContractHooks } from './v1';
16
+ import { V2_CONTRACT } from './v2';
15
17
 
16
18
  export type { ContractHooks, ContractHookSignature };
17
19
 
@@ -26,7 +28,7 @@ export type { ContractHooks, ContractHookSignature };
26
28
  * `ContractHooks` stays exact.
27
29
  *
28
30
  * Returns `undefined` for a name the contract does not define; the caller
29
- * turns that into "Hook 'x' is not part of celilo_contract 1.0".
31
+ * turns that into "Hook 'x' is not part of celilo_contract 2.0".
30
32
  */
31
33
  export function contractHookSignature(
32
34
  hooks: ContractHooks,
@@ -53,16 +55,35 @@ export interface Contract {
53
55
  * `SUPPORTED_CONTRACT_VERSIONS` below so Zod's enum picks it up.
54
56
  */
55
57
  export const CONTRACTS: Record<string, Contract> = {
56
- '1.0': V1_CONTRACT,
58
+ '2.0': V2_CONTRACT,
59
+ };
60
+
61
+ /**
62
+ * Versions a manifest may still NAME but that no longer validate, mapped to
63
+ * the reason. A retired version is kept here rather than simply deleted so
64
+ * the validator can say what broke and how to move, instead of reporting a
65
+ * known version as an unrecognised string.
66
+ */
67
+ export const RETIRED_CONTRACT_VERSIONS: Record<string, string> = {
68
+ '1.0':
69
+ 'contract 1.0 persisted a hook\'s returned record as the module\'s secrets and config. That channel is removed (openspec/changes/hook-owned-state D5) — a 1.0 hook would deploy successfully and store nothing. Declare celilo_contract "2.0" and persist state with context.config.set / context.secrets.set against names the manifest declares.',
57
70
  };
58
71
 
59
72
  /**
60
73
  * Const tuple of supported contract version strings, used to drive the Zod
61
74
  * enum that validates the manifest's `celilo_contract` field.
62
75
  */
63
- export const SUPPORTED_CONTRACT_VERSIONS = ['1.0'] as const;
76
+ export const SUPPORTED_CONTRACT_VERSIONS = ['2.0'] as const;
64
77
  export type SupportedContractVersion = (typeof SUPPORTED_CONTRACT_VERSIONS)[number];
65
78
 
79
+ /**
80
+ * Every version string a manifest is allowed to WRITE — supported plus
81
+ * retired. The schema enum accepts these so a retired version reaches
82
+ * `validateHookContract`, which explains the retirement; anything else is a
83
+ * plain unknown-version error from Zod.
84
+ */
85
+ export const KNOWN_CONTRACT_VERSIONS = ['1.0', '2.0'] as const;
86
+
66
87
  /**
67
88
  * Look up a contract by version string.
68
89
  *
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Celilo Module Contract — v2.0
3
+ *
4
+ * Hook SIGNATURES are unchanged from v1.0 — `V2_HOOKS` is `V1_HOOKS`. What
5
+ * changed is what the framework does with a hook's return value: nothing.
6
+ *
7
+ * Under v1.0, a record returned from `validate_config`, `on_install` or
8
+ * `container_created` was swept back into the module's secrets (and, for
9
+ * validate_config, its config). That was the only write channel a hook had,
10
+ * so a hook that needed to persist something it discovered had to return it.
11
+ * The channel is gone (openspec/changes/hook-owned-state, D5): a hook now
12
+ * persists state explicitly through `context.config.set` / `context.secrets.set`,
13
+ * against names its manifest declares.
14
+ *
15
+ * This is a MAJOR bump rather than a v1.x clarification because a module
16
+ * written against the old channel keeps deploying and silently stores
17
+ * nothing — the deploy reports success and the state is dropped. Refusing
18
+ * `celilo_contract: "1.0"` outright (see `RETIRED_CONTRACT_VERSIONS`) turns
19
+ * that silent loss into a message naming the fix, which is the whole point of
20
+ * versioning the contract.
21
+ */
22
+
23
+ import { V1_HOOKS } from './v1';
24
+ import type { ContractHooks } from './v1';
25
+
26
+ /** v2.0 hook signatures — identical to v1.0. The break is behavioural. */
27
+ export const V2_HOOKS: ContractHooks = V1_HOOKS;
28
+
29
+ export const V2_CONTRACT = {
30
+ version: '2.0' as const,
31
+ hooks: V2_HOOKS,
32
+ };