@celilo/cli 4.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/CELILO_SUBSYSTEMS.md +1 -0
- package/drizzle/0034_private_routes.sql +41 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +5 -4
- package/src/api/sessions.test.ts +1 -1
- package/src/capabilities/registration.test.ts +11 -11
- package/src/capabilities/secrets.test.ts +18 -18
- package/src/capabilities/validation.test.ts +24 -24
- package/src/cli/cli.test.ts +3 -3
- package/src/cli/command-tree-parser.test.ts +18 -0
- package/src/cli/command-tree-parser.ts +12 -0
- package/src/cli/commands/api.ts +67 -14
- package/src/cli/commands/events.ts +5 -2
- package/src/cli/commands/machine-reclassify.ts +79 -0
- package/src/cli/commands/module-changeset.test.ts +1 -1
- package/src/cli/commands/module-generate.test.ts +2 -2
- package/src/cli/commands/module-import-aspect.test.ts +2 -2
- package/src/cli/commands/module-update.test.ts +14 -14
- package/src/cli/commands/module-upgrade-gate.test.ts +26 -1
- package/src/cli/commands/module-upgrade.test.ts +3 -3
- package/src/cli/commands/module-upgrade.ts +12 -0
- package/src/cli/commands/module-version.test.ts +1 -1
- package/src/cli/commands/system-audit.ts +22 -0
- package/src/cli/commands/system-reboot.ts +126 -0
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +3 -2
- package/src/cli/index.ts +41 -2
- package/src/cli/restore-migration-failure.test.ts +1 -1
- package/src/cli/stdout-pipe-flush.test.ts +35 -0
- package/src/cli/tui/audit-state.ts +4 -0
- package/src/db/private-routes.test.ts +159 -0
- package/src/db/schema.ts +49 -0
- package/src/hooks/broker.ts +31 -0
- package/src/hooks/capability-loader.ts +185 -12
- package/src/hooks/executor.test.ts +13 -13
- package/src/hooks/executor.ts +20 -2
- package/src/hooks/hook-state-dir.test.ts +2 -2
- package/src/hooks/test-fixtures/slow-capability-hook.ts +28 -0
- package/src/hooks/test-fixtures/store-backed.ts +5 -3
- package/src/hooks/timeout-names-provider.test.ts +87 -0
- package/src/manifest/contracts/index.ts +28 -7
- package/src/manifest/contracts/v2.ts +32 -0
- package/src/manifest/icon-schema.test.ts +1 -1
- package/src/manifest/schema.ts +13 -2
- package/src/manifest/template-validator.test.ts +1 -1
- package/src/manifest/validate-privileged.test.ts +1 -1
- package/src/manifest/validate.test.ts +68 -29
- package/src/manifest/validate.ts +14 -1
- package/src/module/import.test.ts +7 -7
- package/src/module/packaging/build.test.ts +1 -1
- package/src/packaging/nfpm-depends.test.ts +42 -0
- package/src/policy/capability-shape-baseline.ts +7 -1
- package/src/policy/fixture-capability-coverage.test.ts +5 -5
- package/src/policy/module-business-baseline.ts +33 -4
- package/src/services/api-access.test.ts +44 -0
- package/src/services/aspect-approvals.test.ts +2 -2
- package/src/services/aspect-reconcile.test.ts +2 -2
- package/src/services/aspect-runner.test.ts +4 -4
- package/src/services/audit/backups.test.ts +1 -1
- package/src/services/audit/capability-abi.test.ts +1 -1
- package/src/services/audit/index.test.ts +7 -0
- package/src/services/audit/index.ts +15 -0
- package/src/services/audit/machine-interface-zones.test.ts +30 -0
- package/src/services/audit/machine-interface-zones.ts +43 -0
- package/src/services/audit/module-configs.test.ts +1 -1
- package/src/services/audit/reboot-pending.test.ts +63 -0
- package/src/services/audit/reboot-pending.ts +85 -0
- package/src/services/audit/recurrence-gate.test.ts +21 -0
- package/src/services/audit/server-bun-pin.test.ts +44 -0
- package/src/services/audit/server-bun-pin.ts +95 -0
- package/src/services/audit/types.ts +2 -0
- package/src/services/build-bus/hook-dispatch-executor.test.ts +3 -3
- package/src/services/bus-interview-park.test.ts +1 -1
- package/src/services/bus-secret-flow.test.ts +1 -1
- package/src/services/capability-compat.test.ts +22 -0
- package/src/services/capability-compat.ts +15 -2
- package/src/services/capability-table-rows.test.ts +20 -2
- package/src/services/celilo-events.test.ts +119 -6
- package/src/services/celilo-events.ts +156 -16
- package/src/services/celilo-mgmt-hooks.test.ts +6 -3
- package/src/services/cross-module-read.test.ts +2 -2
- package/src/services/deploy-preflight.ts +12 -3
- package/src/services/deploy-validation.test.ts +86 -7
- package/src/services/deploy-validation.ts +19 -0
- package/src/services/dns-provider-backfill.test.ts +2 -2
- package/src/services/dns-registrations.test.ts +1 -1
- package/src/services/fleet-checks.test.ts +2 -2
- package/src/services/fleet-checks.ts +40 -10
- package/src/services/health-runner.test.ts +1 -1
- package/src/services/infrastructure-variable-resolver.test.ts +11 -11
- package/src/services/instance-ops.test.ts +2 -2
- package/src/services/machine-interface-zones.test.ts +73 -0
- package/src/services/machine-interface-zones.ts +101 -0
- package/src/services/module-deploy.dns-repoint.test.ts +1 -1
- package/src/services/module-deploy.stale-provider-guard.test.ts +98 -0
- package/src/services/module-deploy.ts +31 -10
- package/src/services/module-pause-quiescence.test.ts +1 -1
- package/src/services/module-pause.test.ts +1 -1
- package/src/services/module-subscriptions.test.ts +2 -2
- package/src/services/module-types-generator.test.ts +1 -1
- package/src/services/module-validator/contract-version.test.ts +3 -3
- package/src/services/network-ensure.test.ts +1 -1
- package/src/services/provider-arrival.test.ts +62 -46
- package/src/services/provider-arrival.ts +41 -10
- package/src/services/proxmox-reconcile.test.ts +2 -2
- package/src/services/remove-guard.test.ts +1 -1
- package/src/services/system-reboot.test.ts +231 -0
- package/src/services/system-reboot.ts +357 -0
- package/src/services/update/dep-graph.test.ts +1 -1
- package/src/services/update/orchestrator.test.ts +165 -1
- package/src/services/update/orchestrator.ts +43 -12
- package/src/services/update/types.ts +10 -1
- package/src/templates/generator.test.ts +15 -4
- package/src/templates/generator.ts +12 -4
- package/src/variables/capability-self-ref.test.ts +2 -2
- package/src/variables/computed/computed-integration.test.ts +1 -1
- package/src/variables/context.ts +14 -1
- package/src/variables/declarative-derivation.test.ts +19 -19
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
NETWORK_ZONES,
|
|
37
37
|
capabilities,
|
|
38
38
|
modules,
|
|
39
|
+
privateRoutes,
|
|
39
40
|
secrets,
|
|
40
41
|
systemConfig,
|
|
41
42
|
webRoutes,
|
|
@@ -45,7 +46,10 @@ import { decryptSecret } from '../secrets/encryption';
|
|
|
45
46
|
import { getOrCreateMasterKey } from '../secrets/master-key';
|
|
46
47
|
import { buildControlPlaneApi } from '../services/api-principal-enrolment';
|
|
47
48
|
import { recordCapabilityBinding, withBindingRecord } from '../services/capability-bindings';
|
|
48
|
-
import {
|
|
49
|
+
import {
|
|
50
|
+
emitPrivateRoutesChangedAndWait,
|
|
51
|
+
emitWebRoutesChangedAndWait,
|
|
52
|
+
} from '../services/celilo-events';
|
|
49
53
|
import { CONTROL_PLANE_MODULE_ID, getModuleSystems } from '../services/deployed-systems';
|
|
50
54
|
import { withDnsInternalLedger } from '../services/dns-internal-records';
|
|
51
55
|
import { withDnsRegistrationLedger } from '../services/dns-registrations';
|
|
@@ -95,22 +99,56 @@ import { loadHookConfigMap } from './load-hook-config';
|
|
|
95
99
|
* success anyway, so a consumer could report "ready" while caddy never learned
|
|
96
100
|
* the hostname — no site block, no cert, TLS internal_error for clients.
|
|
97
101
|
*/
|
|
98
|
-
|
|
99
|
-
|
|
102
|
+
/**
|
|
103
|
+
* Which ingress a reconcile is being required for. The two differ only in the
|
|
104
|
+
* event type they emit and the names their failures carry, so they share this
|
|
105
|
+
* function rather than each keeping a copy — a private ingress that silently
|
|
106
|
+
* stopped waiting would look exactly like one with nothing to wait for
|
|
107
|
+
* (providers-converge-declared-state, slice 2).
|
|
108
|
+
*/
|
|
109
|
+
const INGRESS = {
|
|
110
|
+
public_web: { capability: 'public_web', provider: 'caddy', emit: emitWebRoutesChangedAndWait },
|
|
111
|
+
private_web: {
|
|
112
|
+
capability: 'private_web',
|
|
113
|
+
provider: 'caddy-internal',
|
|
114
|
+
emit: emitPrivateRoutesChangedAndWait,
|
|
115
|
+
},
|
|
116
|
+
} as const;
|
|
117
|
+
|
|
118
|
+
async function requireProviderReconcile(
|
|
119
|
+
consumingModuleId: string,
|
|
120
|
+
what: string,
|
|
121
|
+
ingress: keyof typeof INGRESS = 'public_web',
|
|
122
|
+
): Promise<void> {
|
|
123
|
+
const { capability, provider, emit } = INGRESS[ingress];
|
|
124
|
+
const reconcile = await emit(consumingModuleId);
|
|
100
125
|
|
|
101
126
|
if (reconcile.noDispatcher) {
|
|
102
127
|
throw new Error(
|
|
103
|
-
|
|
128
|
+
`${capability} ${what} for ${consumingModuleId} was persisted but NOT delivered to the provider (${provider}): no event dispatcher is running, so ${provider} never reconciled and the hostname has no site block or cert. Run this through \`celilo module deploy\` (which runs the dispatcher) rather than a bare hook.`,
|
|
104
129
|
);
|
|
105
130
|
}
|
|
131
|
+
// The handler's own words, when it left any. A crash is retried, so its
|
|
132
|
+
// delivery sits in `pending` with the error already recorded while the
|
|
133
|
+
// deadline runs out — which is how celilo#1401 reported a TIMEOUT for a
|
|
134
|
+
// caddy crash, for days, with `0 ok, 0 failed of 1` and the cause nowhere.
|
|
135
|
+
const because = reconcile.lastError ? ` The provider reported: ${reconcile.lastError}` : '';
|
|
136
|
+
|
|
106
137
|
if (reconcile.timedOut) {
|
|
138
|
+
// "0 ok, 0 failed" reads as "nobody consumed it", which sends the reader to
|
|
139
|
+
// the dispatcher (celilo#1398/#1399) when the truth is that the provider's
|
|
140
|
+
// deliveries were STILL RUNNING at the deadline (celilo#1416). Say which.
|
|
141
|
+
const inFlight =
|
|
142
|
+
reconcile.stillRunning > 0
|
|
143
|
+
? `${reconcile.stillRunning} delivery(ies) still running`
|
|
144
|
+
: `${reconcile.succeeded} ok, ${reconcile.failed} failed`;
|
|
107
145
|
throw new Error(
|
|
108
|
-
|
|
146
|
+
`${capability} reconcile for ${consumingModuleId} (${what}) did not finish within its ${Math.round(reconcile.budgetMs / 1000)}s budget (${inFlight} of ${reconcile.events} change event(s)) — ${provider} did not confirm the change is live.${because}`,
|
|
109
147
|
);
|
|
110
148
|
}
|
|
111
149
|
if (reconcile.failed > 0) {
|
|
112
150
|
throw new Error(
|
|
113
|
-
|
|
151
|
+
`${capability} reconcile for ${consumingModuleId} (${what}): ${reconcile.failed} delivery(ies) failed — ${provider} could not apply the change, so the hostname is not served as declared.${because}`,
|
|
114
152
|
);
|
|
115
153
|
}
|
|
116
154
|
// ISS-0087: a change WAS persisted and the dispatcher IS alive, yet ZERO
|
|
@@ -120,7 +158,7 @@ async function requireProviderReconcile(consumingModuleId: string, what: string)
|
|
|
120
158
|
// changed and nobody served it — a failure.
|
|
121
159
|
if (reconcile.events > 0 && reconcile.succeeded === 0) {
|
|
122
160
|
throw new Error(
|
|
123
|
-
|
|
161
|
+
`${capability} ${what} for ${consumingModuleId} changed (${reconcile.events} event(s)) but NO provider reconciled it — ${provider} has no reconcile_routes subscription on the bus, so the change is persisted yet never served. Ensure a ${capability} provider (${provider}) is deployed and subscribed.`,
|
|
124
162
|
);
|
|
125
163
|
}
|
|
126
164
|
}
|
|
@@ -169,11 +207,13 @@ export const CAPABILITY_MODULE_MAP: Record<string, { script: string; legacyFacto
|
|
|
169
207
|
legacyFactoryName: 'default',
|
|
170
208
|
},
|
|
171
209
|
// MODULE-provided, unlike its sibling public_web above, which the framework
|
|
172
|
-
// implements (createPublicWeb)
|
|
173
|
-
//
|
|
174
|
-
//
|
|
175
|
-
//
|
|
176
|
-
//
|
|
210
|
+
// implements (createPublicWeb). The two ingresses are separate TABLES, not
|
|
211
|
+
// separate owners: core owns `private_routes` as it owns `web_routes`
|
|
212
|
+
// (migration 0034), and that separation — not the provider keeping its own
|
|
213
|
+
// config — is what stops the PUBLIC caddy, which derives its served
|
|
214
|
+
// hostnames from every row of `web_routes`, from sweeping a private route
|
|
215
|
+
// into the public ingress (celilo#846). What stays module-side is the
|
|
216
|
+
// RENDER: caddy-internal decides what its Caddyfile says.
|
|
177
217
|
private_web: {
|
|
178
218
|
script: 'scripts/private-web-functions.ts',
|
|
179
219
|
legacyFactoryName: 'default',
|
|
@@ -493,6 +533,26 @@ export async function loadCapabilityFunctions(
|
|
|
493
533
|
// same seam. A jailed hook cannot write into its own web root, so a
|
|
494
534
|
// module-provided web provider needs the state overlay to honor it.
|
|
495
535
|
consumerStateWebRoot: resolveModuleStateWebRoot(consumingModuleId, db),
|
|
536
|
+
// THE PRIVATE INGRESS'S DECLARED ROWS, for the provider that serves
|
|
537
|
+
// them. Scoped to private_web deliberately: these are one
|
|
538
|
+
// capability's declared state, and handing them to every
|
|
539
|
+
// module-provided capability would put the private route table in
|
|
540
|
+
// reach of modules that have no business reading it.
|
|
541
|
+
...(capName === 'private_web'
|
|
542
|
+
? {
|
|
543
|
+
privateRoutes: buildPrivateRouteOps(db),
|
|
544
|
+
// The same seam public_web has (see `onRoutesChanged` below):
|
|
545
|
+
// the capability persists the row and hands the converge to
|
|
546
|
+
// the provider's `reconcile_routes` subscription, then AWAITS
|
|
547
|
+
// that delivery so `registerReverseProxy` still returns only
|
|
548
|
+
// once the route is live (design D7). One primitive, two
|
|
549
|
+
// triggers — the subscription is the second, not a
|
|
550
|
+
// replacement for the call.
|
|
551
|
+
onRoutesChanged: async () => {
|
|
552
|
+
await requireProviderReconcile(consumingModuleId, 'route change', 'private_web');
|
|
553
|
+
},
|
|
554
|
+
}
|
|
555
|
+
: {}),
|
|
496
556
|
});
|
|
497
557
|
// Stamp here too, not only on the legacy path: a consumer that cannot
|
|
498
558
|
// get what it needs must be able to name WHICH provider could not give
|
|
@@ -811,6 +871,51 @@ export async function loadCapabilityFunctions(
|
|
|
811
871
|
debugLog('public_web: not registered in DB, skipping');
|
|
812
872
|
}
|
|
813
873
|
|
|
874
|
+
// THE PRIVATE INGRESS, symmetric with the public one above (design D7 of
|
|
875
|
+
// providers-converge-declared-state). Its provider gets a view of the rows it
|
|
876
|
+
// serves and the converge that places what it renders — the same two things
|
|
877
|
+
// caddy gets, for the same reason, now that its routes are core-owned state
|
|
878
|
+
// rather than its own module-config JSON.
|
|
879
|
+
//
|
|
880
|
+
// Consumers never see either: only the module that PROVIDES private_web.
|
|
881
|
+
const privateWebProviders = db
|
|
882
|
+
.select()
|
|
883
|
+
.from(capabilities)
|
|
884
|
+
.where(eq(capabilities.capabilityName, 'private_web'))
|
|
885
|
+
.all();
|
|
886
|
+
const privateWebProvider = privateWebProviders[0];
|
|
887
|
+
if (privateWebProvider && consumingModuleId === privateWebProvider.moduleId) {
|
|
888
|
+
result.private_routes = buildPrivateRouteOps(db);
|
|
889
|
+
debugLog(`private_routes: injected for provider ${consumingModuleId}`);
|
|
890
|
+
|
|
891
|
+
const providerModuleId = privateWebProvider.moduleId;
|
|
892
|
+
const privateConverge: ProviderConvergeView = {
|
|
893
|
+
// The private ingress serves no consumer-built static sites today: every
|
|
894
|
+
// row it renders is a reverse proxy. Answering honestly rather than
|
|
895
|
+
// throwing keeps the shape identical to the public provider's, so one
|
|
896
|
+
// renderer contract covers both (design D1).
|
|
897
|
+
resolveConsumerSite: async (moduleId: string) => ({
|
|
898
|
+
unavailable: `the private ingress serves no static sites, so '${moduleId}' has none here`,
|
|
899
|
+
}),
|
|
900
|
+
converge: async (artifacts) => {
|
|
901
|
+
const { convergeProviderConfig, resolveStaticContentRetention } = await import(
|
|
902
|
+
'../services/provider-converge'
|
|
903
|
+
);
|
|
904
|
+
const result = await convergeProviderConfig(db, providerModuleId, {
|
|
905
|
+
...artifacts,
|
|
906
|
+
retention: resolveStaticContentRetention(db, providerModuleId),
|
|
907
|
+
});
|
|
908
|
+
return {
|
|
909
|
+
success: result.success,
|
|
910
|
+
...(result.error ? { error: result.error } : {}),
|
|
911
|
+
unresolved: result.unresolved,
|
|
912
|
+
};
|
|
913
|
+
},
|
|
914
|
+
};
|
|
915
|
+
result.provider_converge = privateConverge;
|
|
916
|
+
debugLog(`provider_converge: injected for private provider ${consumingModuleId}`);
|
|
917
|
+
}
|
|
918
|
+
|
|
814
919
|
// Framework-granted, so unlike everything above there is no provider row to
|
|
815
920
|
// look up and no script to import — celilo IS the management server whose
|
|
816
921
|
// principals these are (web-ui-console D7b).
|
|
@@ -1482,6 +1587,74 @@ async function buildFirewallChain(
|
|
|
1482
1587
|
return { chain: currentUpstream, self };
|
|
1483
1588
|
}
|
|
1484
1589
|
|
|
1590
|
+
/**
|
|
1591
|
+
* Route ops for the private_web PROVIDER, over `private_routes`.
|
|
1592
|
+
*
|
|
1593
|
+
* The private ingress's rows are core-owned declared state now, exactly as
|
|
1594
|
+
* `web_routes` is for public_web (design D1 of
|
|
1595
|
+
* providers-converge-declared-state). They used to live in caddy-internal's
|
|
1596
|
+
* `routes` module-config JSON, which is why the two ingresses could not share
|
|
1597
|
+
* plumbing: celilo had nothing to render, converge or audit.
|
|
1598
|
+
*
|
|
1599
|
+
* SEPARATE TABLE, and that is the whole safety property. The public provider
|
|
1600
|
+
* derives the hostnames it serves from every row of ITS table, so a private
|
|
1601
|
+
* route sharing that table is one missing WHERE clause from being served
|
|
1602
|
+
* publicly with a public certificate (celilo#846). Injected only into the
|
|
1603
|
+
* private_web provider's own capability functions, the same way the route view
|
|
1604
|
+
* reaches caddy and no one else.
|
|
1605
|
+
*
|
|
1606
|
+
* Rows go out in the snake_case shape the provider's renderer already speaks,
|
|
1607
|
+
* so the module needs no translation layer and its existing route tests keep
|
|
1608
|
+
* asserting the same objects.
|
|
1609
|
+
*/
|
|
1610
|
+
function buildPrivateRouteOps(db: DbClient) {
|
|
1611
|
+
const toRecord = (row: typeof privateRoutes.$inferSelect) => ({
|
|
1612
|
+
module_id: row.moduleId,
|
|
1613
|
+
slug: row.slug,
|
|
1614
|
+
type: row.type,
|
|
1615
|
+
path: row.path,
|
|
1616
|
+
hostname: row.hostname,
|
|
1617
|
+
...(row.targetHost ? { target_host: row.targetHost } : {}),
|
|
1618
|
+
...(row.targetPort ? { target_port: row.targetPort } : {}),
|
|
1619
|
+
...(row.websocket ? { websocket: row.websocket } : {}),
|
|
1620
|
+
});
|
|
1621
|
+
|
|
1622
|
+
return {
|
|
1623
|
+
all: () => db.select().from(privateRoutes).all().map(toRecord),
|
|
1624
|
+
/**
|
|
1625
|
+
* Replace the whole set.
|
|
1626
|
+
*
|
|
1627
|
+
* The provider keeps the full route list in memory for the life of one
|
|
1628
|
+
* hook run and hands it back whole, which is the semantics its config key
|
|
1629
|
+
* had. Honouring that here keeps the change to the module small and the
|
|
1630
|
+
* two halves easy to reason about.
|
|
1631
|
+
*
|
|
1632
|
+
* ponytail: whole-set replace, in one transaction so a reader never sees
|
|
1633
|
+
* an empty ingress. If two providers ever write this table concurrently,
|
|
1634
|
+
* this becomes per-row upsert plus a delete of what the writer dropped.
|
|
1635
|
+
*/
|
|
1636
|
+
replaceAll: (rows: ReturnType<typeof toRecord>[]) => {
|
|
1637
|
+
db.transaction((tx) => {
|
|
1638
|
+
tx.delete(privateRoutes).run();
|
|
1639
|
+
for (const row of rows) {
|
|
1640
|
+
tx.insert(privateRoutes)
|
|
1641
|
+
.values({
|
|
1642
|
+
slug: row.slug,
|
|
1643
|
+
moduleId: row.module_id,
|
|
1644
|
+
type: row.type,
|
|
1645
|
+
path: row.path,
|
|
1646
|
+
hostname: row.hostname,
|
|
1647
|
+
targetHost: row.target_host ?? null,
|
|
1648
|
+
targetPort: row.target_port ?? null,
|
|
1649
|
+
websocket: row.websocket ?? false,
|
|
1650
|
+
})
|
|
1651
|
+
.run();
|
|
1652
|
+
}
|
|
1653
|
+
});
|
|
1654
|
+
},
|
|
1655
|
+
};
|
|
1656
|
+
}
|
|
1657
|
+
|
|
1485
1658
|
/**
|
|
1486
1659
|
* Build RouteOps for the public_web capability implementation.
|
|
1487
1660
|
* Wraps database operations so the capability never touches the DB directly.
|
|
@@ -529,7 +529,7 @@ describe('Hook Executor', () => {
|
|
|
529
529
|
const result = await invokeHook(
|
|
530
530
|
__dirname,
|
|
531
531
|
'container_created',
|
|
532
|
-
'
|
|
532
|
+
'2.0',
|
|
533
533
|
{ script: './test-fixtures/artifact-writing-hook.ts', timeout: 10000 },
|
|
534
534
|
{ vps_ip: '10.0.0.5' },
|
|
535
535
|
{},
|
|
@@ -560,7 +560,7 @@ describe('Hook Executor', () => {
|
|
|
560
560
|
const result = await invokeHook(
|
|
561
561
|
__dirname,
|
|
562
562
|
'container_created',
|
|
563
|
-
'
|
|
563
|
+
'2.0',
|
|
564
564
|
{ script: './test-fixtures/success-hook.ts', timeout: 10000 },
|
|
565
565
|
{ vps_ip: '10.0.0.5' },
|
|
566
566
|
{},
|
|
@@ -585,7 +585,7 @@ describe('Hook Executor', () => {
|
|
|
585
585
|
const result = await invokeHook(
|
|
586
586
|
__dirname,
|
|
587
587
|
'container_created',
|
|
588
|
-
'
|
|
588
|
+
'2.0',
|
|
589
589
|
{ script: './test-fixtures/success-hook.ts', timeout: 10000 },
|
|
590
590
|
{ vps_ip: '10.0.0.5' },
|
|
591
591
|
{},
|
|
@@ -614,7 +614,7 @@ describe('Hook Executor', () => {
|
|
|
614
614
|
const result = await invokeHook(
|
|
615
615
|
__dirname,
|
|
616
616
|
'container_created',
|
|
617
|
-
'
|
|
617
|
+
'2.0',
|
|
618
618
|
definition,
|
|
619
619
|
{ vps_ip: '10.0.0.5' },
|
|
620
620
|
{ username: 'test' },
|
|
@@ -641,7 +641,7 @@ describe('Hook Executor', () => {
|
|
|
641
641
|
const result = await invokeHook(
|
|
642
642
|
__dirname,
|
|
643
643
|
'on_backup',
|
|
644
|
-
'
|
|
644
|
+
'2.0',
|
|
645
645
|
definition,
|
|
646
646
|
{},
|
|
647
647
|
{},
|
|
@@ -663,7 +663,7 @@ describe('Hook Executor', () => {
|
|
|
663
663
|
const result = await invokeHook(
|
|
664
664
|
__dirname,
|
|
665
665
|
'container_created',
|
|
666
|
-
'
|
|
666
|
+
'2.0',
|
|
667
667
|
definition,
|
|
668
668
|
{},
|
|
669
669
|
{},
|
|
@@ -686,7 +686,7 @@ describe('Hook Executor', () => {
|
|
|
686
686
|
const result = await invokeHook(
|
|
687
687
|
__dirname,
|
|
688
688
|
'on_backup',
|
|
689
|
-
'
|
|
689
|
+
'2.0',
|
|
690
690
|
definition,
|
|
691
691
|
{ backup_dir: '/tmp/backup' },
|
|
692
692
|
{},
|
|
@@ -708,7 +708,7 @@ describe('Hook Executor', () => {
|
|
|
708
708
|
const result = await invokeHook(
|
|
709
709
|
__dirname,
|
|
710
710
|
'container_created',
|
|
711
|
-
'
|
|
711
|
+
'2.0',
|
|
712
712
|
definition,
|
|
713
713
|
{},
|
|
714
714
|
{},
|
|
@@ -743,10 +743,10 @@ describe('Hook Executor', () => {
|
|
|
743
743
|
const { logger } = createCapturingLogger();
|
|
744
744
|
const definition: HookDefinition = { script: './test-fixtures/success-hook.ts' };
|
|
745
745
|
|
|
746
|
-
const result = await invokeHook(__dirname, 'on_poop', '
|
|
746
|
+
const result = await invokeHook(__dirname, 'on_poop', '2.0', definition, {}, {}, {}, logger);
|
|
747
747
|
|
|
748
748
|
expect(result.success).toBe(false);
|
|
749
|
-
expect(result.error).toContain("Hook 'on_poop' is not part of celilo_contract
|
|
749
|
+
expect(result.error).toContain("Hook 'on_poop' is not part of celilo_contract 2.0");
|
|
750
750
|
});
|
|
751
751
|
|
|
752
752
|
test('pre-flight: fails when a required capability is missing', async () => {
|
|
@@ -759,7 +759,7 @@ describe('Hook Executor', () => {
|
|
|
759
759
|
const result = await invokeHook(
|
|
760
760
|
__dirname,
|
|
761
761
|
'container_created',
|
|
762
|
-
'
|
|
762
|
+
'2.0',
|
|
763
763
|
definition,
|
|
764
764
|
{ vps_ip: '10.0.0.5' },
|
|
765
765
|
{},
|
|
@@ -786,7 +786,7 @@ describe('Hook Executor', () => {
|
|
|
786
786
|
const result = await invokeHook(
|
|
787
787
|
__dirname,
|
|
788
788
|
'container_created',
|
|
789
|
-
'
|
|
789
|
+
'2.0',
|
|
790
790
|
definition,
|
|
791
791
|
{ vps_ip: '10.0.0.5' },
|
|
792
792
|
{},
|
|
@@ -813,7 +813,7 @@ describe('Hook Executor', () => {
|
|
|
813
813
|
const result = await invokeHook(
|
|
814
814
|
__dirname,
|
|
815
815
|
'container_created',
|
|
816
|
-
'
|
|
816
|
+
'2.0',
|
|
817
817
|
definition,
|
|
818
818
|
{ vps_ip: '10.0.0.5' },
|
|
819
819
|
{},
|
package/src/hooks/executor.ts
CHANGED
|
@@ -511,7 +511,14 @@ export async function executeHookScript(
|
|
|
511
511
|
setTimeout(() => child.kill('SIGKILL'), SIGKILL_GRACE_MS).unref();
|
|
512
512
|
};
|
|
513
513
|
|
|
514
|
-
|
|
514
|
+
// Snapshotted inside the timer rather than read at throw time: the child's
|
|
515
|
+
// death rejects its in-flight calls, so by the time we build the message
|
|
516
|
+
// the map that knows the answer has drained itself.
|
|
517
|
+
let pendingCallsAtKill: string[] = [];
|
|
518
|
+
const totalTimer = setTimeout(() => {
|
|
519
|
+
pendingCallsAtKill = broker.pendingCalls();
|
|
520
|
+
kill('total');
|
|
521
|
+
}, timeoutMs);
|
|
515
522
|
// Debug runs get no idle timer, same as before: an operator stepping
|
|
516
523
|
// through a browser hook is silent for minutes on purpose.
|
|
517
524
|
const idleTimer = context.debug
|
|
@@ -536,7 +543,18 @@ export async function executeHookScript(
|
|
|
536
543
|
// nothing keeps running past this line. That is the difference from the
|
|
537
544
|
// `Promise.race` this replaces (celilo#1003).
|
|
538
545
|
if (killedFor === 'total') {
|
|
539
|
-
|
|
546
|
+
// Name what it was WAITING ON, not just that it waited. A hook whose
|
|
547
|
+
// budget is consumed by one slow provider used to die as a bare
|
|
548
|
+
// "Hook total timeout exceeded", so the operator had to infer the
|
|
549
|
+
// provider from a capability push line with no matching completion
|
|
550
|
+
// (celilo#1403: bna-yard-sale's 120s on_install blocked on caddy's 75s
|
|
551
|
+
// route reconcile). Captured before `close()`, while the calls are
|
|
552
|
+
// still in flight — `stop()` refuses new calls but clears nothing.
|
|
553
|
+
const waitingOn = pendingCallsAtKill;
|
|
554
|
+
const waitingSuffix = waitingOn.length > 0 ? `, waiting on ${waitingOn.join(', ')}` : '';
|
|
555
|
+
throw new Error(
|
|
556
|
+
`Hook total timeout exceeded (${Math.round(timeoutMs / 1000)}s)${waitingSuffix}`,
|
|
557
|
+
);
|
|
540
558
|
}
|
|
541
559
|
if (killedFor === 'idle') {
|
|
542
560
|
throw new Error(
|
|
@@ -62,7 +62,7 @@ describe('celilo#1000: a hook is TOLD where its state directory is', () => {
|
|
|
62
62
|
const result = await invokeHook(
|
|
63
63
|
root,
|
|
64
64
|
'on_install',
|
|
65
|
-
'
|
|
65
|
+
'2.0',
|
|
66
66
|
{ script: 'hook.ts' },
|
|
67
67
|
{},
|
|
68
68
|
{},
|
|
@@ -97,7 +97,7 @@ describe('celilo#1000: a hook is TOLD where its state directory is', () => {
|
|
|
97
97
|
const result = await invokeHook(
|
|
98
98
|
root,
|
|
99
99
|
'on_install',
|
|
100
|
-
'
|
|
100
|
+
'2.0',
|
|
101
101
|
{ script: 'hook.ts' },
|
|
102
102
|
{},
|
|
103
103
|
{},
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test fixture: a hook that blocks inside a capability call it never gets an
|
|
3
|
+
* answer to, and is killed at its own total timeout (celilo#1403).
|
|
4
|
+
*
|
|
5
|
+
* The real shape is a consumer's `on_install` calling
|
|
6
|
+
* `public_web.publishStaticSite`, which blocks on the provider's route
|
|
7
|
+
* reconcile. This fixture reproduces the boundary condition — a call in flight
|
|
8
|
+
* when the kill lands — without a provider.
|
|
9
|
+
*
|
|
10
|
+
* It logs BEFORE the call so the run is not mistaken for an idle timeout, then
|
|
11
|
+
* awaits forever.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { defineHook } from '@celilo/capabilities';
|
|
15
|
+
|
|
16
|
+
interface SlowCapability {
|
|
17
|
+
neverReturns(request: Record<string, unknown>): Promise<unknown>;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export default defineHook({
|
|
21
|
+
hook: 'container_created',
|
|
22
|
+
requires: [],
|
|
23
|
+
handler: async (ctx) => {
|
|
24
|
+
const slow = (ctx.capabilities as unknown as Record<string, SlowCapability>).slow_provider;
|
|
25
|
+
ctx.logger.info('calling the provider');
|
|
26
|
+
await slow.neverReturns({});
|
|
27
|
+
},
|
|
28
|
+
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { HookStore
|
|
1
|
+
import type { HookStore } from '@celilo/capabilities';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* In-memory hook-owned-state stores for tests that build a HookContext
|
|
@@ -16,10 +16,12 @@ export function configStore(
|
|
|
16
16
|
return attach(values);
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
export function secretStore(
|
|
19
|
+
export function secretStore(
|
|
20
|
+
values: Record<string, string> = {},
|
|
21
|
+
): Record<string, string> & HookStore {
|
|
20
22
|
// Downcast: the caller passed Record<string, string>; attach only adds
|
|
21
23
|
// methods and never writes, so the value type is unchanged.
|
|
22
|
-
return attach(values) as
|
|
24
|
+
return attach(values) as Record<string, string> & HookStore;
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
function attach(values: Record<string, unknown>): Record<string, unknown> & HookStore {
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A hook killed at its TOTAL timeout must name what it was waiting on.
|
|
3
|
+
*
|
|
4
|
+
* Measured 2026-09-21 (celilo#1403): bna-yard-sale's `on_install` called
|
|
5
|
+
* `public_web.publishStaticSite`, which blocks on caddy's route reconcile —
|
|
6
|
+
* 70-75s across the fleet's sites. The consumer died at its 120s budget with:
|
|
7
|
+
*
|
|
8
|
+
* [bna-yard-sale:on_install] -> public_web.publishStaticSite
|
|
9
|
+
* [bna-yard-sale:on_install] OK dns_registrar.registerHost
|
|
10
|
+
* [bna-yard-sale:on_install] OK dns_internal.registerRecord
|
|
11
|
+
* [bna-yard-sale:on_install] FAIL Hook total timeout exceeded (120s)
|
|
12
|
+
*
|
|
13
|
+
* Both DNS calls completed; the push line had no matching completion, and that
|
|
14
|
+
* ABSENCE was the only evidence of which provider held it. The error named the
|
|
15
|
+
* hook and never the cause, so the operator had to infer it from a gap in the
|
|
16
|
+
* log — the same class of missing signal the greenwave/axon ISP-router hooks
|
|
17
|
+
* already carry a comment about.
|
|
18
|
+
*/
|
|
19
|
+
import { describe, expect, test } from 'bun:test';
|
|
20
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
21
|
+
import { tmpdir } from 'node:os';
|
|
22
|
+
import { join } from 'node:path';
|
|
23
|
+
import { executeHookScript } from './executor';
|
|
24
|
+
import { createCapturingLogger } from './logger';
|
|
25
|
+
import { configStore, secretStore } from './test-fixtures/store-backed';
|
|
26
|
+
import type { HookContext } from './types';
|
|
27
|
+
|
|
28
|
+
const FIXTURES = join(__dirname, 'test-fixtures');
|
|
29
|
+
|
|
30
|
+
/** A capability whose one method never settles — the provider that holds the hook. */
|
|
31
|
+
function hangingCapabilities(): Record<string, unknown> {
|
|
32
|
+
return {
|
|
33
|
+
slow_provider: {
|
|
34
|
+
providerModuleId: 'caddy',
|
|
35
|
+
version: '1.0.0',
|
|
36
|
+
neverReturns: () => new Promise(() => {}),
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function runUntilTotalTimeout(): Promise<Error> {
|
|
42
|
+
const dir = mkdtempSync(join(tmpdir(), 'celilo-timeout-'));
|
|
43
|
+
try {
|
|
44
|
+
const context: HookContext = {
|
|
45
|
+
config: configStore(),
|
|
46
|
+
secrets: secretStore(),
|
|
47
|
+
systems: [],
|
|
48
|
+
logger: createCapturingLogger().logger,
|
|
49
|
+
debug: false,
|
|
50
|
+
screenshotDir: dir,
|
|
51
|
+
stateDir: dir,
|
|
52
|
+
capabilities: hangingCapabilities(),
|
|
53
|
+
};
|
|
54
|
+
try {
|
|
55
|
+
await executeHookScript(join(FIXTURES, 'slow-capability-hook.ts'), context, {
|
|
56
|
+
// Total well under idle, so the kill is unambiguously the TOTAL one:
|
|
57
|
+
// a hook blocked in a capability call emits nothing, so a shorter idle
|
|
58
|
+
// timer would win the race and prove a different thing.
|
|
59
|
+
timeoutMs: 3_000,
|
|
60
|
+
idleTimeoutMs: 60_000,
|
|
61
|
+
});
|
|
62
|
+
} catch (error) {
|
|
63
|
+
return error as Error;
|
|
64
|
+
}
|
|
65
|
+
throw new Error('the hook returned; it was supposed to be killed at its total timeout');
|
|
66
|
+
} finally {
|
|
67
|
+
rmSync(dir, { recursive: true, force: true });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
describe('a total-timeout error names the provider it was waiting on (celilo#1403)', () => {
|
|
72
|
+
test('the run reached the total timeout, not the idle one', async () => {
|
|
73
|
+
const error = await runUntilTotalTimeout();
|
|
74
|
+
// Anchor the subject: an idle kill would exercise none of this.
|
|
75
|
+
expect(error.message).toContain('Hook total timeout exceeded');
|
|
76
|
+
expect(error.message).not.toContain('idle');
|
|
77
|
+
}, 20_000);
|
|
78
|
+
|
|
79
|
+
test('the message names the provider, the capability and the method', async () => {
|
|
80
|
+
const error = await runUntilTotalTimeout();
|
|
81
|
+
expect(error.message).toContain('waiting on');
|
|
82
|
+
// The provider MODULE is the operationally useful half — it is what the
|
|
83
|
+
// operator goes and looks at. `caddy` comes from `providerModuleId`.
|
|
84
|
+
expect(error.message).toContain('caddy');
|
|
85
|
+
expect(error.message).toContain('slow_provider.neverReturns');
|
|
86
|
+
}, 20_000);
|
|
87
|
+
});
|
|
@@ -5,13 +5,15 @@
|
|
|
5
5
|
* manifest's `celilo_contract` field selects which contract applies; the
|
|
6
6
|
* registry is the single source of truth for what each version promises.
|
|
7
7
|
*
|
|
8
|
-
* To mint a new contract version: add a new file (e.g. `./
|
|
9
|
-
* contract object here, and register it in `CONTRACTS`.
|
|
10
|
-
* registered
|
|
8
|
+
* To mint a new contract version: add a new file (e.g. `./v3.ts`), import its
|
|
9
|
+
* contract object here, and register it in `CONTRACTS`. A version whose
|
|
10
|
+
* modules still work stays registered; a version whose modules would now FAIL
|
|
11
|
+
* SILENTLY moves to `RETIRED_CONTRACT_VERSIONS` with the reason, so the
|
|
12
|
+
* validator refuses it by name rather than dropping state at deploy time.
|
|
11
13
|
*/
|
|
12
14
|
|
|
13
|
-
import { V1_CONTRACT } from './v1';
|
|
14
15
|
import type { ContractHookSignature, ContractHooks } from './v1';
|
|
16
|
+
import { V2_CONTRACT } from './v2';
|
|
15
17
|
|
|
16
18
|
export type { ContractHooks, ContractHookSignature };
|
|
17
19
|
|
|
@@ -26,7 +28,7 @@ export type { ContractHooks, ContractHookSignature };
|
|
|
26
28
|
* `ContractHooks` stays exact.
|
|
27
29
|
*
|
|
28
30
|
* Returns `undefined` for a name the contract does not define; the caller
|
|
29
|
-
* turns that into "Hook 'x' is not part of celilo_contract
|
|
31
|
+
* turns that into "Hook 'x' is not part of celilo_contract 2.0".
|
|
30
32
|
*/
|
|
31
33
|
export function contractHookSignature(
|
|
32
34
|
hooks: ContractHooks,
|
|
@@ -53,16 +55,35 @@ export interface Contract {
|
|
|
53
55
|
* `SUPPORTED_CONTRACT_VERSIONS` below so Zod's enum picks it up.
|
|
54
56
|
*/
|
|
55
57
|
export const CONTRACTS: Record<string, Contract> = {
|
|
56
|
-
'
|
|
58
|
+
'2.0': V2_CONTRACT,
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Versions a manifest may still NAME but that no longer validate, mapped to
|
|
63
|
+
* the reason. A retired version is kept here rather than simply deleted so
|
|
64
|
+
* the validator can say what broke and how to move, instead of reporting a
|
|
65
|
+
* known version as an unrecognised string.
|
|
66
|
+
*/
|
|
67
|
+
export const RETIRED_CONTRACT_VERSIONS: Record<string, string> = {
|
|
68
|
+
'1.0':
|
|
69
|
+
'contract 1.0 persisted a hook\'s returned record as the module\'s secrets and config. That channel is removed (openspec/changes/hook-owned-state D5) — a 1.0 hook would deploy successfully and store nothing. Declare celilo_contract "2.0" and persist state with context.config.set / context.secrets.set against names the manifest declares.',
|
|
57
70
|
};
|
|
58
71
|
|
|
59
72
|
/**
|
|
60
73
|
* Const tuple of supported contract version strings, used to drive the Zod
|
|
61
74
|
* enum that validates the manifest's `celilo_contract` field.
|
|
62
75
|
*/
|
|
63
|
-
export const SUPPORTED_CONTRACT_VERSIONS = ['
|
|
76
|
+
export const SUPPORTED_CONTRACT_VERSIONS = ['2.0'] as const;
|
|
64
77
|
export type SupportedContractVersion = (typeof SUPPORTED_CONTRACT_VERSIONS)[number];
|
|
65
78
|
|
|
79
|
+
/**
|
|
80
|
+
* Every version string a manifest is allowed to WRITE — supported plus
|
|
81
|
+
* retired. The schema enum accepts these so a retired version reaches
|
|
82
|
+
* `validateHookContract`, which explains the retirement; anything else is a
|
|
83
|
+
* plain unknown-version error from Zod.
|
|
84
|
+
*/
|
|
85
|
+
export const KNOWN_CONTRACT_VERSIONS = ['1.0', '2.0'] as const;
|
|
86
|
+
|
|
66
87
|
/**
|
|
67
88
|
* Look up a contract by version string.
|
|
68
89
|
*
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Celilo Module Contract — v2.0
|
|
3
|
+
*
|
|
4
|
+
* Hook SIGNATURES are unchanged from v1.0 — `V2_HOOKS` is `V1_HOOKS`. What
|
|
5
|
+
* changed is what the framework does with a hook's return value: nothing.
|
|
6
|
+
*
|
|
7
|
+
* Under v1.0, a record returned from `validate_config`, `on_install` or
|
|
8
|
+
* `container_created` was swept back into the module's secrets (and, for
|
|
9
|
+
* validate_config, its config). That was the only write channel a hook had,
|
|
10
|
+
* so a hook that needed to persist something it discovered had to return it.
|
|
11
|
+
* The channel is gone (openspec/changes/hook-owned-state, D5): a hook now
|
|
12
|
+
* persists state explicitly through `context.config.set` / `context.secrets.set`,
|
|
13
|
+
* against names its manifest declares.
|
|
14
|
+
*
|
|
15
|
+
* This is a MAJOR bump rather than a v1.x clarification because a module
|
|
16
|
+
* written against the old channel keeps deploying and silently stores
|
|
17
|
+
* nothing — the deploy reports success and the state is dropped. Refusing
|
|
18
|
+
* `celilo_contract: "1.0"` outright (see `RETIRED_CONTRACT_VERSIONS`) turns
|
|
19
|
+
* that silent loss into a message naming the fix, which is the whole point of
|
|
20
|
+
* versioning the contract.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { V1_HOOKS } from './v1';
|
|
24
|
+
import type { ContractHooks } from './v1';
|
|
25
|
+
|
|
26
|
+
/** v2.0 hook signatures — identical to v1.0. The break is behavioural. */
|
|
27
|
+
export const V2_HOOKS: ContractHooks = V1_HOOKS;
|
|
28
|
+
|
|
29
|
+
export const V2_CONTRACT = {
|
|
30
|
+
version: '2.0' as const,
|
|
31
|
+
hooks: V2_HOOKS,
|
|
32
|
+
};
|